update go.mod for release

Merge remote-tracking branch 'upstream/master' into release-cmd/config-v0.0

.golangci-kustomize.yml

@@ -7,13 +7,8 @@ linters:
     - bodyclose
     - deadcode
     - depguard
-    # - dogsled
     - dupl
-    # - errcheck
-    # - funlen
-    # - gochecknoinits
     - goconst
-    # - gocritic
     - gocyclo
     - gofmt
     - goimports
@@ -26,7 +21,6 @@ linters:
     - lll
     - misspell
     - nakedret
-    # - scopelint
     - staticcheck
     - structcheck
     # stylecheck demands that acronyms not be treated as words
@@ -34,10 +28,9 @@ linters:
     # - stylecheck
     - typecheck
     - unconvert
-    - unparam
     - unused
+    - unparam
     - varcheck
-    # - whitespace
 
 linters-settings:
   dupl:

.travis.yml

@@ -36,7 +36,6 @@ install: true
 script:
   - make verify-kustomize
   - ./travis/kyaml-pre-commit.sh
-  - ./travis/check-go-mod.sh
 
 # TBD. Suppressing for now.
 notifications:

Makefile

@@ -17,9 +17,6 @@ verify-kustomize: \
 	test-examples-kustomize-against-HEAD \
 	test-examples-kustomize-against-latest
 
-.PHONY: verify-kustomize-e2e
-verify-kustomize-e2e: test-examples-e2e-kustomize
-
 # Other builds in this repo might want a different linter version.
 # Without one Makefile to rule them all, the different makes
 # cannot assume that golanci-lint is at the version they want
@@ -51,11 +48,6 @@ $(MYGOBIN)/goimports:
 	cd api; \
 	go install golang.org/x/tools/cmd/goimports
 
-# Install resource from whatever is checked out.
-$(MYGOBIN)/resource:
-	cd cmd/resource; \
-	go install .
-
 # To pin pluginator, use this recipe instead:
 #   cd api;
 #   go install sigs.k8s.io/kustomize/pluginator/v2
@@ -201,16 +193,6 @@ test-unit-kustomize-all: \
 test-examples-kustomize-against-HEAD: $(MYGOBIN)/kustomize $(MYGOBIN)/mdrip
 	./hack/testExamplesAgainstKustomize.sh HEAD
 
-.PHONY:
-test-examples-e2e-kustomize: $(MYGOBIN)/mdrip $(MYGOBIN)/kind
-	( \
-		set -e; \
-		/bin/rm -f $(MYGOBIN)/kustomize; \
-		echo "Installing kustomize from ."; \
-		cd kustomize; go install .; cd ..; \
-		./hack/testExamplesE2EAgainstKustomize.sh .; \
-	)
-
 .PHONY:
 test-examples-kustomize-against-latest: $(MYGOBIN)/mdrip
 	( \
@@ -255,16 +237,6 @@ $(MYGOBIN)/helm:
 		rm -rf $$d \
 	)
 
-$(MYGOBIN)/kind:
-	( \
-        set -e; \
-        d=$(shell mktemp -d); cd $$d; \
-        wget -O ./kind https://github.com/kubernetes-sigs/kind/releases/download/v0.7.0/kind-$(shell uname)-amd64; \
-        chmod +x ./kind; \
-        mv ./kind $(MYGOBIN); \
-        rm -rf $$d; \
-	)
-
 .PHONY: clean
 clean: kustomize-external-go-plugin-clean
 	go clean --cache

api/builtins/NamespaceTransformer.go

@@ -52,7 +52,7 @@ func (p *NamespaceTransformerPlugin) Transform(m resmap.ResMap) error {
 
 		matches := m.GetMatchingResourcesByCurrentId(r.CurId().Equals)
 		if len(matches) != 1 {
-			return fmt.Errorf("namespace tranformation produces ID conflict: %+v", matches)
+			return fmt.Errorf("namespace tranformation produces ID conflict: %#v", matches)
 		}
 	}
 	return nil

api/internal/crawl/backend/search_backend.go

@@ -44,7 +44,7 @@ type kustomizeSearch struct {
 // /register: not implemented, but meant as an endpoint for adding new
 // kustomization files to the corpus.
 func NewKustomizeSearch(ctx context.Context) (*kustomizeSearch, error) {
-	idx, err := index.NewKustomizeIndex(ctx, "kustomize")
+	idx, err := index.NewKustomizeIndex(ctx)
 	if err != nil {
 		return nil, err
 	}

api/internal/crawl/cmd/crawler/crawler.go

@@ -2,15 +2,12 @@ package main
 
 import (
 	"context"
-	"flag"
 	"fmt"
 	"log"
 	"net/http"
 	"os"
 	"time"
 
-	"sigs.k8s.io/kustomize/api/internal/crawl/utils"
-
 	"sigs.k8s.io/kustomize/api/internal/crawl/crawler"
 	"sigs.k8s.io/kustomize/api/internal/crawl/crawler/github"
 	"sigs.k8s.io/kustomize/api/internal/crawl/doc"
@@ -28,7 +25,6 @@ const (
 )
 
 type CrawlMode int
-
 const (
 	CrawlUnknown CrawlMode = iota
 	// Crawl all the kustomization files in all the repositories of a Github user
@@ -49,7 +45,7 @@ func NewCrawlMode(s string) CrawlMode {
 		return CrawlUser
 	case "github-repo":
 		return CrawlRepo
-	case "index+github":
+	case "":
 		return CrawlIndexAndGithub
 	case "index":
 		return CrawlIndex
@@ -60,33 +56,30 @@ func NewCrawlMode(s string) CrawlMode {
 	}
 }
 
-func main() {
-	indexNamePtr := flag.String(
-		"index", "kustomize", "The name of the ElasticSearch index.")
-	modePtr := flag.String("mode", "index+github",
-		`The crawling mode, which can be one of [github-user, github-repo, index, github, index+github].
-  * github-user: crawl all the kustomization files in all the repositories of a Github user (--github-user must be specified for this mode).
-  * github-repo: crawl all the kustomization files in a Github repository (--github-repo must be specified for this mode).
-  * index: crawl all the documents in the index.
-  * gihub: crawl all the kustomization files on Github.
-  * index+github: crawl all the documents in the index and crawling all the kustomization files on Github.`)
-	githubUserPtr := flag.String("github-user", "",
-		"A github user name (e.g., kubernetes-sigs). This flag is required for the `github-user` mode.")
-	githubRepoPtr := flag.String("github-repo", "",
-		"A github repository name (e.g., kubernetes-sigs/kustomize). This flag is required for the `github-repo` mode.")
-	flag.Parse()
+func Usage() {
+	fmt.Printf("Usage: %s [mode] [githubUser|githubRepo]\n", os.Args[0])
+	fmt.Printf("\tmode can be one of [github-user, github-repo, index, github]\n")
+	fmt.Printf("%s: crawl all the documents in the index and crawling all the kustomization files on Github\n", os.Args[0])
+	fmt.Printf("%s index: crawl all the documents in the index\n", os.Args[0])
+	fmt.Printf("%s gihub: crawl all the kustomization files on Github\n", os.Args[0])
+	fmt.Printf("%s github-user <github-user>: Crawl all the kustomization files in all the repositories of a Github user\n", os.Args[0])
+	fmt.Printf("\tFor example, %s github-user kubernetes-sigs\n", os.Args[0])
+	fmt.Printf("%s github-repo <github-repo>: Crawl all the kustomization files in a Github repo\n", os.Args[0])
+	fmt.Printf("\tFor example, %s github-repo kubernetes-sigs/kustomize\n", os.Args[0])
+}
 
+func main() {
 	githubToken := os.Getenv(githubAccessTokenVar)
 	if githubToken == "" {
-		log.Printf("Must set the variable '%s' to make github requests.\n",
+		fmt.Printf("Must set the variable '%s' to make github requests.\n",
 			githubAccessTokenVar)
 		return
 	}
 
 	ctx := context.Background()
-	idx, err := index.NewKustomizeIndex(ctx, *indexNamePtr)
+	idx, err := index.NewKustomizeIndex(ctx)
 	if err != nil {
-		log.Printf("Could not create an index: %v\n", err)
+		fmt.Printf("Could not create an index: %v\n", err)
 		return
 	}
 
@@ -94,7 +87,7 @@ func main() {
 	cache, err := redis.DialURL(cacheURL)
 	clientCache := &http.Client{}
 	if err != nil {
-		log.Printf("Error: redis could not make a connection: %v\n", err)
+		fmt.Printf("Error: redis could not make a connection: %v\n", err)
 	} else {
 		clientCache = httpclient.NewClient(cache)
 	}
@@ -115,10 +108,10 @@ func main() {
 		case *doc.KustomizationDocument:
 			switch mode {
 			case index.Delete:
-				log.Printf("Deleting: %v", d)
+				fmt.Println("Deleting: ", d)
 				return idx.Delete(d.ID())
 			default:
-				log.Printf("Inserting: %v", d)
+				fmt.Println("Inserting: ", d)
 				return idx.Put(d.ID(), d)
 			}
 		default:
@@ -126,19 +119,23 @@ func main() {
 		}
 	}
 
-	// seen tracks the IDs of all the documents in the index and their corresponding file types.
+	// seen tracks the IDs of all the documents in the index.
 	// This helps avoid indexing a given document multiple times.
-	seen := utils.NewSeenMap()
+	seen := make(map[string]struct{})
 
-	mode := NewCrawlMode(*modePtr)
+	var mode CrawlMode
+	if len(os.Args) == 1 {
+		mode = CrawlIndexAndGithub
+	} else {
+		mode = NewCrawlMode(os.Args[1])
+	}
 
 	ghCrawlerConstructor := func(user, repo string) crawler.Crawler {
 		if user != "" {
-			return github.NewCrawler(githubToken, retryCount, clientCache,
+			return 	github.NewCrawler(githubToken, retryCount, clientCache,
 				github.QueryWith(
 					github.Filename("kustomization.yaml"),
 					github.Filename("kustomization.yml"),
-					github.Filename("kustomization"),
 					github.User(user)),
 			)
 		} else if repo != "" {
@@ -146,15 +143,13 @@ func main() {
 				github.QueryWith(
 					github.Filename("kustomization.yaml"),
 					github.Filename("kustomization.yml"),
-					github.Filename("kustomization"),
 					github.Repo(repo)),
 			)
 		} else {
 			return github.NewCrawler(githubToken, retryCount, clientCache,
 				github.QueryWith(
 					github.Filename("kustomization.yaml"),
-					github.Filename("kustomization.yml"),
-					github.Filename("kustomization")),
+					github.Filename("kustomization.yml")),
 			)
 		}
 	}
@@ -167,11 +162,11 @@ func main() {
 		it := idx.IterateQuery(query, 10000, 60*time.Second)
 		for it.Next() {
 			for _, hit := range it.Value().Hits.Hits {
-				seedDocs = append(seedDocs, hit.Document.Document.Copy())
+				seedDocs = append(seedDocs, hit.Document.Copy())
 			}
 		}
 		if err := it.Err(); err != nil {
-			log.Fatalf("getSeedDocsFunc Error iterating: %v\n", err)
+			fmt.Printf("Error iterating: %v\n", err)
 		}
 	}
 
@@ -187,29 +182,23 @@ func main() {
 		crawler.CrawlFromSeed(ctx, seedDocs, crawlers, docConverter, indexFunc, seen)
 	case CrawlGithub:
 		crawlers := []crawler.Crawler{ghCrawlerConstructor("", "")}
-		// add all the documents in the index into seen.
-		// this greatly reduces the time overhead of CrawlGithub.
-		getSeedDocsFunc()
-		for _, d := range seedDocs {
-			seen[d.ID()] = d.FileType
-		}
 		crawler.CrawlGithub(ctx, crawlers, docConverter, indexFunc, seen)
 	case CrawlUser:
-		if *githubUserPtr == "" {
-			flag.Usage()
-			log.Fatalf("Please specify a github user with the github-user flag!")
+		if len(os.Args) < 3 {
+			Usage()
+			log.Fatalf("Please specify a github user!")
 		}
-		crawlers := []crawler.Crawler{ghCrawlerConstructor(*githubUserPtr, "")}
+		crawlers := []crawler.Crawler{ghCrawlerConstructor(os.Args[2], "")}
 		crawler.CrawlGithub(ctx, crawlers, docConverter, indexFunc, seen)
 	case CrawlRepo:
-		if *githubRepoPtr == "" {
-			flag.Usage()
-			log.Fatalf("Please specify a github repository with the github-repo flag!")
+		if len(os.Args) < 3 {
+			Usage()
+			log.Fatalf("Please specify a github repo!")
 		}
-		crawlers := []crawler.Crawler{ghCrawlerConstructor("", *githubRepoPtr)}
+		crawlers := []crawler.Crawler{ghCrawlerConstructor("", os.Args[2])}
 		crawler.CrawlGithub(ctx, crawlers, docConverter, indexFunc, seen)
 	case CrawlUnknown:
-		flag.Usage()
-		log.Fatalf("The --mode flag must be one of [github-user, github-repo, index, github, index+github].")
+		Usage()
+		log.Fatalf("The crawler mode must be one of [github-user, github-repo, index, github]")
 	}
 }

api/internal/crawl/cmd/kustomize_stats/Dockerfile

@@ -1,14 +0,0 @@
-FROM golang:1.11 AS build
-
-ARG GO111MODULE=on
-
-WORKDIR /go/src/sigs.k8s.io/kustomize/api/internal/crawl
-COPY . /go/src/sigs.k8s.io/kustomize/api/internal/crawl
-
-RUN go mod download
-RUN CGO_ENABLED=0 go install ./cmd/kustomize_stats
-
-FROM scratch
-COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
-COPY --from=build /go/bin/kustomize_stats /
-ENTRYPOINT ["/kustomize_stats"]

api/internal/crawl/cmd/kustomize_stats/main.go

@@ -1,254 +0,0 @@
-package main
-
-import (
-	"context"
-	"crypto/sha256"
-	"flag"
-	"fmt"
-	"log"
-	"sort"
-	"time"
-
-	"sigs.k8s.io/kustomize/api/internal/crawl/doc"
-
-	"sigs.k8s.io/kustomize/api/internal/crawl/index"
-)
-
-const (
-	githubAccessTokenVar = "GITHUB_ACCESS_TOKEN"
-	retryCount           = 3
-)
-
-// iterateArr adds each item in arr into countMap.
-func iterateArr(arr []string, countMap map[string]int) {
-	for _, item := range arr {
-		if _, ok := countMap[item]; !ok {
-			countMap[item] = 1
-		} else {
-			countMap[item]++
-		}
-	}
-
-}
-
-// SortMapKeyByValueInt takes a map as its input, sorts its keys according to their values
-// in the map, and outputs the sorted keys as a slice.
-func SortMapKeyByValueInt(m map[string]int) []string {
-	keys := make([]string, 0, len(m))
-	for key := range m {
-		keys = append(keys, key)
-	}
-	// sort keys according to their values in the map m
-	sort.Slice(keys, func(i, j int) bool { return m[keys[i]] > m[keys[j]] })
-	return keys
-}
-
-// SortMapKeyByValue takes a map as its input, sorts its keys according to their values
-// in the map, and outputs the sorted keys as a slice.
-func SortMapKeyByValueLen(m map[string][]string) []string {
-	keys := make([]string, 0, len(m))
-	for key := range m {
-		keys = append(keys, key)
-	}
-	// sort keys according to their values in the map m
-	sort.Slice(keys, func(i, j int) bool { return len(m[keys[i]]) > len(m[keys[j]]) })
-	return keys
-}
-
-func GeneratorOrTransformerStats(docs []*doc.KustomizationDocument) {
-	n := len(docs)
-	if n == 0 {
-		return
-	}
-
-	fileType := docs[0].FileType
-	fmt.Printf("There are totally %d %s files.\n", n, fileType)
-
-	GitRepositorySummary(docs, fileType)
-
-	// key of kindToUrls: a string in the KustomizationDocument.Kinds field
-	// value of kindToUrls: a slice of string urls defining a given kind.
-	kindToUrls := make(map[string][]string)
-
-	for _, d := range docs {
-		url := fmt.Sprintf("%s/blob/%s/%s", d.RepositoryURL, d.DefaultBranch, d.FilePath)
-		for _, kind := range d.Kinds {
-			if _, ok := kindToUrls[kind]; !ok {
-				kindToUrls[kind] = []string{url}
-			} else {
-				kindToUrls[kind] = append(kindToUrls[kind], url)
-			}
-		}
-	}
-	fmt.Printf("There are totally %d kinds of %s\n", len(kindToUrls), fileType)
-	sortedKeys := SortMapKeyByValueLen(kindToUrls)
-	for _, k := range sortedKeys {
-		sort.Strings(kindToUrls[k])
-		fmt.Printf("%s kind %s appears %d times\n", fileType, k, len(kindToUrls[k]))
-		for _, url := range kindToUrls[k] {
-			fmt.Printf("%s\n", url)
-		}
-	}
-}
-
-// GitRepositorySummary counts the distribution of docs:
-// 1) how many git repositories are these docs from?
-// 2) how many docs are from each git repository?
-func GitRepositorySummary(docs []*doc.KustomizationDocument, fileType string) {
-	m := make(map[string]int)
-	for _, d := range docs {
-		if _, ok := m[d.RepositoryURL]; ok {
-			m[d.RepositoryURL]++
-		} else {
-			m[d.RepositoryURL] = 1
-		}
-	}
-	sortedKeys := SortMapKeyByValueInt(m)
-	topN := 10
-	i := 0
-	for _, k := range sortedKeys {
-		if i >= topN {
-			break
-		}
-		fmt.Printf("%d %s are from %s\n", m[k], fileType, k)
-		i++
-	}
-}
-
-func main() {
-	topKindsPtr := flag.Int(
-		"kinds", -1,
-		`the number of kubernetes object kinds to be listed according to their popularities.
-By default, all the kinds will be listed.
-If you only want to list the 10 most popular kinds, set the flag to 10.`)
-	topIdentifiersPtr := flag.Int(
-		"identifiers", -1,
-		`the number of identifiers to be listed according to their popularities.
-By default, all the identifiers will be listed.
-If you only want to list the 10 most popular identifiers, set the flag to 10.`)
-	topKustomizeFeaturesPtr := flag.Int(
-		"kustomize-features", -1,
-		`the number of kustomize features to be listed according to their popularities.
-By default, all the features will be listed.
-If you only want to list the 10 most popular features, set the flag to 10.`)
-	indexNamePtr := flag.String(
-		"index", "kustomize", "The name of the ElasticSearch index.")
-	flag.Parse()
-
-	ctx := context.Background()
-	idx, err := index.NewKustomizeIndex(ctx, *indexNamePtr)
-	if err != nil {
-		log.Fatalf("Could not create an index: %v\n", err)
-	}
-
-	// count tracks the number of documents in the index
-	count := 0
-
-	// kustomizationFilecount tracks the number of kustomization files in the index
-	kustomizationFilecount := 0
-
-	kindsMap := make(map[string]int)
-	identifiersMap := make(map[string]int)
-	kustomizeIdentifiersMap := make(map[string]int)
-
-	// ids tracks the unique IDs of the documents in the index
-	ids := make(map[string]struct{})
-
-	// generatorFiles include all the non-kustomization files whose FileType is generator
-	generatorFiles := make([]*doc.KustomizationDocument, 0)
-
-	// transformersFiles include all the non-kustomization files whose FileType is transformer
-	transformersFiles := make([]*doc.KustomizationDocument, 0)
-
-	checksums := make(map[string]int)
-
-	// get all the documents in the index
-	query := []byte(`{ "query":{ "match_all":{} } }`)
-	it := idx.IterateQuery(query, 10000, 60*time.Second)
-	for it.Next() {
-		for _, hit := range it.Value().Hits.Hits {
-			sum := fmt.Sprintf("%x", sha256.Sum256([]byte(hit.Document.DocumentData)))
-			if _, ok := checksums[sum]; ok {
-				checksums[sum]++
-			} else {
-				checksums[sum] = 1
-			}
-
-			// check whether there is any duplicate IDs in the index
-			if _, ok := ids[hit.ID]; !ok {
-				ids[hit.ID] = struct{}{}
-			} else {
-				log.Printf("Found duplicate ID (%s)\n", hit.ID)
-			}
-
-			count++
-			iterateArr(hit.Document.Kinds, kindsMap)
-			iterateArr(hit.Document.Identifiers, identifiersMap)
-
-			if doc.IsKustomizationFile(hit.Document.FilePath) {
-				kustomizationFilecount++
-				iterateArr(hit.Document.Identifiers, kustomizeIdentifiersMap)
-
-			} else {
-				switch hit.Document.FileType {
-				case "generator":
-					generatorFiles = append(generatorFiles, hit.Document.Copy())
-				case "transformer":
-					transformersFiles = append(transformersFiles, hit.Document.Copy())
-				}
-			}
-		}
-	}
-
-	if err := it.Err(); err != nil {
-		log.Fatalf("Error iterating: %v\n", err)
-	}
-
-	sortedKindsMapKeys := SortMapKeyByValueInt(kindsMap)
-	sortedIdentifiersMapKeys := SortMapKeyByValueInt(identifiersMap)
-	sortedKustomizeIdentifiersMapKeys := SortMapKeyByValueInt(kustomizeIdentifiersMap)
-
-	fmt.Printf(`The count of unique document IDs in the kustomize index: %d
-There are %d documents in the kustomize index.
-%d kinds of kubernetes objects are customized:`, len(ids), count, len(kindsMap))
-	fmt.Printf("\n")
-
-	kindCount := 0
-	for _, key := range sortedKindsMapKeys {
-		if *topKindsPtr < 0 || (*topKindsPtr >= 0 && kindCount < *topKindsPtr) {
-			fmt.Printf("\tkind `%s` is customimzed in %d documents\n", key, kindsMap[key])
-			kindCount++
-		}
-	}
-
-	fmt.Printf("%d kinds of identifiers are found:\n", len(identifiersMap))
-	identifierCount := 0
-	for _, key := range sortedIdentifiersMapKeys {
-		if *topIdentifiersPtr < 0 || (*topIdentifiersPtr >= 0 && identifierCount < *topIdentifiersPtr) {
-			fmt.Printf("\tidentifier `%s` appears in %d documents\n", key, identifiersMap[key])
-			identifierCount++
-		}
-	}
-
-	fmt.Printf(`There are %d kustomization files in the kustomize index.
-%d kinds of kustomize features are found:`, kustomizationFilecount, len(kustomizeIdentifiersMap))
-	fmt.Printf("\n")
-	kustomizeFeatureCount := 0
-	for _, key := range sortedKustomizeIdentifiersMapKeys {
-		if *topKustomizeFeaturesPtr < 0 || (*topKustomizeFeaturesPtr >= 0 && kustomizeFeatureCount < *topKustomizeFeaturesPtr) {
-			fmt.Printf("\tfeature `%s` is used in %d documents\n", key, kustomizeIdentifiersMap[key])
-			kustomizeFeatureCount++
-		}
-	}
-
-	GeneratorOrTransformerStats(generatorFiles)
-	GeneratorOrTransformerStats(transformersFiles)
-
-	fmt.Printf("There are total %d checksums of document contents\n", len(checksums))
-	sortedChecksums := SortMapKeyByValueInt(checksums)
-	sortedChecksums = sortedChecksums[:20]
-	fmt.Printf("The top 20 checksums are:\n")
-	for _, key := range sortedChecksums {
-		fmt.Printf("checksum %s apprears %d\n", key, checksums[key])
-	}
-}

api/internal/crawl/cmd/log-parser/README.md

@@ -1,8 +0,0 @@
-This binary takes as its input a json file including GKE logs (which can be
-[exported](https://cloud.google.com/logging/docs/export/configure_export_v2) into 
-[Cloud Storage](https://cloud.google.com/storage/docs/)), 
-and extracts the `textPayload` field of each log entry.
-
-Here is an log entry example:
-
-{"insertId":"1sxuh4jg5lw6w10","labels":{"compute.googleapis.com/resource_name":"gke-crawler2-default-pool-5e55ea05-gzgv","container.googleapis.com/namespace_name":"default","container.googleapis.com/pod_name":"kustomize-stats-5bczg","container.googleapis.com/stream":"stdout"},"logName":"projects/haiyanmeng-gke-dev/logs/kustomize-stats","receiveTimestamp":"2020-01-06T23:33:07.012831742Z","resource":{"labels":{"cluster_name":"crawler2","container_name":"kustomize-stats","instance_id":"8183086081854184383","namespace_id":"default","pod_id":"kustomize-stats-5bczg","project_id":"haiyanmeng-gke-dev","zone":"us-central1-a"},"type":"container"},"severity":"INFO","textPayload":"The kustomize index already exists\n","timestamp":"2020-01-06T23:32:46.628930547Z"}

api/internal/crawl/cmd/log-parser/main.go

@@ -1,49 +0,0 @@
-package main
-
-import (
-	"bufio"
-	"encoding/json"
-	"fmt"
-	"log"
-	"os"
-)
-
-func main() {
-	if len(os.Args) != 2 {
-		log.Fatalf("The usage of the command is: \n\t%s <log-file.json>", os.Args[0])
-	}
-
-	file, err := os.Open(os.Args[1])
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	closeFile := func(file *os.File) {
-		if err := file.Close(); err != nil {
-			log.Fatal(err)
-		}
-	}
-	defer closeFile(file)
-
-	scanner := bufio.NewScanner(file)
-	for scanner.Scan() {
-		line := scanner.Text()
-
-		var entry interface{}
-		if err := json.Unmarshal([]byte(line), &entry); err != nil {
-			log.Printf("failed to unmarshal a log entry: %s\n", line)
-		}
-
-		m := entry.(map[string]interface{})
-		if payload, ok := m["textPayload"]; ok {
-			// use fmt.Printf here instead of log.Printf to avoid the time and code location info the log package provides
-			fmt.Printf("%s", payload)
-		} else {
-			log.Printf("the log entry does not have the `textPayload` field: %s\n", line)
-		}
-	}
-
-	if err := scanner.Err(); err != nil {
-		log.Fatal(err)
-	}
-}

api/internal/crawl/config/base/kustomization.yaml

@@ -2,4 +2,5 @@ configmapGenerator:
 - name: elasticsearch-config
   literals:
   - es-url="http://esbasic-master:9200"
+  - kustomize-index-name="kustomize"
   - plugin-index-name="plugin"

api/internal/crawl/config/crawler/cronjob/cronjob.yaml

@@ -1,10 +1,9 @@
 apiVersion: batch/v1beta1
 kind: CronJob
 metadata:
-  name: crawler-cronjob
+  name: crawler
 spec:
-  # run the cronjob at 00:00 every 7 days
-  schedule: "0 0 */7 * *"
+  schedule: "5 0 * * */1"
   jobTemplate:
     spec:
       template:
@@ -12,9 +11,7 @@ spec:
           restartPolicy: OnFailure
           containers:
           - name: crawler
-            image: gcr.io/haiyanmeng-gke-dev/crawler:v1
-            command: ["/crawler"]
-            args: ["--mode=index+github", "--github-repo=kubernetes-sigs/kustomize", "--index=kustomize"]
+            image: gcr.io/kustomize-search/crawler:latest
             imagePullPolicy: Always
             env:
             - name: GITHUB_ACCESS_TOKEN

api/internal/crawl/config/crawler/job/README.md

@@ -1,4 +1,4 @@
-The crawler job can run in one of the following mode:
+There are three ways of running the crawler job.
 
 # Crawling all the documents in the index and crawling all the kustomization files on Github
 
@@ -7,13 +7,14 @@ of the container should be:
 
 ```
         command: ["/crawler"]
+        args: []
 ```
 
 Or
 
 ```
         command: ["/crawler"]
-        args: ["--mode=index+github"]
+        args: [""]
 ```
 
 # Crawling all the documents in the index
@@ -22,7 +23,7 @@ The `command` and `args` field of the container should be:
 
 ```
         command: ["/crawler"]
-        args: ["--mode=index"]
+        args: ["index"]
 ```
 
 # Crawling all the kustomization files on Github
@@ -31,7 +32,7 @@ The `command` and `args` field of the container should be:
 
 ```
         command: ["/crawler"]
-        args: ["--mode=github"]
+        args: ["github"]
 ```
 
 # Crawling all the kustomization files in a Github repo
@@ -40,7 +41,7 @@ The `command` and `args` field of the container should be like:
 
 ```
         command: ["/crawler"]
-        args: ["--mode=github-repo", "--github-repo=kubernetes-sigs/kustomize"]
+        args: ["github-repo", "kubernetes-sigs/kustomize"]
 ```
 
 # Crawling all the kustomization files in all the repositories of a Github user
@@ -49,5 +50,5 @@ The `command` and `args` field of the container should be like:
 
 ```
         command: ["/crawler"]
-        args: ["--github-user", "--github-user=kubernetes-sigs"]
+        args: ["github-user", "kubernetes-sigs"]
 ```

api/internal/crawl/config/crawler/job/job.yaml

@@ -11,7 +11,7 @@ spec:
         image: gcr.io/haiyanmeng-gke-dev/crawler:v1
         imagePullPolicy: Always
         command: ["/crawler"]
-        args: ["--mode=github-repo", "--github-repo=kubernetes-sigs/kustomize", "--index=kustomize"]
+        args: ["github-repo", "kubernetes-sigs/kustomize"]
         env:
         - name: GITHUB_ACCESS_TOKEN
           valueFrom:

api/internal/crawl/config/crawler/kustomize_stats/job.yaml

@@ -1,25 +0,0 @@
-apiVersion: batch/v1
-kind: Job
-metadata:
-  name: kustomize-stats
-spec:
-  template:
-    spec:
-      restartPolicy: OnFailure
-      containers:
-      - name: kustomize-stats
-        image: gcr.io/haiyanmeng-gke-dev/kustomize_stats:v1
-        imagePullPolicy: Always
-        command: ["/kustomize_stats"]
-        args: ["--index=kustomize", "--kinds=50", "--identifiers=50", "--kustomize-features=50"]
-        env:
-        - name: GITHUB_ACCESS_TOKEN
-          valueFrom:
-            secretKeyRef:
-              name: github-access-token
-              key: token
-        - name: ELASTICSEARCH_URL
-          valueFrom:
-            configMapKeyRef:
-              name: elasticsearch-config
-              key: es-url

api/internal/crawl/config/crawler/kustomize_stats/kustomization.yaml

@@ -1,3 +0,0 @@
-resources:
-- ../base
-- job.yaml

api/internal/crawl/config/elastic/esbackup.yaml

@@ -1,16 +0,0 @@
-# Creating `esbackup/kustomize-backbup` will create the `kustomize-backup` snapshot repository.
-# Deleting `esbackup/kustomize-backbup` will delete the `kustomize-backup` snapshot repository and all the snapshots in the repository.
-# Deleting `esbackup/kustomize-backbup` will NOT delete essnapshot and esrestore objects.
-apiVersion: elasticsearch.cloud.google.com/v1alpha1
-kind: ESBackup
-metadata:
-  name: kustomize-backup
-spec:
-  storage:
-    gcs:
-      bucket: kustomize-backup
-      path: kustomize
-      secret:
-        name: kustomizesa
-  escluster:
-    name: esbasic

api/internal/crawl/config/elastic/escluster.yaml

@@ -8,13 +8,6 @@ spec:
     - repository-gcs
     - ingest-user-agent
     - ingest-geoip
-  # To set `gcpserviceaccount`,
-  # First, create and download a GCP service account into a json file, named `sakey.json` following the instruction:
-  #     https://www.elastic.co/guide/en/elasticsearch/plugins/6.5/repository-gcs-usage.html#repository-gcs-using-service-account
-  # Second, create a secret for the service account using the following command:
-  #     $ kubectl create secret generic kustomizesa --from-file=./sakey.json
-  gcpserviceaccount:
-    name: kustomizesa
   config:
     env:
       example: test

api/internal/crawl/config/elastic/esrestore.yaml

@@ -1,16 +0,0 @@
-# Creating `esrestore/kustomize-restore` will restore the `kuostmize` index in the `kustomize-snapshot` snapshot to a new index named `kusotmize-restore`.
-# Deleting `esrestore/kustomize-restore` will not delete the restored index.
-# Deleting `esrestore/kustomize-restore` should happen before deleting `essnapshot/kustomize-snapshot`.
-apiVersion: elasticsearch.cloud.google.com/v1alpha1
-kind: ESRestore
-metadata:
-  name: kustomize-restore
-spec:
-  include_global_state: true
-  ignore_unavailable: true
-  rename_pattern: kustomize
-  rename_replacement: kustomize-restore
-  essnapshot:
-    name: kustomize-snapshot
-  escluster:
-    name: esbasic

api/internal/crawl/config/elastic/essnaptshot.yaml

@@ -1,15 +0,0 @@
-# Creating `essnapshot/kustomize-snapshot` will create a snapshot named `kustomize-snapshot` in the `kustomize-backup` snapshot repository.
-# Deleting `essnapshot/kustomize-snapshot` will delete the snapshot.
-# Deleting `essnapshot/kustomize-snapshot` should happen before deleting `esbackup/kustomize-backup`.
-apiVersion: elasticsearch.cloud.google.com/v1alpha1
-kind: ESSnapshot
-metadata:
-  name: kustomize-snapshot
-spec:
-  # indices are optional. If not specified all indices are selected.
-  indices:
-  - kustomize
-  include_global_state: true
-  ignore_unavailable: true
-  esbackup:
-    name: kustomize-backup

api/internal/crawl/crawler/crawler.go

@@ -10,8 +10,6 @@ import (
 	"os"
 	"sync"
 
-	"sigs.k8s.io/kustomize/api/internal/crawl/utils"
-
 	"sigs.k8s.io/kustomize/api/internal/crawl/index"
 
 	_ "github.com/gomodule/redigo/redis"
@@ -31,15 +29,13 @@ type Crawler interface {
 	// Crawl returns when it is done processing. This method does not take
 	// ownership of the channel. The channel is write only, and it
 	// designates where the crawler should forward the documents.
-	Crawl(ctx context.Context, output chan<- CrawledDocument, seen utils.SeenMap) error
+	Crawl(ctx context.Context, output chan<- CrawledDocument) error
 
 	// Get the document data given the FilePath, Repo, and Ref/Tag/Branch.
 	FetchDocument(context.Context, *doc.Document) error
 	// Write to the document what the created time is.
 	SetCreated(context.Context, *doc.Document) error
 
-	SetDefaultBranch(*doc.Document)
-
 	Match(*doc.Document) bool
 }
 
@@ -47,12 +43,7 @@ type CrawledDocument interface {
 	ID() string
 	GetDocument() *doc.Document
 	// Get all the Documents directly referred in a Document.
-	// For a Document representing a non-kustomization file, an empty slice will be returned.
-	// For a Document representing a kustomization file:
-	// the `includeResources` parameter determines whether the documents referred in the `resources` field are returned or not;
-	// the `includeTransformers` parameter determines whether the documents referred in the `transformers` field are returned or not;
-	// the `includeGenerators` parameter determines whether the documents referred in the `generators` field are returned or not.
-	GetResources(includeResources, includeTransformers, includeGenerators bool) ([]*doc.Document, error)
+	GetResources() ([]*doc.Document, error)
 	WasCached() bool
 }
 
@@ -78,27 +69,25 @@ func findMatch(d *doc.Document, crawlers []Crawler) Crawler {
 }
 
 func addBranches(cdoc CrawledDocument, match Crawler, indx IndexFunc,
-	seen utils.SeenMap, stack *CrawlSeed) {
+	seen map[string]struct{}, stack *CrawlSeed) {
 
-	seen.Set(cdoc.ID(), cdoc.GetDocument().FileType)
-
-	match.SetDefaultBranch(cdoc.GetDocument())
+	seen[cdoc.ID()] = struct{}{}
 
 	// Insert into index
 	if err := indx(cdoc, index.InsertOrUpdate); err != nil {
-		logger.Printf("Failed to insert or update doc(%s): %v",
-			cdoc.GetDocument().Path(), err)
+		logger.Printf("Failed to insert or update %s %s: %v",
+			cdoc.GetDocument().RepositoryURL, cdoc.GetDocument().FilePath, err)
 		return
 	}
 
-	deps, err := cdoc.GetResources(true, true, true)
+	deps, err := cdoc.GetResources()
 	if err != nil {
 		logger.Println(err)
 		return
 	}
 
 	for _, dep := range deps {
-		if seen.Seen(dep.ID()) && seen.Value(dep.ID()) == dep.FileType {
+		if _, ok := seen[dep.ID()]; ok {
 			continue
 		}
 		*stack = append(*stack, dep)
@@ -106,7 +95,7 @@ func addBranches(cdoc CrawledDocument, match Crawler, indx IndexFunc,
 }
 
 func doCrawl(ctx context.Context, docsPtr *CrawlSeed, crawlers []Crawler, conv Converter, indx IndexFunc,
-	seen utils.SeenMap, stack *CrawlSeed, refreshDoc bool, updateFileType bool) {
+	seen map[string]struct{}, stack *CrawlSeed) {
 
 	UpdatedDocCount := 0
 	seenDocCount := 0
@@ -116,7 +105,6 @@ func doCrawl(ctx context.Context, docsPtr *CrawlSeed, crawlers []Crawler, conv C
 	SetCreatedErrCount := 0
 	convErrCount := 0
 	deleteDocCount := 0
-	crawledDocCount := 0
 
 	// During the execution of the for loop, more Documents may be added into (*docsPtr).
 	for len(*docsPtr) > 0 {
@@ -126,19 +114,13 @@ func doCrawl(ctx context.Context, docsPtr *CrawlSeed, crawlers []Crawler, conv C
 		// remove the last Document in (*docPtr)
 		*docsPtr = (*docsPtr)[:(len(*docsPtr) - 1)]
 
-		crawledDocCount++
-		logger.Printf("Crawling doc %d: %s", crawledDocCount, tail.Path())
-
-		if seen.Seen(tail.ID()) {
-			if !updateFileType || seen.Value(tail.ID()) == tail.FileType {
-				logger.Printf("this doc has been seen before")
-				seenDocCount++
-				continue
-			}
+		if _, ok := seen[tail.ID()]; ok {
+			seenDocCount++
+			continue
 		}
 
 		if tail.WasCached() {
-			logger.Printf("doc(%s) is cached already", tail.Path())
+			logger.Printf("%s %s is cached already", tail.RepositoryURL, tail.FilePath)
 			cachedDocCount++
 			continue
 		}
@@ -150,52 +132,36 @@ func doCrawl(ctx context.Context, docsPtr *CrawlSeed, crawlers []Crawler, conv C
 			continue
 		}
 
-		if tail.User == "" {
-			tail.User = doc.UserName(tail.RepositoryURL)
-		}
-
-		// If the Document represents a kustomization root, FetchDcoument will change
-		// the `filePath` field of the Document by adding `kustomization.yaml` or
-		// `kustomization.yml` or `kustomization` into the the field.
-		// Therefore, it is necessary to add the ID of the Document into seen before
-		// calling FetchDocument. Otherwise, the binary may enter into an infinite loop
-		// if a kustomization file points to its kustmozation root in its `resources` or
-		// `bases` field.
-		seen.Set(tail.ID(), tail.FileType)
-
-		if refreshDoc || tail.DefaultBranch == "" {
-			match.SetDefaultBranch(tail)
-		}
-
-		if refreshDoc || tail.DocumentData == "" {
-			if err := match.FetchDocument(ctx, tail); err != nil {
-				logger.Printf("FetchDocument failed on doc(%s): %v", tail.Path(), err)
-				FetchDocumentErrCount++
-				// delete the document from the index
-				cdoc := &doc.KustomizationDocument{
-					Document: *tail,
-				}
-				seen.Set(cdoc.ID(), tail.FileType)
-				if err := indx(cdoc, index.Delete); err != nil {
-					logger.Printf("Failed to delete doc(%s): %v", cdoc.Path(), err)
-				}
-				deleteDocCount++
-				continue
+		logger.Println("Crawling ", tail.RepositoryURL, tail.FilePath)
+		if err := match.FetchDocument(ctx, tail); err != nil {
+			logger.Printf("FetchDocument failed on %s %s: %v",
+				tail.RepositoryURL, tail.FilePath, err)
+			FetchDocumentErrCount++
+			// delete the document from the index
+			cdoc := &doc.KustomizationDocument{
+				Document: *tail,
 			}
+			seen[cdoc.ID()] = struct{}{}
+			if err := indx(cdoc, index.Delete); err != nil {
+				logger.Printf("Failed to delete %s %s: %v",
+					cdoc.RepositoryURL, cdoc.FilePath, err)
+			}
+			deleteDocCount++
+			continue
 		}
 
-		if refreshDoc || tail.CreationTime == nil {
-			if err := match.SetCreated(ctx, tail); err != nil {
-				logger.Printf("SetCreated failed on doc(%s): %v", tail.Path(), err)
-				SetCreatedErrCount++
-			}
+		if err := match.SetCreated(ctx, tail); err != nil {
+			logger.Printf("SetCreated failed on %s %s: %v",
+				tail.RepositoryURL, tail.FilePath, err)
+			SetCreatedErrCount++
 		}
 
 		cdoc, err := conv(tail)
 		// If conv returns an error, cdoc can still be added into the index so that
 		// cdoc.Document can be searched.
 		if err != nil {
-			logger.Printf("conv failed on doc(%s): %v", tail.Path(), err)
+			logger.Printf("conv failed on %s %s: %v",
+				tail.RepositoryURL, tail.FilePath, err)
 			convErrCount++
 		}
 
@@ -216,7 +182,7 @@ func doCrawl(ctx context.Context, docsPtr *CrawlSeed, crawlers []Crawler, conv C
 // CrawlFromSeed updates all the documents in seed, and crawls all the new
 // documents referred in the seed.
 func CrawlFromSeed(ctx context.Context, seed CrawlSeed, crawlers []Crawler,
-	conv Converter, indx IndexFunc, seen utils.SeenMap) {
+	conv Converter, indx IndexFunc, seen map[string]struct{}) {
 
 	// stack tracks the documents directly referred in other documents.
 	stack := make(CrawlSeed, 0)
@@ -224,14 +190,14 @@ func CrawlFromSeed(ctx context.Context, seed CrawlSeed, crawlers []Crawler,
 	// Exploit seed to update bulk of corpus.
 	logger.Printf("updating %d documents from seed\n", len(seed))
 	// each unique document in seed will be crawled once.
-	doCrawl(ctx, &seed, crawlers, conv, indx, seen, &stack, true, false)
+	doCrawl(ctx, &seed, crawlers, conv, indx, seen, &stack)
 
 	// Traverse any new documents added while updating corpus.
 	logger.Printf("crawling %d new documents found in the seed\n", len(stack))
 	// While crawling each document in stack, the documents directly referred in the document
 	// will be added into stack.
 	// After this statement is done, stack will become empty.
-	doCrawl(ctx, &stack, crawlers, conv, indx, seen, &stack, false, true)
+	doCrawl(ctx, &stack, crawlers, conv, indx, seen, &stack)
 }
 
 // CrawlGithubRunner is a blocking function and only returns once all of the
@@ -252,7 +218,7 @@ func CrawlFromSeed(ctx context.Context, seed CrawlSeed, crawlers []Crawler,
 // from the seed will be processed before any other documents from the
 // crawlers.
 func CrawlGithubRunner(ctx context.Context, output chan<- CrawledDocument,
-	crawlers []Crawler, seen utils.SeenMap) []error {
+	crawlers []Crawler) []error {
 
 	errs := make([]error, len(crawlers))
 	wg := sync.WaitGroup{}
@@ -286,7 +252,7 @@ func CrawlGithubRunner(ctx context.Context, output chan<- CrawledDocument,
 				}
 			}()
 			defer close(docs)
-			errs[idx] = crawler.Crawl(ctx, docs, seen)
+			errs[idx] = crawler.Crawl(ctx, docs)
 		}(i, crawler, docs) // Copies the index and the crawler
 	}
 
@@ -296,7 +262,7 @@ func CrawlGithubRunner(ctx context.Context, output chan<- CrawledDocument,
 
 // CrawlGithub crawls all the kustomization files on Github.
 func CrawlGithub(ctx context.Context, crawlers []Crawler, conv Converter,
-	indx IndexFunc, seen utils.SeenMap) {
+	indx IndexFunc, seen map[string]struct{}) {
 	// stack tracks the documents directly referred in other documents.
 	stack := make(CrawlSeed, 0)
 
@@ -308,14 +274,8 @@ func CrawlGithub(ctx context.Context, crawlers []Crawler, conv Converter,
 	wg.Add(1)
 	go func() {
 		defer wg.Done()
-		docCount := 0
 		for cdoc := range ch {
-			docCount++
-			logger.Printf("Processing doc %d found on Github", docCount)
-			// all the docs here are kustomization files found by querying Github, and
-			// their `FileType` fields all should be empty.
-			if seen.Seen(cdoc.ID()) {
-				logger.Printf("the doc has been seen before")
+			if _, ok := seen[cdoc.ID()]; ok {
 				continue
 			}
 			match := findMatch(cdoc.GetDocument(), crawlers)
@@ -329,7 +289,7 @@ func CrawlGithub(ctx context.Context, crawlers []Crawler, conv Converter,
 	}()
 
 	logger.Println("processing the documents found from crawling github")
-	if errs := CrawlGithubRunner(ctx, ch, crawlers, seen); errs != nil {
+	if errs := CrawlGithubRunner(ctx, ch, crawlers); errs != nil {
 		for _, err := range errs {
 			logIfErr(err)
 		}
@@ -340,5 +300,5 @@ func CrawlGithub(ctx context.Context, crawlers []Crawler, conv Converter,
 	// Handle deps of newly discovered documents.
 	logger.Printf("crawling the %d new documents referred by other documents",
 		len(stack))
-	doCrawl(ctx, &stack, crawlers, conv, indx, seen, &stack, false, true)
+	doCrawl(ctx, &stack, crawlers, conv, indx, seen, &stack)
 }

api/internal/crawl/crawler/crawler_test.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	"log"
 	"reflect"
 	"sort"
 	"strings"
@@ -12,8 +11,6 @@ import (
 	"testing"
 	"time"
 
-	"sigs.k8s.io/kustomize/api/internal/crawl/utils"
-
 	"sigs.k8s.io/kustomize/api/internal/crawl/index"
 
 	"sigs.k8s.io/kustomize/api/internal/crawl/doc"
@@ -37,8 +34,6 @@ func (c testCrawler) Match(d *doc.Document) bool {
 	return d != nil
 }
 
-func (c testCrawler) SetDefaultBranch(d *doc.Document) {}
-
 func (c testCrawler) FetchDocument(_ context.Context, d *doc.Document) error {
 	if i, ok := c.lukp[d.ID()]; ok {
 		d.DocumentData = c.docs[i].DocumentData
@@ -80,7 +75,7 @@ func newCrawler(matchPrefix string, err error,
 
 // Crawl implements the Crawler interface for testing.
 func (c testCrawler) Crawl(_ context.Context,
-	output chan<- CrawledDocument, _ utils.SeenMap) error {
+	output chan<- CrawledDocument) error {
 
 	for i, d := range c.docs {
 		isResource := true
@@ -115,7 +110,7 @@ func (s sortableDocs) Len() int {
 }
 
 func TestCrawlGithubRunner(t *testing.T) {
-	log.Println("testing CrawlGithubRunner")
+	fmt.Println("testing CrawlGithubRunner")
 	tests := []struct {
 		tc   []Crawler
 		errs []error
@@ -186,9 +181,8 @@ func TestCrawlGithubRunner(t *testing.T) {
 			defer close(output)
 			defer wg.Done()
 
-			seen := utils.NewSeenMap()
 			errs := CrawlGithubRunner(context.Background(),
-				output, test.tc, seen)
+				output, test.tc)
 
 			// Check that errors are returned as they should be.
 			if !reflect.DeepEqual(errs, test.errs) {
@@ -221,7 +215,7 @@ func TestCrawlGithubRunner(t *testing.T) {
 }
 
 func TestCrawlFromSeed(t *testing.T) {
-	log.Println("testing CrawlFromSeed")
+	fmt.Println("testing CrawlFromSeed")
 
 	tests := []struct {
 		seed    CrawlSeed
@@ -328,7 +322,7 @@ resources:
 				visited[d.ID()]++
 				return nil
 			},
-			utils.NewSeenMap(),
+			make(map[string]struct{}),
 		)
 		if lv, lc := len(visited), len(tc.corpus); lv != lc {
 			t.Errorf("error: %d of %d documents visited.", lv, lc)

api/internal/crawl/crawler/github/crawler.go

@@ -16,8 +16,6 @@ import (
 	"strings"
 	"time"
 
-	"sigs.k8s.io/kustomize/api/internal/crawl/utils"
-
 	"sigs.k8s.io/kustomize/api/internal/crawl/crawler"
 	"sigs.k8s.io/kustomize/api/internal/crawl/doc"
 	"sigs.k8s.io/kustomize/api/internal/crawl/httpclient"
@@ -32,8 +30,6 @@ var logger = log.New(os.Stdout, "Github Crawler: ",
 type githubCrawler struct {
 	client GhClient
 	query  Query
-	// branchMap maps github repositories to their default branches
-	branchMap map[string]string
 }
 
 type GhClient struct {
@@ -55,60 +51,13 @@ func NewCrawler(accessToken string, retryCount uint64, client *http.Client,
 			},
 			accessToken: accessToken,
 		},
-		query:     query,
-		branchMap: map[string]string{},
+		query: query,
 	}
 }
 
-func (gc githubCrawler) SetDefaultBranch(d *doc.Document) {
-	url := gc.client.ReposRequest(d.RepositoryFullName())
-	defaultBranch, err := gc.client.GetDefaultBranch(url, d.RepositoryURL, gc.branchMap)
-	if err != nil {
-		logger.Printf(
-			"(error: %v) setting default_branch to master\n", err)
-		defaultBranch = "master"
-	}
-	d.DefaultBranch = defaultBranch
-	gc.branchMap[d.RepositoryURL] = d.DefaultBranch
-}
-
-func (gc githubCrawler) DefaultBranch(repo string) string {
-	return gc.branchMap[repo]
-}
-
 // Implements crawler.Crawler.
-func (gc githubCrawler) Crawl(ctx context.Context,
-	output chan<- crawler.CrawledDocument, seen utils.SeenMap) error {
-
-	ranges := []RangeWithin{
-		RangeWithin{
-		start: uint64(0),
-		end:   githubMaxFileSize,
-	},
-	}
-
-	errs := make(multiError, 0)
-	for len(ranges) > 0 {
-		tailRange := ranges[len(ranges) - 1]
-		ranges = ranges[:(len(ranges) - 1)]
-		reProcessQueryRanges, err := gc.CrawlSingleRange(ctx, output, seen, tailRange.start, tailRange.end)
-		if err != nil {
-			errs = append(errs, err)
-		}
-		ranges = append(ranges, reProcessQueryRanges...)
-	}
-
-	if len(errs) > 0 {
-		return errs
-	}
-	return nil
-}
-
-func (gc githubCrawler) CrawlSingleRange(ctx context.Context,
-	output chan<- crawler.CrawledDocument, seen utils.SeenMap,
-	lowerBound, upperBound uint64) ([]RangeWithin, error) {
-
-	log.Printf("CrawlSingleRange [%d, %d]", lowerBound, upperBound)
+func (gc githubCrawler) Crawl(
+	ctx context.Context, output chan<- crawler.CrawledDocument) error {
 
 	noETagClient := GhClient{
 		RequestConfig: gc.client.RequestConfig,
@@ -117,25 +66,12 @@ func (gc githubCrawler) CrawlSingleRange(ctx context.Context,
 		accessToken:   gc.client.accessToken,
 	}
 
-	var reProcessQueryRanges []RangeWithin
-
-	var ranges []string
-	var err error
 	// Since Github returns a max of 1000 results per query, we can use
 	// multiple queries that split the search space into chunks of at most
 	// 1000 files to get all of the data.
-	for i := 0; i < 5; i++ {
-		ranges, err = FindRangesForRepoSearch(newCache(noETagClient, gc.query),
-			lowerBound, upperBound)
-		if err == nil {
-			logger.Printf("FindRangesForRepoSearch succeeded after %d retries", i)
-			break
-		} else {
-			time.Sleep(time.Minute)
-		}
-	}
+	ranges, err := FindRangesForRepoSearch(newCache(noETagClient, gc.query))
 	if err != nil {
-		return reProcessQueryRanges, fmt.Errorf("could not split %v into ranges, %v\n",
+		return fmt.Errorf("could not split %v into ranges, %v\n",
 			gc.query, err)
 	}
 
@@ -143,25 +79,18 @@ func (gc githubCrawler) CrawlSingleRange(ctx context.Context,
 
 	// Query each range for files.
 	errs := make(multiError, 0)
-	queryResult := RangeQueryResult{}
 	for _, query := range ranges {
-		reProcessQuery, rangeResult, err := processQuery(ctx, gc.client, query, output, seen, gc.branchMap)
+		err := processQuery(ctx, gc.client, query, output)
 		if err != nil {
 			errs = append(errs, err)
 		}
-		queryResult.Add(rangeResult)
-		if reProcessQuery {
-			reProcessQueryRanges = append(reProcessQueryRanges, RangeSizes(query))
-		}
 	}
 
-	logger.Printf("Summary of Crawl: %s", queryResult.String())
-
 	if len(errs) > 0 {
-		return reProcessQueryRanges, errs
+		return errs
 	}
 
-	return reProcessQueryRanges, nil
+	return nil
 }
 
 // FetchDocument first tries to fetch the document with d.FilePath. If it fails,
@@ -178,13 +107,9 @@ func (gc githubCrawler) FetchDocument(_ context.Context, d *doc.Document) error
 		"/" + repoSpec.Ref + "/" + repoSpec.Path
 
 	handle := func(resp *http.Response, err error, path string) error {
-		if resp == nil {
-			return fmt.Errorf("empty http response (url: %s; path: %s), error: %v",
-				url, path, err)
-		}
 		if err == nil && resp.StatusCode == http.StatusOK {
 			d.IsSame = httpclient.FromCache(resp.Header)
-			defer CloseResponseBody(resp)
+			defer resp.Body.Close()
 			data, err := ioutil.ReadAll(resp.Body)
 			if err != nil {
 				return err
@@ -236,32 +161,10 @@ func (gc githubCrawler) Match(d *doc.Document) bool {
 	return strings.Contains(repoSpec.Host, "github.com")
 }
 
-type RangeQueryResult struct {
-	totalDocCnt uint64
-	seenDocCnt  uint64
-	newDocCnt   uint64
-	errorCnt    uint64
-}
-
-func (r *RangeQueryResult) Add(other RangeQueryResult) {
-	r.totalDocCnt += other.totalDocCnt
-	r.newDocCnt += other.newDocCnt
-	r.seenDocCnt += other.seenDocCnt
-	r.errorCnt += other.errorCnt
-}
-
-func (r *RangeQueryResult) String() string {
-	return fmt.Sprintf("got %d files from API. "+
-		"%d have been seen before. %d are new and sent to the output channel."+
-		" %d have kustomizationResultAdapter errors.",
-		r.totalDocCnt, r.seenDocCnt, r.newDocCnt, r.errorCnt)
-}
-
 // processQuery follows all of the pages in a query, and updates/adds the
 // documents from the crawl to the datastore/index.
 func processQuery(ctx context.Context, gcl GhClient, query string,
-	output chan<- crawler.CrawledDocument, seen utils.SeenMap,
-	branchMap map[string]string) (bool, RangeQueryResult, error) {
+	output chan<- crawler.CrawledDocument) error {
 
 	queryPages := make(chan GhResponseInfo)
 
@@ -277,84 +180,57 @@ func processQuery(ctx context.Context, gcl GhClient, query string,
 		close(queryPages)
 	}()
 
-	reProcessQuery := false
-
 	errs := make(multiError, 0)
-	result := RangeQueryResult{}
-	pageID := 1
+	errorCnt := 0
+	totalCnt := 0
 	for page := range queryPages {
 		if page.Error != nil {
 			errs = append(errs, page.Error)
 			continue
 		}
-		pageResult := RangeQueryResult{}
+
 		for _, file := range page.Parsed.Items {
-			k, err := kustomizationResultAdapter(gcl, file, seen, branchMap)
+			k, err := kustomizationResultAdapter(gcl, file)
 			if err != nil {
 				logger.Printf("kustomizationResultAdapter failed: %v", err)
 				errs = append(errs, err)
-				pageResult.errorCnt++
+				errorCnt++
 			}
 			if k != nil {
-				pageResult.newDocCnt++
 				output <- k
-			} else {
-				pageResult.seenDocCnt++
 			}
-			pageResult.totalDocCnt++
+			totalCnt++
 		}
 
-		logger.Printf("processQuery [TotalCount %d - page %d]: %s",
-			page.Parsed.TotalCount, pageID, pageResult.String())
-		result.Add(pageResult)
-
-		pageID++
-
-		if page.Parsed.TotalCount > githubMaxResultsPerQuery {
-			reProcessQuery = true
-		}
+		logger.Printf("got %d files out of %d from API. %d of %d had errors\n",
+			totalCnt, page.Parsed.TotalCount, errorCnt, totalCnt)
 	}
 
-	logger.Printf("Summary of processQuery: %s", result.String())
-
-	return reProcessQuery, result, errs
+	return errs
 }
 
-func kustomizationResultAdapter(gcl GhClient, k GhFileSpec, seen utils.SeenMap,
-	branchMap map[string]string) (crawler.CrawledDocument, error) {
-	url := gcl.ReposRequest(k.Repository.FullName)
-	defaultBranch, err := gcl.GetDefaultBranch(url, k.Repository.URL, branchMap)
-	if err != nil {
-		logger.Printf(
-			"(error: %v) setting default_branch to master\n", err)
-		defaultBranch = "master"
-	}
-
-	// document here is a kustomization file found by querying Github, whose
-	// `FileType` field should be empty.
-	document := doc.Document{
-		FilePath:      k.Path,
-		DefaultBranch: defaultBranch,
-		RepositoryURL: k.Repository.URL,
-		User:          doc.UserName(k.Repository.URL),
-	}
-
-	if seen.Seen(document.ID()) {
-		return nil, nil
-	}
+func kustomizationResultAdapter(gcl GhClient, k GhFileSpec) (
+	crawler.CrawledDocument, error) {
 
 	data, err := gcl.GetFileData(k)
 	if err != nil {
 		return nil, err
 	}
 
+	url := gcl.ReposRequest(k.Repository.FullName)
+	defaultBranch, err := gcl.GetDefaultBranch(url)
+	if err != nil {
+		logger.Printf(
+			"(error: %v) setting default_branch to master\n", err)
+		defaultBranch = "master"
+	}
+
 	d := doc.KustomizationDocument{
 		Document: doc.Document{
 			DocumentData:  string(data),
 			FilePath:      k.Path,
 			DefaultBranch: defaultBranch,
 			RepositoryURL: k.Repository.URL,
-			User:          doc.UserName(k.Repository.URL),
 		},
 	}
 	creationTime, err := gcl.GetFileCreationTime(k)
@@ -379,7 +255,7 @@ func (gcl GhClient) ForwardPaginatedQuery(ctx context.Context, query string,
 	output chan<- GhResponseInfo) error {
 
 	logger.Println("querying: ", query)
-	response := gcl.parseGithubResponseWithRetry(query)
+	response := gcl.parseGithubResponse(query)
 
 	if response.Error != nil {
 		return response.Error
@@ -392,7 +268,7 @@ func (gcl GhClient) ForwardPaginatedQuery(ctx context.Context, query string,
 		case <-ctx.Done():
 			return nil
 		default:
-			response = gcl.parseGithubResponseWithRetry(response.NextURL)
+			response = gcl.parseGithubResponse(response.NextURL)
 			if response.Error != nil {
 				return response.Error
 			}
@@ -420,10 +296,7 @@ func (gcl GhClient) GetFileData(k GhFileSpec) ([]byte, error) {
 		return nil, fmt.Errorf("%+v: could not read '%s' metadata: %v",
 			k, url, err)
 	}
-
-	if err := resp.Body.Close(); err != nil {
-		return nil, err
-	}
+	resp.Body.Close()
 
 	type githubContentRawURL struct {
 		DownloadURL string `json:"download_url,omitempty"`
@@ -442,32 +315,18 @@ func (gcl GhClient) GetFileData(k GhFileSpec) ([]byte, error) {
 			k, rawURL.DownloadURL, err)
 	}
 
-	defer CloseResponseBody(resp)
+	defer resp.Body.Close()
 	data, err = ioutil.ReadAll(resp.Body)
 	return data, err
 }
 
-func CloseResponseBody(resp *http.Response) {
-	if err := resp.Body.Close(); err != nil {
-		log.Printf("failed to close response body: %v", err)
-	}
-}
-
-// GetDefaultBranch gets the default branch of a github repository.
-// m is a map which maps a github repository to its default branch.
-// If repo is already in m, the default branch for url will be obtained from m;
-// otherwise, a query will be made to github to obtain the default branch.
-func (gcl GhClient) GetDefaultBranch(url, repo string, m map[string]string) (string, error) {
-	if v, ok := m[repo]; ok {
-		return v, nil
-	}
-
+func (gcl GhClient) GetDefaultBranch(url string) (string, error) {
 	resp, err := gcl.GetReposData(url)
 	if err != nil {
 		return "", fmt.Errorf(
 			"'%s' could not get default_branch: %v", url, err)
 	}
-	defer CloseResponseBody(resp)
+	defer resp.Body.Close()
 	data, err := ioutil.ReadAll(resp.Body)
 	if err != nil {
 		return "", fmt.Errorf(
@@ -519,7 +378,7 @@ func (gcl GhClient) GetFileCreationTime(
 		}
 	}
 
-	defer CloseResponseBody(resp)
+	defer resp.Body.Close()
 	data, err := ioutil.ReadAll(resp.Body)
 	if err != nil {
 		return defaultTime, fmt.Errorf(
@@ -587,8 +446,6 @@ type githubResponse struct {
 	// This is the number of files that match the query.
 	TotalCount uint64 `json:"total_count,omitempty"`
 
-	IncompleteResults bool `json:"incomplete_results,omitempty"`
-
 	// Github representation of a file.
 	Items []GhFileSpec `json:"items,omitempty"`
 }
@@ -631,17 +488,6 @@ func parseGithubLinkFormat(links string) (string, string) {
 	return next, last
 }
 
-func (gcl GhClient) parseGithubResponseWithRetry(getRequest string) GhResponseInfo {
-	resp := gcl.parseGithubResponse(getRequest)
-	retries := 0
-	for resp.Parsed.IncompleteResults {
-		resp = gcl.parseGithubResponse(getRequest)
-		retries++
-	}
-	log.Printf("The result of query(%s) is complete after %d retries", getRequest, retries)
-	return resp
-}
-
 func (gcl GhClient) parseGithubResponse(getRequest string) GhResponseInfo {
 	resp, err := gcl.SearchGithubAPI(getRequest)
 	requestInfo := GhResponseInfo{
@@ -655,7 +501,7 @@ func (gcl GhClient) parseGithubResponse(getRequest string) GhResponseInfo {
 	}
 
 	var data []byte
-	defer CloseResponseBody(resp)
+	defer resp.Body.Close()
 	data, requestInfo.Error = ioutil.ReadAll(resp.Body)
 	if requestInfo.Error != nil {
 		return requestInfo
@@ -719,8 +565,8 @@ func (gcl GhClient) Do(query string) (*http.Response, error) {
 
 	// gcl.client.Do: a non-2xx status code doesn't cause an error.
 	// See https://golang.org/pkg/net/http/#Client.Do for more info.
-	resp, err := gcl.client.Do(req)
-	if resp != nil && resp.StatusCode != http.StatusOK {
+	resp, err :=  gcl.client.Do(req)
+	if resp.StatusCode != http.StatusOK {
 		err = fmt.Errorf("GhClient.Do(%s) failed with response code: %d",
 			query, resp.StatusCode)
 	}
@@ -734,7 +580,7 @@ func (gcl GhClient) getWithRetry(
 
 	retryCount := gcl.retryCount
 
-	for resp != nil && resp.StatusCode == http.StatusForbidden && retryCount > 0 {
+	for resp.StatusCode == http.StatusForbidden && retryCount > 0 {
 		retryTime := resp.Header.Get("Retry-After")
 		i, errAtoi := strconv.Atoi(retryTime)
 		if errAtoi != nil {

api/internal/crawl/crawler/github/queries.go

@@ -116,11 +116,9 @@ type RequestConfig struct {
 // the URL method to get the string value of the URL. See request.CopyWith, to
 // understand why the request object is useful.
 func (rc RequestConfig) CodeSearchRequestWith(query Query) request {
-	vals := url.Values{
-		"sort":  []string{"indexed"},
-		"order": []string{"desc"},
-	}
-	req := rc.makeRequest("search/code", query, vals)
+	req := rc.makeRequest("search/code", query)
+	req.vals.Set("sort", "indexed")
+	req.vals.Set("order", "desc")
 	return req
 }
 
@@ -128,25 +126,27 @@ func (rc RequestConfig) CodeSearchRequestWith(query Query) request {
 // query for the Github API to find the dowload information of this filepath.
 func (rc RequestConfig) ContentsRequest(fullRepoName, path string) string {
 	uri := fmt.Sprintf("repos/%s/contents/%s", fullRepoName, path)
-	return rc.makeRequest(uri, Query{}, url.Values{}).URL()
+	return rc.makeRequest(uri, Query{}).URL()
 }
 
 func (rc RequestConfig) ReposRequest(fullRepoName string) string {
 	uri := fmt.Sprintf("repos/%s", fullRepoName)
-	return rc.makeRequest(uri, Query{}, url.Values{}).URL()
+	return rc.makeRequest(uri, Query{}).URL()
+}
+
+func escapeSpace(s string) string {
+	return strings.Replace(s, " ", "%20", -1)
 }
 
 // CommitsRequest given the repo name, and a filepath returns a formatted query
 // for the Github API to find the commits that affect this file.
 func (rc RequestConfig) CommitsRequest(fullRepoName, path string) string {
 	uri := fmt.Sprintf("repos/%s/commits", fullRepoName)
-	vals := url.Values{
-		"path": []string{path},
-	}
-	return rc.makeRequest(uri, Query{}, vals).URL()
+	return rc.makeRequest(uri, Query{Path(escapeSpace(path))}).URL()
 }
 
-func (rc RequestConfig) makeRequest(path string, query Query, vals url.Values) request {
+func (rc RequestConfig) makeRequest(path string, query Query) request {
+	vals := url.Values{}
 	vals.Set(perPageArg, fmt.Sprint(rc.perPage))
 
 	return request{
@@ -166,7 +166,7 @@ type request struct {
 	query Query
 }
 
-// CopyWith copies the requests and adds the extra query parameters. It is useful
+// CopyWith copies the requests and adds the extra query parameters. Usefull
 // for dynamically adding sizes to a filename only query without modifying it.
 func (r request) CopyWith(queryParams ...queryField) request {
 	cpy := r

api/internal/crawl/crawler/github/queries_test.go

@@ -101,7 +101,7 @@ func TestGithubSearchQuery(t *testing.T) {
 				"examples/helloWorld/kustomization.yaml?per_page=100",
 
 			expectedCommitsQuery: "https://api.github.com/repos/kubernetes-sigs/kustomize/commits?" +
-				"path=examples%2FhelloWorld%2Fkustomization.yaml&per_page=100",
+				"q=path:examples/helloWorld/kustomization.yaml&per_page=100",
 		},
 		{
 			rc: RequestConfig{
@@ -121,7 +121,7 @@ func TestGithubSearchQuery(t *testing.T) {
 				"examples%201/helloWorld/kustomization.yaml?per_page=100",
 
 			expectedCommitsQuery: "https://api.github.com/repos/kubernetes-sigs/kustomize/commits?" +
-				"path=examples+1%2FhelloWorld%2Fkustomization.yaml&per_page=100",
+				"q=path:examples%201/helloWorld/kustomization.yaml&per_page=100",
 		},
 	}
 

api/internal/crawl/crawler/github/split_search_ranges.go

@@ -93,15 +93,13 @@ package github
 //   apiCallsPerResult * 10(pages) * 100(resultsPerPage) * totalResults / 1000
 //   = apiCallsPerResult * totalResults.
 //
-// So it could very well take apiCallsPerResult * 50 times longer to actually
+// So it could very well take apiCallsPerResult * 50 times longer to acutally
 // fetch the results (assuming the quotas for the API calls are the same as the
 // search API), than it does to perform these range searches.
 
 import (
 	"fmt"
 	"math/bits"
-	"strconv"
-	"strings"
 )
 
 // Files cannot be more than 2^19 bytes, according to
@@ -114,7 +112,7 @@ const (
 // Interface instead of struct for testing purposes.
 // Not expecting to have multiple implementations.
 type cachedSearch interface {
-	CountResults(uint64, uint64) (uint64, error)
+	CountResults(uint64) (uint64, error)
 	RequestString(filesize rangeFormatter) string
 }
 
@@ -141,7 +139,7 @@ type cachedSearch interface {
 //    problematic). The current cache implementation looks at the
 //    predecessor entry to find out if the current value is monotonic.
 //    This is where the bit trick is used, since each step in the binary
-//    search is adding or omitting to add a decreasing power of 2 to the query
+//    search is adding or ommiting to add a decreasing power of 2 to the query
 //    value, we can remove the least significant set bit to find the
 //    predecessor in constant time. Ultimately since the search is rate
 //    limited, we could also easily afford to compute this in linear time
@@ -163,16 +161,16 @@ func newCache(client GhClient, query Query) githubCachedSearch {
 	}
 }
 
-func (c githubCachedSearch) CountResults(lowerBound, upperBound uint64) (uint64, error) {
+func (c githubCachedSearch) CountResults(upperBound uint64) (uint64, error) {
 	count, cached := c.cache[upperBound]
 	if cached {
 		return count, nil
 	}
 
-	sizeRange := RangeWithin{lowerBound, upperBound}
+	sizeRange := RangeWithin{0, upperBound}
 	rangeRequest := c.RequestString(sizeRange)
 
-	result := c.gcl.parseGithubResponseWithRetry(rangeRequest)
+	result := c.gcl.parseGithubResponse(rangeRequest)
 	if result.Error != nil {
 		return count, result.Error
 	}
@@ -206,7 +204,7 @@ func (c githubCachedSearch) CountResults(lowerBound, upperBound uint64) (uint64,
 			"Retrying query... current lower bound: %d, got: %d\n",
 			c.cache[prev], result.Parsed.TotalCount)
 
-		result = c.gcl.parseGithubResponseWithRetry(rangeRequest)
+		result = c.gcl.parseGithubResponse(rangeRequest)
 		if result.Error != nil {
 			return count, result.Error
 		}
@@ -221,8 +219,8 @@ func (c githubCachedSearch) CountResults(lowerBound, upperBound uint64) (uint64,
 	}
 
 	count = result.Parsed.TotalCount
-	logger.Printf("Caching new query %s, with count %d (incomplete_results: %v)\n",
-		sizeRange.RangeString(), count, result.Parsed.IncompleteResults)
+	logger.Printf("Caching new query %s, with count %d\n",
+		sizeRange.RangeString(), count)
 	c.cache[upperBound] = count
 	return count, nil
 }
@@ -240,8 +238,8 @@ func (c githubCachedSearch) RequestString(filesize rangeFormatter) string {
 // This would mean that the search as it is could not find all files. If queries
 // are sorted by last indexed, and retrieved on regular intervals, it should be
 // sufficient to get most if not all documents.
-func FindRangesForRepoSearch(cache cachedSearch, lowerBound, upperBound uint64) ([]string, error) {
-	totalFiles, err := cache.CountResults(lowerBound, upperBound)
+func FindRangesForRepoSearch(cache cachedSearch) ([]string, error) {
+	totalFiles, err := cache.CountResults(githubMaxFileSize)
 	if err != nil {
 		return nil, err
 	}
@@ -249,7 +247,7 @@ func FindRangesForRepoSearch(cache cachedSearch, lowerBound, upperBound uint64)
 
 	if githubMaxResultsPerQuery >= totalFiles {
 		return []string{
-			cache.RequestString(RangeWithin{lowerBound, upperBound}),
+			cache.RequestString(RangeWithin{0, githubMaxFileSize}),
 		}, nil
 	}
 
@@ -277,7 +275,6 @@ func FindRangesForRepoSearch(cache cachedSearch, lowerBound, upperBound uint64)
 	// range.
 	filesAccessible := uint64(0)
 	sizes := make([]uint64, 0)
-	sizes = append(sizes, lowerBound)
 	for filesAccessible < totalFiles {
 		target := filesAccessible + githubMaxResultsPerQuery
 		if target >= totalFiles {
@@ -287,22 +284,22 @@ func FindRangesForRepoSearch(cache cachedSearch, lowerBound, upperBound uint64)
 		logger.Printf("%d accessible files, next target = %d\n",
 			filesAccessible, target)
 
-		size, err := FindFileSize(cache, target, lowerBound, upperBound)
+		cur, err := lowerBoundFileCount(cache, target)
 		if err != nil {
 			return nil, err
 		}
 
 		// If there are more than 1000 files in the next bucket, we must
 		// advance anyway and lose out on some files :(.
-		if l := len(sizes); l > 0 && sizes[l-1] == size {
-			size++
+		if l := len(sizes); l > 0 && sizes[l-1] == cur {
+			cur++
 		}
 
-		nextAccessible, err := cache.CountResults(lowerBound, size)
+		nextAccessible, err := cache.CountResults(cur)
 		if err != nil {
 			return nil, fmt.Errorf(
 				"cache should be populated at %d already, got %v",
-				size, err)
+				cur, err)
 		}
 		if nextAccessible < filesAccessible {
 			return nil, fmt.Errorf(
@@ -312,31 +309,31 @@ func FindRangesForRepoSearch(cache cachedSearch, lowerBound, upperBound uint64)
 
 		filesAccessible = nextAccessible
 		if nextAccessible < totalFiles {
-			sizes = append(sizes, size)
+			sizes = append(sizes, cur)
 		}
 	}
-	sizes = append(sizes, upperBound)
+
 	return formatFilesizeRanges(cache, sizes), nil
 }
 
-// FindFileSize finds the filesize range from [lowerBound, return value] that has
+// lowerBoundFileCount finds the filesize range from [0, return value] that has
 // the largest file count that is smaller than or equal to
 // githubMaxResultsPerQuery. It is important to note that this returned value
 // could already be in a previous range if the next file size has more than 1000
 // results. It is left to the caller to handle this bit of logic and guarantee
 // forward progession in this case.
-func FindFileSize(
-	cache cachedSearch, targetFileCount, lowerBound, upperBound uint64) (uint64, error) {
+func lowerBoundFileCount(
+	cache cachedSearch, targetFileCount uint64) (uint64, error) {
 
 	// Binary search for file sizes that make up the next <=1000 element
 	// chunk.
-	cur := lowerBound
-	increase := (upperBound - lowerBound) / 2
+	cur := uint64(0)
+	increase := githubMaxFileSize / 2
 
 	for increase > 0 {
 		mid := cur + increase
 
-		count, err := cache.CountResults(lowerBound, mid)
+		count, err := cache.CountResults(mid)
 		if err != nil {
 			return count, err
 		}
@@ -356,24 +353,26 @@ func FindFileSize(
 }
 
 func formatFilesizeRanges(cache cachedSearch, sizes []uint64) []string {
-	n := len(sizes)
-	if n < 2 {
-		return []string{}
+	ranges := make([]string, 0, len(sizes)+1)
+
+	if len(sizes) > 0 {
+		ranges = append(ranges, cache.RequestString(
+			RangeLessThan{sizes[0] + 1},
+		))
 	}
 
-	ranges := make([]string, 0, n-1)
-	ranges = append(ranges, cache.RequestString(RangeWithin{sizes[0], sizes[1]}))
-	for i := 1; i < n-1; i++ {
-		ranges = append(ranges, cache.RequestString(RangeWithin{sizes[i] + 1, sizes[i+1]}))
+	for i := 0; i < len(sizes)-1; i += 1 {
+		ranges = append(ranges, cache.RequestString(
+			RangeWithin{sizes[i] + 1, sizes[i+1]},
+		))
+
+		if i != len(sizes)-2 {
+			continue
+		}
+		ranges = append(ranges, cache.RequestString(
+			RangeGreaterThan{sizes[i+1]},
+		))
 	}
+
 	return ranges
 }
-
-func RangeSizes(s string) RangeWithin {
-	start := strings.Index(s, "+size:") + len("+size:")
-	end := strings.Index(s, "&")
-	ranges := strings.Split(s[start:end], "..")
-	lowerBound, _ := strconv.ParseUint(ranges[0], 10, 64)
-	upperBound, _ := strconv.ParseUint(ranges[1], 10, 64)
-	return RangeWithin{lowerBound, upperBound}
-}

api/internal/crawl/crawler/github/split_search_ranges_test.go

@@ -2,7 +2,6 @@ package github
 
 import (
 	"fmt"
-	"log"
 	"reflect"
 	"testing"
 )
@@ -11,8 +10,8 @@ type testCachedSearch struct {
 	cache map[uint64]uint64
 }
 
-func (c testCachedSearch) CountResults(lowerBound, upperBound uint64) (uint64, error) {
-	log.Printf("CountResults(%05x)\n", upperBound)
+func (c testCachedSearch) CountResults(upperBound uint64) (uint64, error) {
+	fmt.Printf("CountResults(%05x)\n", upperBound)
 	count, ok := c.cache[upperBound]
 	if !ok {
 		return count, fmt.Errorf("cache not set at %x", upperBound)
@@ -73,29 +72,19 @@ func TestRangeSplitting(t *testing.T) {
 		},
 	}
 
-	requests, err := FindRangesForRepoSearch(cache, 0, 524288)
+	requests, err := FindRangesForRepoSearch(cache)
 	if err != nil {
 		t.Errorf("Error while finding ranges: %v", err)
 	}
 	expected := []string{
-		"0..106",       // cache.RequestString(RangeWithin{0x00, 0x6a}),
-		"107..128",     // cache.RequestString(RangeWithin{0x6b, 0x80}),
-		"129..256",     // cache.RequestString(RangeWithin{0x81, 0x100}),
-		"257..4095",    // cache.RequestString(RangeWithin{0x101, 0xfff}),
-		"4096..524288", // cache.RequestString(RangeWithin{0x1000, 0x80000}),
+		"<107",      // cache.RequestString(RangeLessThan{0x6b}),
+		"107..128",  // cache.RequestString(RangeWithin{0x6b, 0x80}),
+		"129..256",  // cache.RequestString(RangeWithin{0x81, 0x100}),
+		"257..4095", // cache.RequestString(RangeWithin{0x101, 0xfff}),
+		">4095",     // cache.RequestString(RangeGreaterThan{0xfff}),
 	}
 
 	if !reflect.DeepEqual(requests, expected) {
 		t.Errorf("Expected requests (%v) to equal (%v)", requests, expected)
 	}
 }
-
-func TestRangeSizes(t *testing.T) {
-	s := "https://api.github.com/search/code?q=filename:kustomization.yaml+filename:kustomization.yml" +
-		"+filename:kustomization+size:2365..10000&order=desc&per_page=100&sort=indexed"
-	returnedResult := RangeSizes(s)
-	expectedResult := RangeWithin{uint64(2365), uint64(10000)}
-	if !reflect.DeepEqual(returnedResult, expectedResult) {
-		t.Errorf("RangeSizes expected (%v), got (%v)",expectedResult, returnedResult)
-	}
-}

api/internal/crawl/doc/doc.go

@@ -2,8 +2,6 @@ package doc
 
 import (
 	"fmt"
-	"log"
-	"path/filepath"
 	"sort"
 	"strings"
 
@@ -46,36 +44,21 @@ type KustomizationDocument struct {
 
 type set map[string]struct{}
 
-func (doc *KustomizationDocument) Copy() *KustomizationDocument {
-	return &KustomizationDocument{
-		Document:    *(doc.Document.Copy()),
-		Kinds:       doc.Kinds,
-		Identifiers: doc.Identifiers,
-		Values:      doc.Values,
-	}
-}
-
 func (doc *KustomizationDocument) String() string {
 	return fmt.Sprintf("%s %s %s %v %v %v len(identifiers):%v  len(values):%v",
 		doc.RepositoryURL, doc.FilePath, doc.DefaultBranch, doc.CreationTime,
 		doc.IsSame, doc.Kinds, len(doc.Identifiers), len(doc.Values))
 }
 
-// IsKustomizationFile determines whether a file path is a kustomization file
-func IsKustomizationFile(path string) bool {
-	basename := filepath.Base(path)
-	for _, name := range konfig.RecognizedKustomizationFileNames() {
-		if basename == name {
-			return true
+// Implements the CrawlerDocument interface.
+func (doc *KustomizationDocument) GetResources() ([]*Document, error) {
+	isResource := true
+	for _, suffix := range konfig.RecognizedKustomizationFileNames() {
+		if strings.HasSuffix(doc.FilePath, "/"+suffix) {
+			isResource = false
 		}
 	}
-	return false
-}
-
-// Implements the CrawlerDocument interface.
-func (doc *KustomizationDocument) GetResources(
-	includeResources, includeTransformers, includeGenerators bool) ([]*Document, error) {
-	if !IsKustomizationFile(doc.FilePath) {
+	if isResource {
 		return []*Document{}, nil
 	}
 
@@ -93,46 +76,19 @@ func (doc *KustomizationDocument) GetResources(
 	}
 	k.FixKustomizationPostUnmarshalling()
 
-	res := make([]*Document, 0)
-
-	if includeResources {
-		resourceDocs := doc.CollectDocuments(k.Resources, "resource")
-		res = append(res, resourceDocs...)
-	}
-
-	if includeGenerators {
-		generatorDocs := doc.CollectDocuments(k.Generators, "generator")
-		res = append(res, generatorDocs...)
-	}
-
-	if includeTransformers {
-		transformerDocs := doc.CollectDocuments(k.Transformers, "transformer")
-		res = append(res, transformerDocs...)
+	res := make([]*Document, 0, len(k.Resources))
+	for _, r := range k.Resources {
+		next, err := doc.Document.FromRelativePath(r)
+		if err != nil {
+			fmt.Printf("GetResources error: %v\n", err)
+			continue
+		}
+		res = append(res, &next)
 	}
 
 	return res, nil
 }
 
-// CollectDocuments construct a Document for each path in paths, and return
-// a slice of Document pointers.
-func (doc *KustomizationDocument) CollectDocuments(
-	paths []string, fileType string) []*Document {
-	docs := make([]*Document, 0, len(paths))
-	for _, r := range paths {
-		if strings.TrimSpace(r) == "" {
-			continue
-		}
-		next, err := doc.Document.FromRelativePath(r)
-		if err != nil {
-			log.Printf("CollectDocuments error: %v\n", err)
-			continue
-		}
-		next.FileType = fileType
-		docs = append(docs, &next)
-	}
-	return docs
-}
-
 func (doc *KustomizationDocument) readBytes() ([]map[string]interface{}, error) {
 	data := []byte(doc.DocumentData)
 

api/internal/crawl/doc/doc_test.go

@@ -189,13 +189,11 @@ metadata:
 	}
 }
 
-type TestStructForGetResources struct {
-	doc       KustomizationDocument
-	resources []*Document
-}
-
 func TestGetResources(t *testing.T) {
-	tests := []TestStructForGetResources{
+	tests := []struct {
+		doc       KustomizationDocument
+		resources []*Document
+	}{
 		{
 			doc: KustomizationDocument{
 				Document: Document{
@@ -215,27 +213,19 @@ resources:
 				{
 					RepositoryURL: "sigs.k8s.io/kustomize",
 					FilePath:      "some/path/to/base",
-					FileType:      "resource",
-					User:          "sigs.k8s.io",
 				},
 				{
 					RepositoryURL: "sigs.k8s.io/kustomize",
 					FilePath:      "some/path/to/otherbase",
-					FileType:      "resource",
-					User:          "sigs.k8s.io",
 				},
 				{
 					RepositoryURL: "sigs.k8s.io/kustomize",
 					FilePath:      "some/path/to/kdir/file.yaml",
-					FileType:      "resource",
-					User:          "sigs.k8s.io",
 				},
 				{
 					RepositoryURL: "https://github.com/kubernetes-sigs/kustomize",
 					FilePath:      "examples/helloWorld",
 					DefaultBranch: "v3.1.0",
-					FileType:      "resource",
-					User:          "kubernetes-sigs",
 				},
 			},
 		},
@@ -258,12 +248,9 @@ resources:
 			resources: []*Document{},
 		},
 	}
-	runTest(t, tests, true, false, false)
-}
 
-func runTest(t *testing.T, tests []TestStructForGetResources, includeResources, includeTransformers, includeGenerators bool) {
 	for _, test := range tests {
-		res, err := test.doc.GetResources(includeResources, includeTransformers, includeGenerators)
+		res, err := test.doc.GetResources()
 		if err != nil {
 			t.Errorf("Unexpected error: %v\n", err)
 			continue
@@ -297,83 +284,3 @@ func runTest(t *testing.T, tests []TestStructForGetResources, includeResources,
 		}
 	}
 }
-
-func TestGetResourcesAndGenerators(t *testing.T) {
-	tests := []TestStructForGetResources{
-		{
-			doc: KustomizationDocument{
-				Document: Document{
-					RepositoryURL: "sigs.k8s.io/kustomize",
-					FilePath:      "some/path/to/kdir/kustomization.yaml",
-					DocumentData: `
-resources:
-- file.yaml
-
-generators:
-- gen.yaml
-
-transformers:
-- tr.yaml
-`},
-			},
-			resources: []*Document{
-				{
-					RepositoryURL: "sigs.k8s.io/kustomize",
-					FilePath:      "some/path/to/kdir/gen.yaml",
-					FileType:      "generator",
-					User:          "sigs.k8s.io",
-				},
-				{
-					RepositoryURL: "sigs.k8s.io/kustomize",
-					FilePath:      "some/path/to/kdir/file.yaml",
-					FileType:      "resource",
-					User:          "sigs.k8s.io",
-				},
-			},
-		},
-	}
-	runTest(t, tests, true, false, true)
-}
-
-func TestGetResourcesAndGeneratorsAndTransformers(t *testing.T) {
-	tests := []TestStructForGetResources{
-		{
-			doc: KustomizationDocument{
-				Document: Document{
-					RepositoryURL: "sigs.k8s.io/kustomize",
-					FilePath:      "some/path/to/kdir/kustomization.yaml",
-					DocumentData: `
-resources:
-- file.yaml
-
-generators:
-- gen.yaml
-
-transformers:
-- tr.yaml
-`},
-			},
-			resources: []*Document{
-				{
-					RepositoryURL: "sigs.k8s.io/kustomize",
-					FilePath:      "some/path/to/kdir/tr.yaml",
-					FileType:      "transformer",
-					User:          "sigs.k8s.io",
-				},
-				{
-					RepositoryURL: "sigs.k8s.io/kustomize",
-					FilePath:      "some/path/to/kdir/gen.yaml",
-					FileType:      "generator",
-					User:          "sigs.k8s.io",
-				},
-				{
-					RepositoryURL: "sigs.k8s.io/kustomize",
-					FilePath:      "some/path/to/kdir/file.yaml",
-					FileType:      "resource",
-					User:          "sigs.k8s.io",
-				},
-			},
-		},
-	}
-	runTest(t, tests, true, true, true)
-}

api/internal/crawl/doc/docname.go

@@ -11,18 +11,12 @@ import (
 )
 
 type Document struct {
-	RepositoryURL string `json:"repositoryUrl,omitempty"`
-	// User makes it easy to aggregate data in the user level instead
-	// of the repository level
-	User          string     `json:"user,omitempty"`
+	RepositoryURL string     `json:"repositoryUrl,omitempty"`
 	FilePath      string     `json:"filePath,omitempty"`
 	DefaultBranch string     `json:"defaultBranch,omitempty"`
 	DocumentData  string     `json:"document,omitempty"`
 	CreationTime  *time.Time `json:"creationTime,omitempty"`
 	IsSame        bool       `json:"-"`
-	// FileType can be one of the following:
-	// "generator", "transformer", "resource", "".
-	FileType string `json:"fileType,omitempty"`
 }
 
 // Implements the CrawlerDocument interface.
@@ -33,21 +27,14 @@ func (doc *Document) GetDocument() *Document {
 func (doc *Document) Copy() *Document {
 	return &Document{
 		RepositoryURL: doc.RepositoryURL,
-		User:          doc.User,
 		FilePath:      doc.FilePath,
 		DefaultBranch: doc.DefaultBranch,
 		DocumentData:  doc.DocumentData,
 		CreationTime:  doc.CreationTime,
 		IsSame:        doc.IsSame,
-		FileType:      doc.FileType,
 	}
 }
 
-func (doc *Document) Path() string {
-	return fmt.Sprintf("repoURL: %s filePath: %s branch: %s",
-		doc.RepositoryURL, doc.FilePath, doc.DefaultBranch)
-}
-
 // Implements the CrawlerDocument interface.
 func (doc *Document) WasCached() bool {
 	return doc.IsSame
@@ -60,7 +47,6 @@ func (doc *Document) FromRelativePath(newFile string) (Document, error) {
 			RepositoryURL: repoSpec.Host + path.Clean(repoSpec.OrgRepo),
 			FilePath:      path.Clean(repoSpec.Path),
 			DefaultBranch: repoSpec.Ref,
-			User:          UserName(repoSpec.Host + path.Clean(repoSpec.OrgRepo)),
 		}, nil
 	}
 	// else document is probably relative path.
@@ -68,7 +54,6 @@ func (doc *Document) FromRelativePath(newFile string) (Document, error) {
 	ret := Document{
 		RepositoryURL: doc.RepositoryURL,
 		DefaultBranch: doc.DefaultBranch,
-		User:          UserName(doc.RepositoryURL),
 	}
 	ogDir, _ := path.Split(doc.FilePath)
 
@@ -93,7 +78,13 @@ func (doc *Document) ID() string {
 }
 
 func (doc *Document) RepositoryFullName() string {
-	url := TrimUrl(doc.RepositoryURL)
+	url := strings.TrimRight(doc.RepositoryURL, "/")
+
+	gitPrefix := "git@github.com:"
+	if strings.HasPrefix(url, gitPrefix) {
+		url = url[len(gitPrefix):]
+	}
+
 	sections := strings.Split(url, "/")
 	l := len(sections)
 	if l < 2 {
@@ -101,24 +92,3 @@ func (doc *Document) RepositoryFullName() string {
 	}
 	return path.Join(sections[l-2], sections[l-1])
 }
-
-// TrimUrl removes all the trailing slashes and the "git@github.com:" prefix (if exists).
-func TrimUrl(s string) string {
-	url := strings.TrimRight(s, "/")
-
-	gitPrefix := "git@github.com:"
-	if strings.HasPrefix(url, gitPrefix) {
-		url = url[len(gitPrefix):]
-	}
-	return url
-}
-
-func UserName(repositoryURL string) string {
-	url := TrimUrl(repositoryURL)
-	sections := strings.Split(url, "/")
-	l := len(sections)
-	if l < 2 {
-		return url
-	}
-	return sections[l-2]
-}

api/internal/crawl/doc/docname_test.go

@@ -28,7 +28,6 @@ func TestFromRelativePath(t *testing.T) {
 						RepositoryURL: "example.com/repo",
 						FilePath:      "path/to/other/file/resource.yaml",
 						DefaultBranch: "master",
-						User:          "example.com",
 					},
 				},
 				{
@@ -37,7 +36,6 @@ func TestFromRelativePath(t *testing.T) {
 						RepositoryURL: "example.com/repo",
 						FilePath:      "path/to/other/file/patch.yaml",
 						DefaultBranch: "master",
-						User:          "example.com",
 					},
 				},
 				{
@@ -46,7 +44,6 @@ func TestFromRelativePath(t *testing.T) {
 						RepositoryURL: "example.com/repo",
 						FilePath:      "path/to/file/service.yaml",
 						DefaultBranch: "master",
-						User:          "example.com",
 					},
 				},
 			},
@@ -68,7 +65,7 @@ func TestFromRelativePath(t *testing.T) {
 
 func TestDocument_RepositoryFullName(t *testing.T) {
 	testCases := []struct {
-		doc                        Document
+		doc Document
 		expectedRepositoryFullName string
 	}{
 		{
@@ -111,40 +108,4 @@ func TestDocument_RepositoryFullName(t *testing.T) {
 				returnedRepositoryFullName)
 		}
 	}
-}
-
-func TestDocument_UserName(t *testing.T) {
-	testCases := []struct {
-		repositoryURL    string
-		expectedUserName string
-	}{
-		{
-			repositoryURL:    "https://github.com/user/repo",
-			expectedUserName: "user",
-		},
-		{
-			repositoryURL:    "https://github.com//user/repo////",
-			expectedUserName: "user",
-		},
-		{
-			repositoryURL:    "repo/",
-			expectedUserName: "repo",
-		},
-		{
-			repositoryURL:    "",
-			expectedUserName: "",
-		},
-		{
-			repositoryURL:    "git@github.com:user/repo",
-			expectedUserName: "user",
-		},
-	}
-
-	for _, tc := range testCases {
-		returnedUserName := UserName(tc.repositoryURL)
-		if returnedUserName != tc.expectedUserName {
-			t.Errorf("UserName expected %s, got %s",
-				tc.expectedUserName, returnedUserName)
-		}
-	}
-}
+}

api/internal/crawl/doc/unique_doc.go

@@ -1,36 +0,0 @@
-package doc
-
-import (
-	"sigs.k8s.io/kustomize/api/internal/crawl/utils"
-)
-
-// UniqueDocuments make sure a Document with a given ID appears only once
-type UniqueDocuments struct {
-	docs   []*Document
-	docIDs utils.SeenMap
-}
-
-func NewUniqueDocuments() UniqueDocuments {
-	return UniqueDocuments{
-		docs:   []*Document{},
-		docIDs: utils.NewSeenMap(),
-	}
-}
-
-func (uds *UniqueDocuments) Add(d *Document) {
-	if uds.docIDs.Seen(d.ID()) {
-		return
-	}
-	uds.docs = append(uds.docs, d)
-	uds.docIDs.Set(d.ID(), "")
-}
-
-func (uds *UniqueDocuments) AddDocuments(docs []*Document) {
-	for _, d := range docs {
-		uds.Add(d)
-	}
-}
-
-func (uds *UniqueDocuments) Documents() []*Document {
-	return uds.docs
-}

api/internal/crawl/go.mod

@@ -8,6 +8,6 @@ require (
 	github.com/gorilla/mux v1.7.3
 	github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79
 	github.com/rs/cors v1.7.0
-	sigs.k8s.io/kustomize/api v0.3.1
+	sigs.k8s.io/kustomize/api v0.3.0
 	sigs.k8s.io/yaml v1.1.0
 )

api/internal/crawl/go.sum

@@ -133,8 +133,13 @@ github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+
 github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d h1:7XGaL1e6bYS1yIonGp9761ExpPPV1ui0SAC59Yube9k=
 github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY=
 github.com/gophercloud/gophercloud v0.1.0/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEoIEcSTewFxm1c5g8=
+github.com/gorilla/context v0.0.0-20160226214623-1ea25387ff6f/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg=
+github.com/gorilla/mux v1.6.0/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
 github.com/gorilla/mux v1.7.3 h1:gnP5JzjVOuiZD07fKKToCAOjS0yOpj/qPETTXCCS6hw=
 github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
+github.com/gorilla/securecookie v0.0.0-20160422134519-667fe4e3466a/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4=
+github.com/gorilla/sessions v0.0.0-20160922145804-ca9ada445741/go.mod h1:+WVp8kdw6VhyKExm03PAMRn2ZxnPtm58pV0dBVPdhHE=
+github.com/gorilla/websocket v1.2.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
 github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
 github.com/gostaticanalysis/analysisutil v0.0.0-20190318220348-4088753ea4d3/go.mod h1:eEOZF4jCKGi+aprrirO9e7WKB3beBRtWgqGunKl6pKE=
 github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
@@ -193,6 +198,7 @@ github.com/modern-go/reflect2 v0.0.0-20180320133207-05fbef0ca5da/go.mod h1:bx2lN
 github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI=
 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/monopole/mdrip v1.0.1/go.mod h1:/7E04hlzRG9Jrp6WILZfYYm/REoJWL2l+MlsCO1eH74=
 github.com/mozilla/tls-observatory v0.0.0-20190404164649-a3c1b6cfecfd/go.mod h1:SrKMQvPiws7F7iqYp8/TX+IhxCYhzr6N/1yb8cwHsGk=
 github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
@@ -234,6 +240,7 @@ github.com/shirou/gopsutil v0.0.0-20190901111213-e4ec7b275ada/go.mod h1:WWnYX4lz
 github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc=
 github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk=
 github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041/go.mod h1:N5mDOmsrJOB+vfqUK+7DmDyjhSLIIBnXo9lvZJj3MWQ=
+github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
 github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
@@ -380,6 +387,7 @@ gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMy
 gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
 gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
 gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
+gopkg.in/russross/blackfriday.v2 v2.0.0/go.mod h1:6sSBNz/GtOm/pJTuh5UmBK2ZHfmnxGbl2NZg1UliSOI=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
 gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
@@ -407,8 +415,8 @@ k8s.io/utils v0.0.0-20191114184206-e782cd3c129f/go.mod h1:sZAwmy6armz5eXlNoLmJcl
 mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed/go.mod h1:Xkxe497xwlCKkIaQYRfC7CSLworTXY9RMqwhhCm+8Nc=
 mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b/go.mod h1:2odslEg/xrtNQqCYg2/jCoyKnw3vv5biOc3JnIcYfL4=
 mvdan.cc/unparam v0.0.0-20190720180237-d51796306d8f/go.mod h1:4G1h5nDURzA3bwVMZIVpwbkw+04kSxk3rAtzlimaUJw=
-sigs.k8s.io/kustomize/api v0.3.1 h1:oqMIXvS6tFEUVuKIRUKDa05eC4Hh+cb9JYg8Zhp2d24=
-sigs.k8s.io/kustomize/api v0.3.1/go.mod h1:A+ATnlHqzictQfQC1q3KB/T6MSr0UWQsrrLxMWkge2E=
+sigs.k8s.io/kustomize/api v0.3.0 h1:riR/YsL75nGb+aIPFdIRiqu21+OZbAXQybDS7+FUYRg=
+sigs.k8s.io/kustomize/api v0.3.0/go.mod h1:DWNMJBV1xvLruMpihGgnIPznMwHpwUSrxz6v3gnw5kw=
 sigs.k8s.io/structured-merge-diff v0.0.0-20190525122527-15d366b2352e/go.mod h1:wWxsB5ozmmv/SG7nM11ayaAW51xMvak/t1r0CSlcokI=
 sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs=
 sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=

api/internal/crawl/index/elasticsearch.go

@@ -20,18 +20,12 @@ const IndexConfig = `
 				"repositoryUrl": {
 					"type": "keyword"
 				},
-				"user": {
-					"type": "keyword"
-				},
 				"filePath": {
 					"type": "keyword"
 				},
 				"defaultBranch": {
 					"type": "keyword"
 				},
-				"fileType": {
-					"type": "keyword"
-				},
 				"document": {
 					"type": "text"
 				},
@@ -93,7 +87,7 @@ func (idx *index) responseErrorOrNil(info string, res *esapi.Response,
 
 	defer res.Body.Close()
 	if res.IsError() {
-		return fmt.Errorf("%s: %s [%d]", messageStart, res.String(), res.StatusCode)
+		return fmt.Errorf("%s: %s", messageStart, res.String())
 	}
 
 	if reader != nil {
@@ -315,9 +309,9 @@ func (idx *index) Exists(id string) (bool, error) {
 		op.WithPretty(),
 	)
 
-	if res != nil && !res.IsError() {
+	if !res.IsError() {
 		return true, nil
-	} else if res != nil && res.StatusCode == 404 {
+	} else if res.StatusCode == 404 {
 		return false, nil
 	} else {
 		return false, idx.responseErrorOrNil(

api/internal/crawl/index/kustomize.go

@@ -6,7 +6,6 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
-	"log"
 	"strings"
 	"time"
 
@@ -18,7 +17,6 @@ const (
 )
 
 type Mode int
-
 const (
 	InsertOrUpdate = iota
 	Delete
@@ -99,14 +97,14 @@ type KustomizeIndex struct {
 }
 
 // Create index reference to the index containing the kustomize documents.
-func NewKustomizeIndex(ctx context.Context, indexName string) (*KustomizeIndex, error) {
-	idx, err := newIndex(ctx, indexName)
+func NewKustomizeIndex(ctx context.Context) (*KustomizeIndex, error) {
+	idx, err := newIndex(ctx, "kustomize")
 	if err != nil {
 		return nil, err
 	}
 
 	indicesExistsOp := idx.client.Indices.Exists
-	resp, err := indicesExistsOp([]string{indexName},
+	resp, err := indicesExistsOp([]string{"kustomize"},
 		indicesExistsOp.WithContext(idx.ctx),
 		indicesExistsOp.WithPretty())
 	if err != nil {
@@ -114,9 +112,9 @@ func NewKustomizeIndex(ctx context.Context, indexName string) (*KustomizeIndex,
 	}
 
 	if resp.StatusCode == 200 {
-		log.Printf("The %s index already exists", indexName)
+		fmt.Printf("The kustomize index already exists\n")
 	} else {
-		log.Printf("Creating the %s index\n", indexName)
+		fmt.Printf("Creating the kustomize index\n")
 		if err := idx.CreateIndex([]byte(IndexConfig)); err != nil {
 			return nil, err
 		}
@@ -254,7 +252,7 @@ func (it *KustomizeIterator) Next() bool {
 	}
 
 	if it.err == nil {
-		log.Printf("updating scroll: %s\n", *it.scrollImpl.ScrollID)
+		fmt.Printf("updating scroll: %s\n", *it.scrollImpl.ScrollID)
 		it.err = it.update(*it.scrollImpl.ScrollID, reader)
 	}
 
@@ -270,7 +268,7 @@ func (it *KustomizeIterator) Value() KustomizeResult {
 	return it.scrollImpl
 }
 
-// Check if any errors have occurred.
+// Check if any errors have occured.
 func (it *KustomizeIterator) Err() error {
 	return it.err
 }
@@ -343,7 +341,7 @@ func (ki *KustomizeIndex) Search(query string,
 	if err != nil {
 		return nil, fmt.Errorf("failed to format query %s", query)
 	}
-	log.Printf("formated query: %s\n", data)
+	fmt.Printf("formated query: %s\n", data)
 
 	var kr ElasticKustomizeResult
 	err = ki.index.Search(data, opts.SearchOptions, func(results io.Reader) error {

api/internal/crawl/search_cmds/creationTime.md

@@ -1,411 +0,0 @@
-Find out the largest value of the `creationTime` field:
-```
-curl -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
-{
-    "aggs" : {
-        "max_creationTime" : { "max" : { "field" : "creationTime" } }
-    }
-}
-'
-```
-
-Find out the smallest value of the `creationTime` field:
-```
-curl -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
-{
-    "aggs" : {
-        "min_creationTime" : { "min" : { "field" : "creationTime" } }
-    }
-}
-'
-```
-
-Find out the smallest value of the `creationTime` field of all the kustomization files:
-```
-curl -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
-{
-  "query": {
-    "bool": {
-      "filter": [
-        { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }}
-      ]
-    }
-  },
-  "aggs" : {
-    "min_creationTime" : { "min" : { "field" : "creationTime" } }
-  }
-}
-'
-```
-
-Find out the smallest value of the `creationTime` field of all kustomize resource files:
-```
-curl -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
-{
-  "query": {
-    "bool": {
-      "must_not": {
-        "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }
-      }, 
-      "filter": [
-        { "regexp": { "fileType": "resource"  }}
-      ]
-    }
-  },
-  "aggs" : {
-    "min_creationTime" : { "min" : { "field" : "creationTime" } }
-  }
-}
-'
-```
-
-Find out the smallest value of the `creationTime` field of all kustomize generator files:
-```
-curl -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
-{
-  "query": {
-    "bool": {
-      "must_not": {
-        "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }
-      }, 
-      "filter": [
-        { "regexp": { "fileType": "generator"  }}
-      ]
-    }
-  },
-  "aggs" : {
-    "min_creationTime" : { "min" : { "field" : "creationTime" } }
-  }
-}
-'
-```
-
-Find out the smallest value of the `creationTime` field of all kustomize transformer files:
-```
-curl -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
-{
-  "query": {
-    "bool": {
-      "must_not": {
-        "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }
-      }, 
-      "filter": [
-        { "regexp": { "fileType": "transformer"  }}
-      ]
-    }
-  },
-  "aggs" : {
-    "min_creationTime" : { "min" : { "field" : "creationTime" } }
-  }
-}
-'
-```
-
-Query all the documents whose `creationTime` <= `2016-07-29T17:38:26.000Z`:
-```
-curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
-{
-  "query": {
-    "range": {
-      "creationTime": {
-       "lte": "2016-07-29T17:38:26.000Z"
-      }
-    }
-  }
-}
-'
-```
-
-Query all the documents whose `creationTime` falls within the specific range:
-```
-curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
-{
-  "query": {
-    "range": {
-      "creationTime": {
-       "gte": "2016-07-29T17:38:26.000Z",
-       "lte": "2016-08-29T17:38:26.000Z"
-      }
-    }
-  }
-}
-'
-```
-
-Query all the kustomization files whose `creationTime` falls within the specific range:
-```
-curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
-{
-  "size": 20,
-  "query": {
-    "bool": {
-      "filter": {
-        "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" }
-      },
-      "must": {
-        "range": {
-          "creationTime": {
-            "gte": "2017-09-24T15:49:57.000Z",
-            "lte": "2017-09-24T15:49:57.000Z"
-          }
-        }
-      }
-    }
-  }
-}
-'
-```
-
-Aggregate how many new kustomization files were added into Github each month:
-```
-curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
-{
-    "query": {
-        "bool": {
-            "filter": [
-                { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" }}
-            ]
-        }
-    },
-    "aggs" : {
-        "newFiles_over_time" : {
-            "date_histogram" : {
-                "field" : "creationTime",
-                "interval" : "month"
-            }
-        }
-    }
-}
-'
-```
-
-Aggregate how many new kustomize resource files were added into Github each month:
-```
-curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
-{
-    "query": {
-        "bool": {
-            "must_not": [
-                { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" }}
-            ],
-            "filter": [
-                { "regexp": { "fileType": "resource" }}
-            ]
-        }
-    },
-    "aggs" : {
-        "newFiles_over_time" : {
-            "date_histogram" : {
-                "field" : "creationTime",
-                "interval" : "month"
-            }
-        }
-    }
-}
-'
-```
-
-Aggregate how many new kustomize generator files were added into Github each month:
-```
-curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
-{
-    "query": {
-        "bool": {
-            "must_not": [
-                { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" }}
-            ],
-            "filter": [
-                { "regexp": { "fileType": "generator" }}
-            ]
-        }
-    },
-    "aggs" : {
-        "newFiles_over_time" : {
-            "date_histogram" : {
-                "field" : "creationTime",
-                "interval" : "month"
-            }
-        }
-    }
-}
-'
-```
-
-Aggregate how many new kustomize transformer files were added into Github each month:
-```
-curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
-{
-    "query": {
-        "bool": {
-            "must_not": [
-                { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" }}
-            ],
-            "filter": [
-                { "regexp": { "fileType": "transformer" }}
-            ]
-        }
-    },
-    "aggs" : {
-        "newFiles_over_time" : {
-            "date_histogram" : {
-                "field" : "creationTime",
-                "interval" : "month"
-            }
-        }
-    }
-}
-'
-```
-
-Aggregate how many new kustomization files were added into Github each year:
-```
-curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
-{
-    "query": {
-        "bool": {
-            "filter": [
-                { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" }}
-            ]
-        }
-    },
-    "aggs" : {
-        "newFiles_over_time" : {
-            "date_histogram" : {
-                "field" : "creationTime",
-                "interval" : "year"
-            }
-        }
-    }
-}
-'
-```
-
-Aggregate how many new kustomize resource files were added into Github each year:
-```
-curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
-{
-    "query": {
-        "bool": {
-            "must_not": [
-                { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" }}
-            ],
-            "filter": [
-                { "regexp": { "fileType": "resource" }}
-            ]            
-        }
-    },
-    "aggs" : {
-        "newFiles_over_time" : {
-            "date_histogram" : {
-                "field" : "creationTime",
-                "interval" : "year"
-            }
-        }
-    }
-}
-'
-```
-
-Aggregate how many new kustomize generator files were added into Github each year:
-```
-curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
-{
-    "query": {
-        "bool": {
-            "must_not": [
-                { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" }}
-            ],
-            "filter": [
-                { "regexp": { "fileType": "generator" }}
-            ]            
-        }
-    },
-    "aggs" : {
-        "newFiles_over_time" : {
-            "date_histogram" : {
-                "field" : "creationTime",
-                "interval" : "year"
-            }
-        }
-    }
-}
-'
-```
-
-Aggregate how many new kustomize transformer files were added into Github each year:
-```
-curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
-{
-    "query": {
-        "bool": {
-            "must_not": [
-                { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" }}
-            ],
-            "filter": [
-                { "regexp": { "fileType": "transformer" }}
-            ]            
-        }
-    },
-    "aggs" : {
-        "newFiles_over_time" : {
-            "date_histogram" : {
-                "field" : "creationTime",
-                "interval" : "year"
-            }
-        }
-    }
-}
-'
-```
-
-Find the generator files created within the given time range:
-```
-curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
-{
-  "query": {
-    "bool": {
-      "filter": [
-       { "regexp": { "fileType": "generator" }}
-      ],
-      "must_not": {
-        "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }
-      },
-      "must": {
-        "range": {
-          "creationTime": {
-           "gte": "2019-04-26T16:40:02.000Z",
-           "lte": "2019-04-26T16:40:02.000Z"
-          }
-        }
-      }
-    }
-  }
-}
-'
-```
-
-Find the transformer files created within the given time range:
-```
-curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
-{
-  "query": {
-    "bool": {
-      "filter": [
-       { "regexp": { "fileType": "transformer" }}
-      ],
-      "must_not": {
-        "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }
-      },
-      "must": {
-        "range": {
-          "creationTime": {
-           "gte": "2019-04-26T16:40:02.000Z",
-           "lte": "2019-04-26T16:40:02.000Z"
-          }
-        }
-      }
-    }
-  }
-}
-'
-```

api/internal/crawl/search_cmds/defaultBranch.md

@@ -1,32 +0,0 @@
-Count distinct values of the `defaultBranch` field:
-```
-curl -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
-{
-    "aggs" : {
-        "defaultBranch_count" : {
-            "cardinality" : {
-                "field" : "defaultBranch",
-                "precision_threshold": 40000
-            }
-        }
-    }
-}
-'
-```
-
-List all the github branches where kustomization files and kustomize resource files live, 
-and how many kustomization files and kustomize resource files live in each branch:
-```
-curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
-{
-    "aggs" : {
-        "defaultBranch" : {
-            "terms" : {
-              "field" : "defaultBranch",
-              "size": 41
-            }
-        }
-    }
-}
-'
-```

api/internal/crawl/search_cmds/fieldExistence.md

@@ -1,55 +0,0 @@
-Count the documents whose `document` field is empty (The reason why the `document` field
-of a document is empty is because of empty documents):
-```
-curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
-{
-   "size": 10000,
-    "query": {
-        "bool": {
-            "must_not": {
-                "exists": {
-                    "field": "document"
-                }
-            }
-        }
-    }
-}
-'
-```
-
-Find all the documents having the `creationTime` field set:
-```
-curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
-{
-    "query": {
-        "exists": {
-            "field": "creationTime"
-        }
-    }
-}
-'
-```
-
-Find all the documents whose `creationTime` field is not set:
-```
-curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
-{
-   "size": 10000,
-    "query": {
-        "bool": {
-            "must_not": {
-                "exists": {
-                    "field": "creationTime"
-                }
-            }
-        }
-    }
-}
-'
-```
-
-The following fields of a document in the kustomize index are always non-empty:
-`repositoryUrl`, `filePath`, `defaultBranch`.
-
-The following fields of a document in the kustomize index may be empty:
-`kinds`, `identifiers`, `values`.

api/internal/crawl/search_cmds/fileType.md

@@ -1,301 +0,0 @@
-Find all the documents having the `fileType` field set:
-```
-curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
-{
-    "query": {
-        "exists": {
-            "field": "fileType"
-        }
-    }
-}
-'
-```
-
-Find all the documents whose `fileType` field is not set:
-```
-curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
-{
-   "size": 10000,
-    "query": {
-        "bool": {
-            "must_not": {
-                "exists": {
-                    "field": "fileType"
-                }
-            }
-        }
-    }
-}
-'
-```
-
-Search for all the documents whose `fileType` field is `resource`:
-```
-curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
-{
-  "query": {
-    "bool": {
-      "filter": [
-       { "regexp": { "fileType": "resource" }}
-      ]
-    }
-  }
-}
-'
-```
-
-Search for all the kustomization files whose `fileType` field is `resource`:
-```
-curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
-{
-  "query": {
-    "bool": {
-      "filter": [
-       { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }}, 
-       { "regexp": { "fileType": "resource" }}
-      ]
-    }
-  }
-}
-'
-```
-
-Search for all the kustomize resource files:
-```
-curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
-{
-  "query": {
-    "bool": {
-      "filter": [
-       { "regexp": { "fileType": "resource" }}
-      ],
-      "must_not": {
-        "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }
-      }
-    }
-  }
-}
-'
-```
-
-Search all the kustomization files including a `generators` field:
-```
-curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
-{
-  "size": 10000,
-  "query": {
-    "bool": {
-      "must": {
-        "match" : {
-          "identifiers" : {
-            "query" : "generators"
-          }
-        }
-      },
-      "filter": {
-        "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }
-      }
-    }
-  }
-}
-'
-```
-
-Search for all the documents whose `fileType` field is `generator`:
-```
-curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
-{
-  "query": {
-    "bool": {
-      "filter": [
-       { "regexp": { "fileType": "generator" }}
-      ]
-    }
-  }
-}
-'
-```
-
-Search for all the kustomization files whose `fileType` field is `generator`:
-```
-curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
-{
-  "query": {
-    "bool": {
-      "filter": [
-       { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }}, 
-       { "regexp": { "fileType": "generator" }}
-      ]
-    }
-  }
-}
-'
-```
-
-Search for all the kustomize generator files:
-```
-curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
-{
-  "query": {
-    "bool": {
-      "filter": [
-       { "regexp": { "fileType": "generator" }}
-      ],
-      "must_not": {
-        "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }
-      }
-    }
-  }
-}
-'
-```
-
-Search all the kustomization files including a `transformers` field:
-```
-curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
-{
-  "size": 10000,
-  "query": {
-    "bool": {
-      "must": {
-        "match" : {
-          "identifiers" : {
-            "query" : "transformers"
-          }
-        }
-      },
-      "filter": {
-        "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }
-      }
-    }
-  }
-}
-'
-```
-
-Search for all the documents whose `fileType` field is `transformer`:
-```
-curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
-{
-  "query": {
-    "bool": {
-      "filter": [
-       { "regexp": { "fileType": "transformer" }}
-      ]
-    }
-  }
-}
-'
-```
-
-Search for all the kustomization files whose `fileType` field is `transformer`:
-```
-curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
-{
-  "query": {
-    "bool": {
-      "filter": [
-       { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }}, 
-       { "regexp": { "fileType": "transformer" }}
-      ]
-    }
-  }
-}
-'
-```
-
-Search for all the kustomize transformer files:
-```
-curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
-{
-  "query": {
-    "bool": {
-      "filter": [
-       { "regexp": { "fileType": "transformer" }}
-      ],
-      "must_not": {
-        "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }
-      }
-    }
-  }
-}
-'
-```
-
-Count distinct values of the `fileType` field:
-```
-curl -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
-{
-    "aggs" : {
-        "fileType_count" : {
-            "cardinality" : {
-                "field" : "fileType",
-                "precision_threshold": 40000
-            }
-        }
-    }
-}
-'
-```
-
-List all the values of the `fileType` field and the frequency of each value:
-```
-curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
-{
-    "aggs" : {
-        "fileType" : {
-            "terms" : {
-              "field" : "fileType"
-            }
-        }
-    }
-}
-'
-```
-
-
-For all the kustomization files in the index, list all the values of the
-`fileType` field and the frequency of each value:
-```
-curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
-{
-    "query": {
-        "bool": {
-             "filter": [
-                { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }}
-             ]
-        }
-    },
-    "aggs" : {
-        "fileType" : {
-            "terms" : {
-              "field" : "fileType"
-            }
-        }
-    }
-}
-'
-```
-
-For all the non-kustomization files in the index, list all the values of the
-`fileType` field and the frequency of each value:
-```
-curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
-{
-    "query": {
-        "bool": {
-            "must_not": {
-                "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }
-            }
-        }
-    },
-    "aggs" : {
-        "fileType" : {
-            "terms" : {
-              "field" : "fileType"
-            }
-        }
-    }
-}
-'
-```

api/internal/crawl/search_cmds/generator.md

@@ -1,29 +0,0 @@
-Find all the generator files whose `kinds` field includes `ChartRenderer`, and
-only output certain fields of each document:
-```
-curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
-{
-  "size": 200,
-  "_source": {
-    "includes": ["kinds", "repositoryUrl", "defaultBranch", "filePath"]
-  },
-  "query": {
-    "bool": {
-      "filter": [
-       { "regexp": { "fileType": "generator" }}
-      ],
-      "must_not": {
-        "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }
-      },
-      "must": {
-        "match" : {
-          "kinds" : {
-            "query" : "ChartRenderer"
-          }
-        }
-      }
-    }
-  }
-}
-'
-```

api/internal/crawl/search_cmds/id.md

@@ -1,12 +0,0 @@
-Find the document with the given `_id`:
-```
-curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
-{
-  "query": {
-    "terms": {
-      "_id": [ "b3a03f3327841617db696e2d6abc30e1a1bd653f1a2bbce05637f7dcae1a43f7" ] 
-    }
-  }
-}
-'
-```

api/internal/crawl/search_cmds/keyword_search.md

@@ -1,82 +0,0 @@
-Count the documents in the index whose `repositoryUrl` field starts with
-`https://github.com/`:
-```
-curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
-{
-  "query": {
-    "bool": {
-      "filter": [
-       { "regexp": { "repositoryUrl": "https://github.com/.*" }}
-      ]
-    }
-  }
-}
-'
-```
-
-Count the documents in the index whose `repositoryUrl` field does not start with
-`https://github.com/`:
-```
-curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
-{
-  "query": {
-    "bool": {
-      "must_not": [
-       { "regexp": { "repositoryUrl": "https://github.com/.*" }}
-      ]
-    }
-  }
-}
-'
-```
-
-Search all the documents matching the given `repositoryUrl` and `filePath`, and return 
-a version for each search hit:
-```
-curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
-{
-  "size": 10000,
-  "version": true,
-  "query": {
-    "bool": {
-      "filter": [
-       { "regexp": { "repositoryUrl": "git@github.com:talos-systems/talos-controller-manager" }},
-       { "regexp": { "filePath": "hack/config.*" }}
-      ]
-    }
-  }
-}
-'
-```
-
-Search all the documents whose filePath ends with one of these following three filenames:
-`kustomization.yaml`, `kustomization.yml`, `kustomization`:
-```
-curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
-{
-  "query": {
-    "bool": {
-      "filter": [
-       { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" }}
-      ]
-    }
-  }
-}
-'
-```
-
-Search all the documents whose filePath does not end with any of these following
-three filenames: `kustomization.yaml`, `kustomization.yml`, `kustomization`:
-```
-curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
-{
-  "query": {
-    "bool": {
-      "must_not": [
-       { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" }}
-      ]
-    }
-  }
-}
-'
-```

api/internal/crawl/search_cmds/misc.md

@@ -1,32 +0,0 @@
-Check the health status of an ElasticSearch cluster:
-```
-curl -X GET "${ElasticSearchURL}:9200/_cat/health?v&pretty"
-```
-
-Check the indices in an ElasticSearch cluster:
-```
-curl "${ElasticSearchURL}:9200/_cat/indices?v"
-```
-
-Get the mapping of the index:
-```
-curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_mapping?pretty"
-```
-
-Delete the kustomize index from the ElasticSearch cluster (**Use this command with caution**):
-```
-curl -X DELETE "${ElasticSearchURL}:9200/${INDEXNAME}?pretty"
-```
-
-Add a new field into an existing index.
-```
-curl -X PUT "${ElasticSearchURL}:9200/${INDEXNAME}/_mapping/_doc?pretty" -H 'Content-Type: application/json' -d'
-{
-  "properties": {
-    "fileType": {
-      "type": "keyword"
-    }
-  }
-}
-'
-```

api/internal/crawl/search_cmds/repositoryUrl.md

@@ -1,255 +0,0 @@
-Count distinct values of the `repositoryUrl` field:
-```
-curl -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
-{
-    "aggs" : {
-        "repositoryUrl_count" : {
-            "cardinality" : {
-                "field" : "repositoryUrl",
-                "precision_threshold": 40000
-            }
-        }
-    }
-}
-'
-```
-
-Count how many Github repositories include kustomization files:
-```
-curl -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
-{
-    "query": {
-        "bool": {
-             "filter": [
-                { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" }}
-             ]
-        }
-    },
-    "aggs" : {
-        "repositoryUrl_count" : {
-            "cardinality" : {
-                "field" : "repositoryUrl",
-                "precision_threshold": 40000
-            }
-        }
-    }
-}
-'
-```
-
-Count distinct values of the `repositoryUrl` field for all the kustomize resource files in the index:
-```
-curl -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
-{
-    "query": {
-        "bool": {
-            "must_not": {
-                "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }
-            },
-            "filter": [
-                { "regexp": { "fileType": "resource" }}
-            ]
-        }
-    },
-    "aggs" : {
-        "repositoryUrl_count" : {
-            "cardinality" : {
-                "field" : "repositoryUrl",
-                "precision_threshold": 40000
-            }
-        }
-    }
-}
-'
-```
-
-Count distinct values of the `repositoryUrl` field for all the kustomize generator files in the index:
-```
-curl -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
-{
-    "query": {
-        "bool": {
-            "must_not": {
-                "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }
-            },
-            "filter": [
-                { "regexp": { "fileType": "generator" }}
-            ]
-        }
-    },
-    "aggs" : {
-        "repositoryUrl_count" : {
-            "cardinality" : {
-                "field" : "repositoryUrl",
-                "precision_threshold": 40000
-            }
-        }
-    }
-}
-'
-```
-
-Count distinct values of the `repositoryUrl` field for all the kustomize transformer files in the index:
-```
-curl -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
-{
-    "query": {
-        "bool": {
-            "must_not": {
-                "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }
-            },
-            "filter": [
-                { "regexp": { "fileType": "transformer" }}
-            ]
-        }
-    },
-    "aggs" : {
-        "repositoryUrl_count" : {
-            "cardinality" : {
-                "field" : "repositoryUrl",
-                "precision_threshold": 40000
-            }
-        }
-    }
-}
-'
-```
-
-Count distinct values of the `repositoryUrl` field for all the kustomize resource dirs in the index:
-```
-curl -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
-{
-    "query": {
-        "bool": {
-            "filter": [
-                { "regexp": { "fileType": "resource" }},
-                { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }}
-            ]
-        }
-    },
-    "aggs" : {
-        "repositoryUrl_count" : {
-            "cardinality" : {
-                "field" : "repositoryUrl",
-                "precision_threshold": 40000
-            }
-        }
-    }
-}
-'
-```
-
-Count distinct values of the `repositoryUrl` field for all the kustomize generator dirs in the index:
-```
-curl -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
-{
-    "query": {
-        "bool": {
-            "filter": [
-                { "regexp": { "fileType": "generator" }},
-                { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }}
-            ]
-        }
-    },
-    "aggs" : {
-        "repositoryUrl_count" : {
-            "cardinality" : {
-                "field" : "repositoryUrl",
-                "precision_threshold": 40000
-            }
-        }
-    }
-}
-'
-```
-
-Count distinct values of the `repositoryUrl` field for all the kustomize transformer dirs in the index:
-```
-curl -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
-{
-    "query": {
-        "bool": {
-            "filter": [
-                { "regexp": { "fileType": "transformer" }},
-                { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }}
-            ]
-        }
-    },
-    "aggs" : {
-        "repositoryUrl_count" : {
-            "cardinality" : {
-                "field" : "repositoryUrl",
-                "precision_threshold": 40000
-            }
-        }
-    }
-}
-'
-```
-
-List all the github repositories including kustomization files and kustomize resource files,
-and how many kustomization files and kustomize resource files each github repository includes
-(the github repository including the most kustomization files is listed first):
-```
-curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
-{
-    "aggs" : {
-        "repositoryUrl" : {
-            "terms" : {
-              "field" : "repositoryUrl",
-              "size": 2082
-            }
-        }
-    }
-}
-'
-```
-
-List the top 20 Github repositories including the most amount of kustomization files:
-```
-curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
-{
-    "query": {
-        "bool": {
-             "filter": [
-                { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" }}
-             ]
-        }
-    },
-    "aggs" : {
-        "repositoryUrl" : {
-            "terms" : {
-              "field" : "repositoryUrl",
-              "size": 20
-            }
-        }
-    }
-}
-'
-```
-
-List the top 20 Github repositories including the most amount of kustomize resource files:
-```
-curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
-{
-    "query": {
-        "bool": {
-            "must_not": {
-                "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" }
-            },
-            "filter": [
-                { "regexp": { "fileType": "resource" }}
-            ]
-        }
-    },
-    "aggs" : {
-        "repositoryUrl" : {
-            "terms" : {
-              "field" : "repositoryUrl",
-              "size": 20
-            }
-        }
-    }
-}
-'
-```

api/internal/crawl/search_cmds/snapshot.md

@@ -1,29 +0,0 @@
-Retrieve information about all registered snapshot repositories:
-```
-curl -X GET "${ElasticSearchURL}:9200/_snapshot?pretty"
-```
-
-Retrieve information about a given snapshot repository, `kustomize-backup`:
-```
-curl -X GET "${ElasticSearchURL}:9200/_snapshot/kustomize-backup?pretty"
-```
-
-Verify a snapshot repository, `kustomize-backup`, manually:
-```
-curl -X POST "${ElasticSearchURL}:9200/_snapshot/kustomize-backup/_verify?pretty"
-```
-
-List all the snapshots in a given snapshot repository:
-```
-curl -X GET "${ElasticSearchURL}:9200/_cat/snapshots/kustomize-backup?v&s=id&pretty"
-```
-
-Retrieve a summary information about a given snapshot:
-```
-curl -X GET "${ElasticSearchURL}:9200/_snapshot/kustomize-backup/kustomize-snapshot?pretty"
-```
-
-Retrieve a detailed information about a given snapshot:
-```
-curl -X GET "${ElasticSearchURL}:9200/_snapshot/kustomize-backup/kustomize-snapshot/_status?pretty"
-```

api/internal/crawl/search_cmds/text_search.md

@@ -1,148 +0,0 @@
-Search for all the kustomize resource files including a Deployment object:
-```
-curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
-{
-  "query": {
-    "match" : {
-      "kinds" : {
-        "query" : "Deployment"
-      }
-    }
-  }
-}
-'
-```
-
-Search for all the kustomize resource files including a Deployment object, but only
-including the `kinds` field in the result:
-```
-curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
-{
-  "_source": {
-    "includes": ["kinds"]
-  },
-  "query": {
-    "match" : {
-      "kinds" : {
-        "query" : "Deployment"
-      }
-    }
-  }
-}
-'
-```
-
-Search for all the kustomize resource files including both a Deployment object and 
-a Service object:
-```
-curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
-{
-  "query": {
-    "match" : {
-      "kinds" : {
-        "query" : "Deployment Service",
-        "operator" : "and"
-      }
-    }
-  }
-}
-'
-```
-
-Count the number of documents including Deployment and the number of documents 
-including Service:
-```
-curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
-{
-  "size": 0,
-  "aggs" : {
-    "messages" : {
-      "filters" : {
-        "filters" : {
-          "Deployment" : { "match" : { "kinds" : "Deployment"   }},
-          "Service" : { "match" : { "kinds" : "Service" }}
-        }
-      }
-    }
-  }
-}
-'
-```
-
-Search for all the kustomization files involving CRDs:
-```
-curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
-{
-  "size": 10000,
-  "query": {
-    "match" : {
-      "identifiers" : {
-        "query" : "crds"
-      }
-    }
-  }
-}
-'
-```
-
-Search for all the kustomization files defining configMapGenerator:
-```
-curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
-{
-  "size": 10000,
-  "query": {
-    "match" : {
-      "identifiers" : {
-        "query" : "configMapGenerator"
-      }
-    }
-  }
-}
-'
-```
-
-Search for all the documents having a `kind` field:
-```
-curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
-{
-  "query": {
-    "bool": {
-      "filter": [
-       { "match" : { "identifiers" : { "query" : "kind" }}}
-      ]
-    }
-  }
-}
-'
-```
-
-Search for all the kuostmization files having a `kind` field:
-```
-curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
-{
-  "query": {
-    "bool": {
-      "filter": [
-       { "regexp": { "filePath": ".*/kustomization((.yaml)?|(.yml)?)" }},
-       { "match" : { "identifiers" : { "query" : "kind" }}}
-      ]
-    }
-  }
-}
-'
-```
-
-Search for all the kustomization files defining the `generatorOptions:disableNameSuffixHash` feature:
-```
-curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
-{
-  "query": {
-    "match" : {
-      "identifiers" : {
-        "query" : "generatorOptions:disableNameSuffixHash"
-      }
-    }
-  }
-}
-'
-```

api/internal/crawl/search_cmds/transformer.md

@@ -1,29 +0,0 @@
-Find all the trasnformer files whose `kinds` field includes `HelmValues`, and
-only output certain fields of each document:
-```
-curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
-{
-  "size": 200,
-  "_source": {
-    "includes": ["kinds", "repositoryUrl", "defaultBranch", "filePath"]
-  },
-  "query": {
-    "bool": {
-      "filter": [
-       { "regexp": { "fileType": "transformer" }}
-      ],
-      "must_not": {
-        "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }
-      },
-      "must": {
-        "match" : {
-          "kinds" : {
-            "query" : "HelmValues"
-          }
-        }
-      }
-    }
-  }
-}
-'
-```

api/internal/crawl/search_cmds/user.md

@@ -1,380 +0,0 @@
-Find all the documents having the `user` field set:
-```
-curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
-{
-    "query": {
-        "exists": {
-            "field": "user"
-        }
-    }
-}
-'
-```
-
-Find all the documents whose `user` field is not set:
-```
-curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
-{
-   "size": 10000,
-    "query": {
-        "bool": {
-            "must_not": {
-                "exists": {
-                    "field": "user"
-                }
-            }
-        }
-    }
-}
-'
-```
-
-Search for all the documents whose `user` field is `kubernetes-sigs`:
-```
-curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
-{
-  "query": {
-    "bool": {
-      "filter": [
-       { "regexp": { "user": "kubernetes-sigs" }}
-      ]
-    }
-  }
-}
-'
-```
-
-Count distinct values of the `user` field:
-```
-curl -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
-{
-    "aggs" : {
-        "user_count" : {
-            "cardinality" : {
-                "field" : "user",
-                "precision_threshold": 40000
-            }
-        }
-    }
-}
-'
-```
-
-List all the values of the `user` field and the frequency of each value:
-```
-curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
-{
-    "aggs" : {
-        "user" : {
-            "terms" : {
-                "field" : "user",
-                "size" : 20
-            }
-        }
-    }
-}
-'
-```
-
-Count distinct values of the `user` field for all the kustomization files in the index:
-```
-curl -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
-{
-    "query": {
-        "bool": {
-             "filter": [
-                { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }}
-             ]
-        }
-    },
-    "aggs" : {
-        "user_count" : {
-            "cardinality" : {
-                "field" : "user",
-                "precision_threshold": 40000
-            }
-        }
-    }
-}
-'
-```
-
-For all the kustomization files in the index, list all the values of the
-`user` field and the frequency of each value:
-```
-curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
-{
-    "query": {
-        "bool": {
-             "filter": [
-                { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }}
-             ]
-        }
-    },
-    "aggs" : {
-        "user" : {
-            "terms" : {
-                "field" : "user",
-                "size": 20
-            }
-        }
-    }
-}
-'
-```
-
-Count distinct values of the `user` field for all the kustomize resource files in the index:
-```
-curl -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
-{
-    "query": {
-        "bool": {
-            "must_not": {
-                "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }
-            },
-            "filter": [
-                { "regexp": { "fileType": "resource" }}
-            ]
-        }
-    },
-    "aggs" : {
-        "user_count" : {
-            "cardinality" : {
-                "field" : "user",
-                "precision_threshold": 40000
-            }
-        }
-    }
-}
-'
-```
-
-For all the kustomize resource files in the index, list all the values of the
-`user` field and the frequency of each value:
-```
-curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
-{
-    "query": {
-        "bool": {
-            "must_not": {
-                "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }
-            },
-            "filter": [
-                { "regexp": { "fileType": "resource" }}
-            ]
-        }
-    },
-    "aggs" : {
-        "user" : {
-            "terms" : {
-                "field" : "user",
-                "size": 20
-            }
-        }
-    }
-}
-'
-```
-
-Count distinct values of the `user` field for all the kustomize generator files in the index:
-```
-curl -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
-{
-    "query": {
-        "bool": {
-            "must_not": {
-                "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }
-            },
-            "filter": [
-                { "regexp": { "fileType": "generator" }}
-            ]
-        }
-    },
-    "aggs" : {
-        "user_count" : {
-            "cardinality" : {
-                "field" : "user",
-                "precision_threshold": 40000
-            }
-        }
-    }
-}
-'
-```
-
-For all the kustomize generator files in the index, list all the values of the
-`user` field and the frequency of each value:
-```
-curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
-{
-    "query": {
-        "bool": {
-            "must_not": {
-                "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }
-            },
-            "filter": [
-                { "regexp": { "fileType": "generator" }}
-            ]
-        }
-    },
-    "aggs" : {
-        "user" : {
-            "terms" : {
-              "field" : "user",
-              "size": 20  
-            }
-        }
-    }
-}
-'
-```
-
-Count distinct values of the `user` field for all the kustomize transformer files in the index:
-```
-curl -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
-{
-    "query": {
-        "bool": {
-            "must_not": {
-                "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }
-            },
-            "filter": [
-                { "regexp": { "fileType": "transformer" }}
-            ]
-        }
-    },
-    "aggs" : {
-        "user_count" : {
-            "cardinality" : {
-                "field" : "user",
-                "precision_threshold": 40000
-            }
-        }
-    }
-}
-'
-```
-
-For all the kustomize transformer files in the index, list all the values of the
-`user` field and the frequency of each value:
-```
-curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
-{
-    "query": {
-        "bool": {
-            "must_not": {
-                "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }
-            },
-            "filter": [
-                { "regexp": { "fileType": "transformer" }}
-            ]
-        }
-    },
-    "aggs" : {
-        "user" : {
-            "terms" : {
-                "field" : "user",
-                "size": 20
-            }
-        }
-    }
-}
-'
-```
-
-Count distinct values of the `user` field for all the kustomize generator dirs in the index:
-```
-curl -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
-{
-    "query": {
-        "bool": {
-            "filter": [
-                { "regexp": { "fileType": "generator" }},
-                { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }}
-            ]
-        }
-    },
-    "aggs" : {
-        "user_count" : {
-            "cardinality" : {
-                "field" : "user",
-                "precision_threshold": 40000
-            }
-        }
-    }
-}
-'
-```
-
-For all the kustomize generator dirs in the index, list all the values of the
-`user` field and the frequency of each value:
-```
-curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
-{
-    "query": {
-        "bool": {
-            "filter": [
-                { "regexp": { "fileType": "generator" }},
-                { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }}
-            ]
-        }
-    },
-    "aggs" : {
-        "user" : {
-            "terms" : {
-              "field" : "user",
-              "size": 20  
-            }
-        }
-    }
-}
-'
-```
-
-Count distinct values of the `user` field for all the kustomize transformer dirs in the index:
-```
-curl -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
-{
-    "query": {
-        "bool": {
-            "filter": [
-                { "regexp": { "fileType": "transformer" }},
-                { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }}
-            ]
-        }
-    },
-    "aggs" : {
-        "user_count" : {
-            "cardinality" : {
-                "field" : "user",
-                "precision_threshold": 40000
-            }
-        }
-    }
-}
-'
-```
-
-For all the kustomize transformer dirs in the index, list all the values of the
-`user` field and the frequency of each value:
-```
-curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
-{
-    "query": {
-        "bool": {
-            "filter": [
-                { "regexp": { "fileType": "transformer" }},
-                { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }}
-            ]
-        }
-    },
-    "aggs" : {
-        "user" : {
-            "terms" : {
-              "field" : "user"
-            }
-        }
-    }
-}
-'
-```

api/internal/crawl/utils/utils.go

@@ -1,21 +0,0 @@
-package utils
-
-type SeenMap map[string]string
-
-func (seen SeenMap) Seen(item string) bool {
-	_, ok := seen[item]
-	return ok
-}
-
-func (seen SeenMap) Set(k, v string) {
-	seen[k] = v
-}
-
-// The caller should make sure that key is in the map.
-func (seen SeenMap) Value(k string) string {
-	return seen[k]
-}
-
-func NewSeenMap() SeenMap {
-	return make(map[string]string)
-}

api/internal/git/cloner.go

@@ -27,28 +27,90 @@ func ClonerUsingGitExec(repoSpec *RepoSpec) error {
 	if err != nil {
 		return err
 	}
-
-	if repoSpec.Ref == "" {
-		repoSpec.Ref = "master"
-	}
 	cmd := exec.Command(
 		gitProgram,
-		"clone",
-		"--depth=1",
-		repoSpec.CloneSpec(),
-		"-b",
-		repoSpec.Ref,
+		"init",
 		repoSpec.Dir.String())
 	var out bytes.Buffer
 	cmd.Stdout = &out
 	cmd.Stderr = &out
 	err = cmd.Run()
 	if err != nil {
-		log.Printf("Error cloning git repo: %s", out.String())
+		log.Printf("Error initializing empty git repo: %s", out.String())
 		return errors.Wrapf(
 			err,
-			"trouble cloning git repo %v in %s",
-			repoSpec.CloneSpec(), repoSpec.Dir.String())
+			"trouble initializing empty git repo in %s",
+			repoSpec.Dir.String())
+	}
+
+	cmd = exec.Command(
+		gitProgram,
+		"remote",
+		"add",
+		"origin",
+		repoSpec.CloneSpec())
+	cmd.Stdout = &out
+	cmd.Stderr = &out
+	cmd.Dir = repoSpec.Dir.String()
+	err = cmd.Run()
+	if err != nil {
+		log.Printf("Error setting git remote: %s", out.String())
+		return errors.Wrapf(
+			err,
+			"trouble adding remote %s",
+			repoSpec.CloneSpec())
+	}
+	if repoSpec.Ref == "" {
+		repoSpec.Ref = "master"
+	}
+	cmd = exec.Command(
+		gitProgram,
+		"fetch",
+		"--depth=1",
+		"origin",
+		repoSpec.Ref)
+	cmd.Stdout = &out
+	cmd.Stderr = &out
+	cmd.Dir = repoSpec.Dir.String()
+	err = cmd.Run()
+	if err != nil {
+		cmd = exec.Command(
+			gitProgram,
+			"pull",
+			"origin",
+			"master")
+		var out bytes.Buffer
+		cmd.Stdout = &out
+		cmd.Dir = repoSpec.Dir.String()
+		err := cmd.Run()
+		if err != nil {
+			return errors.Wrapf(err, "trouble pulling %s", repoSpec.OrgRepo)
+		}
+		if repoSpec.Ref == "" {
+			repoSpec.Ref = "master"
+		}
+		cmd = exec.Command(gitProgram, "checkout", repoSpec.Ref)
+		cmd.Dir = repoSpec.Dir.String()
+		err = cmd.Run()
+		if err != nil {
+			return errors.Wrapf(
+				err, "trouble checking out href %s", repoSpec.Ref)
+		}
+	}
+
+	cmd = exec.Command(
+		gitProgram,
+		"reset",
+		"--hard",
+		"FETCH_HEAD")
+	cmd.Stdout = &out
+	cmd.Stderr = &out
+	cmd.Dir = repoSpec.Dir.String()
+	err = cmd.Run()
+	if err != nil {
+		log.Printf("Error performing git reset: %s", out.String())
+		return errors.Wrapf(
+			err, "trouble hard resetting empty repository to %s", repoSpec.Ref)
 	}
 
 	cmd = exec.Command(
@@ -58,11 +120,10 @@ func ClonerUsingGitExec(repoSpec *RepoSpec) error {
 		"--init",
 		"--recursive")
 	cmd.Stdout = &out
-	cmd.Stderr = &out
 	cmd.Dir = repoSpec.Dir.String()
 	err = cmd.Run()
 	if err != nil {
-		return errors.Wrapf(err, "trouble fetching submodules for %s", repoSpec.CloneSpec())
+		return errors.Wrapf(err, "trouble fetching submodules for %s", repoSpec.Ref)
 	}
 
 	return nil

api/internal/tools/tools.go

@@ -4,7 +4,7 @@
 // SPDX-License-Identifier: Apache-2.0
 
 // This file exists to automatically trigger installs
-// of the given tools, and is the official 'unofficial'
+// of the given tools, and is the offical 'unofficial'
 // way to declare a dependence on a Go binary until
 // some better technique comes along.
 

api/krusty/baseandoverlaymedium_test.go

@@ -154,9 +154,8 @@ LEGUME=chickpea
 `)
 	th.WriteF("/app/overlay/configmap/dummy.txt",
 		`Lorem ipsum dolor sit amet, consectetur
- 		 
-adipiscing elit, sed do eiusmod tempor		
-incididunt ut labore et dolore magna aliqua.   
+adipiscing elit, sed do eiusmod tempor
+incididunt ut labore et dolore magna aliqua. 
 `)
 	th.WriteF("/app/overlay/deployment/deployment.yaml", `
 apiVersion: apps/v1
@@ -293,11 +292,8 @@ metadata:
 ---
 apiVersion: v1
 data:
-  nonsense: |
-    Lorem ipsum dolor sit amet, consectetur
-
-    adipiscing elit, sed do eiusmod tempor
-    incididunt ut labore et dolore magna aliqua.
+  nonsense: "Lorem ipsum dolor sit amet, consectetur\nadipiscing elit, sed do eiusmod
+    tempor\nincididunt ut labore et dolore magna aliqua. \n"
 kind: ConfigMap
 metadata:
   annotations:
@@ -306,6 +302,6 @@ metadata:
     app: mungebot
     org: kubernetes
     repo: test-infra
-  name: test-infra-app-config-hh272bg5d4
+  name: test-infra-app-config-f462h769f9
 `)
 }

api/kv/kv.go

@@ -9,7 +9,6 @@ import (
 	"fmt"
 	"os"
 	"path"
-	"regexp"
 	"strings"
 	"unicode"
 	"unicode/utf8"
@@ -86,17 +85,11 @@ func (kvl *loader) keyValuesFromFileSources(sources []string) ([]types.Pair, err
 		if err != nil {
 			return nil, err
 		}
-		kvs = append(kvs, types.Pair{Key: k, Value: trimTrailingSpacesInLines(string(content))})
+		kvs = append(kvs, types.Pair{Key: k, Value: string(content)})
 	}
 	return kvs, nil
 }
 
-// trimTrailingSpacesInLines takes string with multiple lines and trims the trailing white spaces and tabs from each line.
-func trimTrailingSpacesInLines(str string) string {
-	re := regexp.MustCompile(`[ \t]*\n`)
-	return re.ReplaceAllString(str, "\n")
-}
-
 func (kvl *loader) keyValuesFromEnvFiles(paths []string) ([]types.Pair, error) {
 	var kvs []types.Pair
 	for _, p := range paths {

api/kv/kv_test.go

@@ -95,12 +95,3 @@ func TestKeyValuesFromFileSources(t *testing.T) {
 		}
 	}
 }
-
-func TestTrimTrailingSpacesInLines(t *testing.T) {
-	input := "\"fooKey\": \"fooValue\"   \t\n \t\t  \n\t\"barKey\": \"barValue\""
-	expected := "\"fooKey\": \"fooValue\"\n\n\t\"barKey\": \"barValue\""
-	res := trimTrailingSpacesInLines(input)
-	if !reflect.DeepEqual(res, expected) {
-		t.Errorf("Trim trailing spaces in lines should succeed, got: %s exptected: %s", res, expected)
-	}
-}

cmd/config/.golangci.yml

@@ -23,7 +23,7 @@ linters:
     - gofmt
     - goimports
 #    - golint
-#    - gosec
+    - gosec
     - gosimple
     - govet
     - ineffassign

cmd/config/configcobra/cmds.go

@@ -43,29 +43,6 @@ Advanced Documentation Topics:
 `,
 }
 
-// Export commands publicly for composition
-var (
-	Annotate     = commands.AnnotateCommand
-	Cat          = commands.CatCommand
-	Count        = commands.CountCommand
-	CreateSetter = commands.CreateSetterCommand
-	Fmt          = commands.FmtCommand
-	Grep         = commands.GrepCommand
-	ListSetters  = commands.ListSettersCommand
-	Merge        = commands.MergeCommand
-	Merge3       = commands.Merge3Command
-	RunFn        = commands.RunFnCommand
-	Set          = commands.SetCommand
-	Sink         = commands.SinkCommand
-	Source       = commands.SourceCommand
-	Tree         = commands.TreeCommand
-	Wrap         = commands.WrapCommand
-	XArgs        = commands.XArgsCommand
-
-	StackOnError = &commands.StackOnError
-	ExitOnError  = &commands.ExitOnError
-)
-
 // NewConfigCommand returns a new *cobra.Command for the config command group.  This may
 // be embedded into other go binaries as a way of packaging the "config" command as part
 // of another binary.
@@ -92,7 +69,6 @@ func NewConfigCommand(name string) *cobra.Command {
 
 	name = strings.TrimSpace(name + " config")
 	commands.ExitOnError = true
-	root.AddCommand(commands.AnnotateCommand(name))
 	root.AddCommand(commands.GrepCommand(name))
 	root.AddCommand(commands.TreeCommand(name))
 	root.AddCommand(commands.CatCommand(name))
@@ -101,14 +77,8 @@ func NewConfigCommand(name string) *cobra.Command {
 	root.AddCommand(commands.Merge3Command(name))
 	root.AddCommand(commands.CountCommand(name))
 	root.AddCommand(commands.RunFnCommand(name))
-	root.AddCommand(commands.XArgsCommand())
-	root.AddCommand(commands.WrapCommand())
-
 	root.AddCommand(commands.SetCommand(name))
-	root.AddCommand(commands.ListSettersCommand(name))
 	root.AddCommand(commands.CreateSetterCommand(name))
-	root.AddCommand(commands.SinkCommand(name))
-	root.AddCommand(commands.SourceCommand(name))
 
 	root.AddCommand(&cobra.Command{
 		Use:   "docs-merge",
@@ -122,13 +92,8 @@ func NewConfigCommand(name string) *cobra.Command {
 	})
 	root.AddCommand(&cobra.Command{
 		Use:   "docs-fn",
-		Short: "[Alpha] Documentation for developing and invoking Configuration Functions.",
-		Long:  api.FunctionsImplLong,
-	})
-	root.AddCommand(&cobra.Command{
-		Use:   "docs-fn-spec",
-		Short: "[Alpha] Documentation for Configuration Functions Specification.",
-		Long:  api.FunctionsSpecLong,
+		Short: "[Alpha] Documentation for writing containerized functions run by run.",
+		Long:  api.ConfigFnLong,
 	})
 	root.AddCommand(&cobra.Command{
 		Use:   "docs-io-annotations",

cmd/config/docs/api-conventions/config-fn.md

@@ -0,0 +1,281 @@
+# Configuration Functions API Semantics
+
+  Configuration Functions are functions packaged as executables in containers which enable
+  **shift-left practices**.  They configure applications and infrastructure through
+  Kubernetes style Resource Configuration, but run locally pre-commit.
+
+  Configuration functions enable shift-left practices (client-side) through:
+
+  - Pre-commit / delivery validation and linting of configuration
+    - e.g. Fail if any containers don't have PodSecurityPolicy or CPU / Memory limits
+  - Implementation of abstractions as client actuated APIs (e.g. templating)
+    - e.g. Create a client-side *"CRD"* for generating configuration checked into git
+  - Aspect Orient configuration / Injection of cross-cutting configuration
+    - e.g. T-Shirt size containers by annotating Resources with `small`, `medium`, `large`
+      and inject the cpu and memory resources into containers accordingly.
+    - e.g. Inject `init` and `side-car` containers into Resources based off of Resource
+      Type, annotations, etc.
+
+  Performing these on the client rather than the server enables:
+
+  - Configuration to be reviewed prior to being sent to the API server
+  - Configuration to be validated as part of the CD pipeline
+  - Configuration for Resources to validated holistically rather than individually
+    per-Resource -- e.g. ensure the `Service.selector` and `Deployment.spec.template` labels
+    match.
+    - MutatingWebHooks are scoped to a single Resource instance at a time.
+  - Low-level tweaks to the output of high-level abstractions -- e.g. add an `init container`
+    to a client *"CRD"* Resource after it was generated.
+  - Composition and layering of multiple functions together
+    - Compose generation, injection, validation together
+
+  Configuration Functions are implemented as executable programs published in containers which:
+
+  - Accept as input (stdin):
+    - A list of Resource Configuration
+    - A Function Configuration (to configure the function itself)
+  - Emit as output (stdout + exit):
+    - A list of Resource Configuration
+    - An exit code for success / failure
+
+### Function Specification
+
+  - Functions **SHOULD** be published as container images containing a `CMD` invoking an executable.
+  - Functions **MUST** accept input on STDIN a `ResourceList` containing the Resources and
+    `functionConfig`.
+  - Functions **MUST** emit output on STDOUT a `ResourceList` containing the modified
+    Resources.
+  - Functions **MUST** exit non-0 on failure, and exit 0 on success.
+  - Functions **MAY** emit output on STDERR with error messaging.
+  - Functions performing validation **SHOULD** exit failure and emit error messaging
+    on a validation failure.
+  - Functions generating Resources **SHOULD** retain non-conflicting changes on the
+    generated Resources -- e.g. 1. the function generates a Deployment, but doesn't
+    specify `cpu`, 2. the user sets the `cpu` on the generated Resource, 3. the
+    function should keep the `cpu` when regenerating the Resource a second time.
+  - Functions **SHOULD** be usable outside `kustomize config run` -- e.g. though pipeline
+    mechanisms such as Tekton.
+
+#### Input Format
+
+  Functions must accept on STDIN:
+
+  `ResourceList`:
+  - contains `items` field, same as `List.items`
+  - contains `functionConfig` field -- a single item with the configuration for the function itself
+
+  Example `ResourceList` Input:
+
+    apiVersion: config.kubernetes.io/v1alpha1
+    kind: ResourceList
+    functionConfig:
+      apiVersion: example.com/v1beta1
+      kind: Nginx
+      metadata:
+        name: my-instance
+        annotations:
+          config.kubernetes.io/local-config: "true"
+      spec:
+        replicas: 5
+    items:
+    -  apiVersion: apps/v1
+       kind: Deployment
+       metadata:
+         name: my-instance
+       spec:
+         replicas: 3
+         ...
+    - apiVersion: v1
+      kind: Service
+      metadata:
+        name: my-instance
+      spec:
+        ...
+
+#### Output Format
+
+  Functions must emit on STDOUT:
+
+  `ResourceList`:
+  - contains `items` field, same as `List.items`
+
+  Example `ResourceList` Output:
+
+    apiVersion: config.kubernetes.io/v1alpha1
+    kind: ResourceList
+    items:
+    -  apiVersion: apps/v1
+       kind: Deployment
+       metadata:
+         name: my-instance
+       spec:
+         replicas: 5
+         ...
+    - apiVersion: v1
+      kind: Service
+      metadata:
+        name: my-instance
+      spec:
+        ...
+
+#### Container Environment
+
+  When run by `kustomize config run`, functions are run in containers with the
+  following environment:
+
+  - Network: `none`
+  - User: `nobody`
+  - Security Options: `no-new-privileges`
+  - Volumes: the volume containing the `functionConfig` yaml is mounted under `/local` as `ro`
+
+### Example Function Implementation
+
+  Following is an example for implementing an nginx abstraction using a config
+  function.
+
+#### `nginx-template.sh`
+
+  `nginx-template.sh` is a simple bash script which uses a *heredoc* as a templating solution
+  for generating Resources from the functionConfig input fields.
+
+  The script wraps itself using `config run wrap -- $0` which will:
+
+  1. Parse the `ResourceList.functionConfig` (provided to the container stdin) into env vars
+  2. Merge the stdout into the original list of Resources
+  3. Defaults filenames for newly generated Resources (if they are not set as annotations)
+     to `config/NAME_KIND.yaml`
+  4. Format the output
+
+    #!/bin/bash
+    # script must run wrapped by `kustomize config run wrap`
+    # for parsing input the functionConfig into env vars
+    if [ -z ${WRAPPED} ]; then
+      export WRAPPED=true
+      config run wrap -- $0
+      exit $?
+    fi
+
+    cat <<End-of-message
+    apiVersion: v1
+    kind: Service
+    metadata:
+      name: ${NAME}
+      labels:
+        app: nginx
+        instance: ${NAME}
+    spec:
+      ports:
+      - port: 80
+        targetPort: 80
+        name: http
+      selector:
+        app: nginx
+        instance: ${NAME}
+    ---
+    apiVersion: apps/v1
+    kind: Deployment
+    metadata:
+      name: ${NAME}
+      labels:
+        app: nginx
+        instance: ${NAME}
+    spec:
+      replicas: ${REPLICAS}
+      selector:
+        matchLabels:
+          app: nginx
+          instance: ${NAME}
+      template:
+        metadata:
+          labels:
+            app: nginx
+            instance: ${NAME}
+        spec:
+          containers:
+          - name: nginx
+            image: nginx:1.7.9
+            ports:
+            - containerPort: 80
+    End-of-message
+
+#### `Dockerfile`
+
+  `Dockerfile` installs `kustomize config` and copies the script into the container image.
+
+    FROM golang:1.13-stretch
+    RUN go get sigs.k8s.io/kustomize/cmd/config
+    RUN mv /go/bin/config /usr/bin/config
+    COPY nginx-template.sh /usr/bin/nginx-template.sh
+    CMD ["nginx-template.sh]
+
+### Example Function Usage
+
+Following is an example of running the `kustomize config run` using the preceding API.
+
+#### `nginx.yaml` (Input)
+
+  `dir/nginx.yaml` contains a reference to the Function.  The contents of `nginx.yaml`
+  are passed to the Function through the `ResourceList.functionConfig` field.
+
+    apiVersion: example.com/v1beta1
+    kind: Nginx
+    metadata:
+      name: my-instance
+      annotations:
+        config.kubernetes.io/local-config: "true"
+      configFn:
+        container:
+          image: gcr.io/example-functions/nginx-template:v1.0.0
+    spec:
+      replicas: 5
+
+  - `configFn.container.image`: the image to use for this API
+  - `annotations[config.kubernetes.io/local-config]`: mark this as not a Resource that should
+    be applied
+
+#### `kustomize config run dir/` (Output)
+
+  `dir/my-instance_deployment.yaml` contains the Deployment:
+
+    apiVersion: apps/v1
+    kind: Deployment
+    metadata:
+      name: my-instance
+      labels:
+        app: nginx
+        instance: my-instance
+    spec:
+      replicas: 5
+      selector:
+        matchLabels:
+          app: nginx
+          instance: my-instance
+      template:
+        metadata:
+          labels:
+            app: nginx
+            instance: my-instance
+        spec:
+          containers:
+          - name: nginx
+            image: nginx:1.7.9
+            ports:
+            - containerPort: 80
+
+  `dir/my-instance_service.yaml` contains the Service:
+
+    apiVersion: v1
+    kind: Service
+    metadata:
+      name: my-instance
+      labels:
+        app: nginx
+        instance: my-instance
+    spec:
+      ports:
+      - port: 80
+        targetPort: 80
+        name: http
+      selector:
+        app: nginx
+        instance: my-instance

cmd/config/docs/api-conventions/config-io.md

@@ -29,7 +29,7 @@ metadata:
 ### `config.kubernetes.io/index`
 
 Records the index of a Resource in file. In a multi-object YAML file, Resources are separated
-by three dashes (`---`), and the index represents the position of the Resource starting from zero.
+by three dashes (`---`), and the index represents the positon of the Resource starting from zero.
 
 This annotation **SHOULD** be set when reading Resources from files.
 It **SHOULD** be unset when writing Resources to files.

cmd/config/docs/api-conventions/functions-impl.md

@@ -1,181 +0,0 @@
-# Running Configuration Functions using kustomize CLI
-
-Configuration functions can be implemented using any toolchain and invoked using any
-container workflow orchestrator including Tekton, Cloud Build, or run directly using `docker run`.
-
-Run `config help docs-fn-spec` to see the Configuration Functions Specification.
-
-`kustomize config run` is an example orchestrator for invoking Configuration Functions. This
-document describes how to implement and invoke an example function.
-
-## Example Function Implementation
-
-Following is an example for implementing an nginx abstraction using a configuration
-function.
-
-### `nginx-template.sh`
-
-`nginx-template.sh` is a simple bash script which uses a _heredoc_ as a templating solution
-for generating Resources from the functionConfig input fields.
-
-The script wraps itself using `config run wrap -- $0` which will:
-
-1. Parse the `ResourceList.functionConfig` (provided to the container stdin) into env vars
-2. Merge the stdout into the original list of Resources
-3. Defaults filenames for newly generated Resources (if they are not set as annotations)
-   to `config/NAME_KIND.yaml`
-4. Format the output
-
-```bash
-#!/bin/bash
-# script must run wrapped by "kustomize config run wrap"
-# for parsing input the functionConfig into env vars
-if [ -z ${WRAPPED} ]; then
-  export WRAPPED=true
-  config run wrap -- $0
-  exit $?
-fi
-
-cat <<End-of-message
-apiVersion: v1
-kind: Service
-metadata:
-  name: ${NAME}
-  labels:
-    app: nginx
-    instance: ${NAME}
-spec:
-  ports:
-  - port: 80
-    targetPort: 80
-    name: http
-  selector:
-    app: nginx
-    instance: ${NAME}
----
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: ${NAME}
-  labels:
-    app: nginx
-    instance: ${NAME}
-spec:
-  replicas: ${REPLICAS}
-  selector:
-    matchLabels:
-      app: nginx
-      instance: ${NAME}
-  template:
-    metadata:
-      labels:
-        app: nginx
-        instance: ${NAME}
-    spec:
-      containers:
-      - name: nginx
-        image: nginx:1.7.9
-        ports:
-        - containerPort: 80
-End-of-message
-```
-
-### Dockerfile
-
-`Dockerfile` installs `kustomize config` and copies the script into the container image.
-
-```
-FROM golang:1.13-stretch
-RUN go get sigs.k8s.io/kustomize/cmd/config
-RUN mv /go/bin/config /usr/bin/config
-COPY nginx-template.sh /usr/bin/nginx-template.sh
-CMD ["nginx-template.sh]
-```
-
-## Example Function Usage
-
-Following is an example of running the `kustomize config run` using the preceding API.
-
-When run by `kustomize config run`, functions are run in containers with the
-following environment:
-
-- Network: `none`
-- User: `nobody`
-- Security Options: `no-new-privileges`
-- Volumes: the volume containing the `functionConfig` yaml is mounted under `/local` as `ro`
-
-### Input
-
-`dir/nginx.yaml` contains a reference to the Function. The contents of `nginx.yaml`
-are passed to the Function through the `ResourceList.functionConfig` field.
-
-```yaml
-apiVersion: example.com/v1beta1
-kind: Nginx
-metadata:
-  name: my-instance
-  annotations:
-    config.kubernetes.io/local-config: "true"
-    config.k8s.io/function: |
-      container:
-        image: gcr.io/example-functions/nginx-template:v1.0.0
-spec:
-  replicas: 5
-```
-
-- `annotations[config.k8s.io/function].container.image`: the image to use for this API
-- `annotations[config.kubernetes.io/local-config]`: mark this as not a Resource that should
-  be applied
-
-### Output
-
-The function is invoked using byrunning `kustomize config run dir/`.
-
-`dir/my-instance_deployment.yaml` contains the Deployment:
-
-```yaml
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: my-instance
-  labels:
-    app: nginx
-    instance: my-instance
-spec:
-  replicas: 5
-  selector:
-    matchLabels:
-      app: nginx
-      instance: my-instance
-  template:
-    metadata:
-      labels:
-        app: nginx
-        instance: my-instance
-    spec:
-      containers:
-        - name: nginx
-          image: nginx:1.7.9
-          ports:
-            - containerPort: 80
-```
-
-`dir/my-instance_service.yaml` contains the Service:
-
-```yaml
-apiVersion: v1
-kind: Service
-metadata:
-  name: my-instance
-  labels:
-    app: nginx
-    instance: my-instance
-spec:
-  ports:
-    - port: 80
-      targetPort: 80
-      name: http
-  selector:
-    app: nginx
-    instance: my-instance
-```

cmd/config/docs/api-conventions/functions-spec.md

@@ -1,186 +0,0 @@
-# Configuration Functions Specification
-
-This document specifies a standard for client-side functions that operate on
-Kubernetes declarative configurations. This standard enables creating
-small, interoperable, and language-independent executable programs packaged as
-containers that can be chained together as part of a configuration management pipeline.
-The end result of such a pipeline are fully rendered configurations that can then be
-applied to a control plane (e.g. Using ‘kubectl apply’ for Kubernetes control plane).
-As such, although this document references Kubernetes Resource Model and API conventions,
-it is completely decoupled from Kuberentes API machinery and does not depend on any
-in-cluster components.
-
-This document references terms described in [Kubernetes API Conventions][1].
-
-The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", "SHOULD",
-"SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this document are to be
-interpreted as described in [RFC 2119][2].
-
-## Use Cases
-
-_Configuration functions_ enable shift-left practices (client-side) through:
-
-- Pre-commit / delivery validation and linting of configuration
-  - e.g. Fail if any containers don't have PodSecurityPolicy or CPU / Memory limits
-- Implementation of abstractions as client actuated APIs (e.g. templating)
-  - e.g. Create a client-side _"CRD"_ for generating configuration checked into git
-- Aspect Orient configuration / Injection of cross-cutting configuration
-  - e.g. T-Shirt size containers by annotating Resources with `small`, `medium`, `large`
-    and inject the cpu and memory resources into containers accordingly.
-  - e.g. Inject `init` and `side-car` containers into Resources based off of Resource
-    Type, annotations, etc.
-
-Performing these on the client rather than the server enables:
-
-- Configuration to be reviewed prior to being sent to the API server
-- Configuration to be validated as part of the CI?CD pipeline
-- Configuration for Resources to validated holistically rather than individually
-  per-Resource
-  - e.g. ensure the `Service.selector` and `Deployment.spec.template` labels
-    match.
-  - e.g. MutatingWebHooks are scoped to a single Resource instance at a time.
-- Low-level tweaks to the output of high-level abstractions
-  - e.g. add an `init container` to a client _"CRD"_ Resource after it was generated.
-- Composition and layering of multiple functions together
-  - Compose generation, injection, validation together
-
-## Spec
-
-### Input Type
-
-A function MUST accept as input a single [Kubernetes List type][3].
-The `items` field in the input will contain a sequence of [Object types][3].
-A function MAY not support [Simple types][3] and List types.
-
-An example using `v1/ConfigMapList` as input:
-
-```yaml
-apiVersion: v1
-kind: ConfigMapList
-items:
-  - apiVersion: v1
-    kind: ConfigMap
-    metadata:
-      name: config1
-    data:
-      p1: v1
-      p2: v2
-  - apiVersion: v1
-    kind: ConfigMap
-    metadata:
-      name: config2
-```
-
-An example using `v1/List` as input:
-
-```yaml
-apiVersion: v1
-kind: List
-items:
-  spec:
-   - apiVersion: foo-corp.com/v1
-    kind: FulfillmentCenter
-    metadata:
-      name: staging
-    address: "100 Main St."
-  - apiVersion: rbac.authorization.k8s.io/v1
-    kind: ClusterRole
-    metadata:
-      name: namespace-reader
-    rules:
-      - resources:
-          - namespaces
-        apiGroups:
-          - ""
-        verbs:
-          - get
-          - watch
-          - list
-```
-
-In addition, a function MUST accept as input a List of kind `ResourceList` where the
-`functionConfig` field, if present, will contain the invocation-specific configuration passed to the function
-by the orchestrator.
-Functions MAY consider this field optional so that they can be triggered in an ad-hoc fashion.
-
-An example using `config.kubernetes.io/v1beta1/ResourceList` as input:
-
-```yaml
-apiVersion: config.kubernetes.io/v1beta1
-kind: ResourceList
-functionConfig:
-  apiVersion: foo-corp.com/v1
-  kind: FulfillmentCenter
-  metadata:
-    name: staging
-    metadata:
-      annotations:
-        config.k8s.io/function: |
-          container:
-            image: gcr.io/example/foo:v1.0.0
-  spec:
-    address: "100 Main St."
-items:
-  - apiVersion: rbac.authorization.k8s.io/v1
-    kind: ClusterRole
-    metadata:
-      name: namespace-reader
-    rules:
-      - resources:
-          - namespaces
-        apiGroups:
-          - ""
-        verbs:
-          - get
-          - watch
-          - list
-```
-
-Here `FulfillmentCenter` kind with name `staging` is passed as the invocation-specific configuration
-to the function.
-
-### Output Type
-
-A function’s output MUST be the same as the input specification above
--- i.e. `ResourceList` or `List`.
-This is necessary to enable chaining two or more functions together in a pipeline.
-The serialization format of the output SHOULD match that of its input on each invocation
--- e.g. if the input was a `ResourceList`, the output should also be a `ResourceList`.
-
-### Serialization Format
-
-A function MUST support YAML as a serialization format for the input and output.
-A function MUST use utf8 encoding (as YAML is a superset of JSON, JSON will also be supported
-by any conforming function).
-
-### Operations
-
-A function MAY Create, Update, or Delete any number of items in the `items` field and output the
-resultant list.
-
-A function MAY modify annotations with prefix `config.kubernetes.io`, but must be careful about
-doing so since they’re used for orchestration purposes and will likely impact subsequent functions
-in the pipeline.
-
-A function SHOULD preserve comments when input serialization format is YAML.
-This allows for human authoring of configuration to coexist with changes made by functions.
-
-### Containerization
-
-A function MUST be implemented as a container.
-
-A function container MUST be capable of running as a non-root user if it does not require
-access to host filesystem or makes network calls.
-
-### stdin/stdout/stderr and Exit Codes
-
-A function MUST accept input from stdin and emit output to stdout.
-
-Any error messages MUST be emitted to stderr.
-
-An exit code of zero indicates function execution was successful.
-A non-zero exit code indicates a failure.
-
-[1]: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md
-[2]: https://tools.ietf.org/html/rfc2119
-[3]: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#types-kinds

cmd/config/docs/commands/annotate.md

@@ -1,18 +0,0 @@
-## annotate
-
-[Alpha] Set an annotation on Resources.
-
-### Synopsis
-
-[Alpha]  Set an annotation on Resources.
-
-  DIR:
-    Path to local directory.
-
-### Examples
-
-    kustomize config annotate my-dir/ --kv foo=bar
-
-    kustomize config annotate my-dir/ --kv foo=bar --kv a=b
-
-    kustomize config annotate my-dir/ --kv foo=bar --kind Deployment --name foo

cmd/config/docs/commands/list-setters.md

@@ -1,23 +0,0 @@
-## set
-
-[Alpha] List setters for Resources.
-
-### Synopsis
-
-List setters for Resources.
-
-  DIR
-
-    A directory containing Resource configuration.
-
-  NAME
-
-    Optional.  The name of the setter to display.
-
-### Examples
-
-  Show setters:
-
-    $ config set DIR/
-        NAME      DESCRIPTION   VALUE     TYPE     COUNT   SETBY  
-    name-prefix   ''            PREFIX    string   2

cmd/config/docs/commands/run-fns.md

@@ -22,8 +22,8 @@ order they appear in the file).
 
 #### Config Functions:
 
-  Config functions are specified as Kubernetes types containing a metadata.annotations.[config.kubernetes.io/function]
-  field specifying an image for the container to run.  This image tells run how to invoke the container.
+  Config functions are specified as Kubernetes types containing a metadata.configFn.container.image
+  field.  This field tells run how to invoke the container.
 
   Example config function:
 
@@ -31,17 +31,17 @@ order they appear in the file).
 	apiVersion: fn.example.com/v1beta1
 	kind: ExampleFunctionKind
 	metadata:
+	  configFn:
+	    container:
+	      # function is invoked as a container running this image
+	      image: gcr.io/example/examplefunction:v1.0.1
 	  annotations:
-	    config.kubernetes.io/function: |
-	      container:
-	        # function is invoked as a container running this image
-	        image: gcr.io/example/examplefunction:v1.0.1
 	    config.kubernetes.io/local-config: "true" # tools should ignore this
 	spec:
 	  configField: configValue
 
   In the preceding example, 'kustomize config run example/' would identify the function by
-  the metadata.annotations.[config.kubernetes.io/function] field.  It would then write all Resources in the directory to
+  the metadata.configFn field.  It would then write all Resources in the directory to
   a container stdin (running the gcr.io/example/examplefunction:v1.0.1 image).  It
   would then write the container stdout back to example/, replacing the directory
   file contents.

cmd/config/docs/commands/set.md

@@ -59,19 +59,19 @@ To create a custom setter for a field see: `kustomize help config create-setter`
   List setters: Show the possible setters
 
     $ config set DIR/
-        NAME      DESCRIPTION   VALUE     TYPE     COUNT   SETBY  
+    NAME          DESCRIPTION    VALUE     TYPE    COUNT   OWNER
     name-prefix   ''            PREFIX    string   2
 
-  Perform set: set a new value, owner and description
+  Perform substitution: set a new value, owner and description
 
     $ kustomize config set DIR/ name-prefix "test" --description "test environment" --set-by "dev"
-    set 2 values
+    performed 2 substitutions
 
-  List setters: Show the new values
+  Show substitutions: Show the new values
 
-    $ config set DIR/
-        NAME      DESCRIPTION         VALUE     TYPE     COUNT     SETBY 
-    name-prefix   'test environment'   test     string   2          dev
+    $ config set dir
+    NAME       DESCRIPTION       VALUE    TYPE    COUNT   SUBSTITUTED   OWNER
+    prefix   'test environment'   test   string   2       true          dev
 
   New Resource YAML:
 

cmd/config/docs/commands/sink.md

@@ -1,18 +0,0 @@
-## sink
-
-[Alpha] Implement a Sink by writing input to a local directory.
-
-### Synopsis
-
-[Alpha] Implement a Sink by writing input to a local directory.
-
-    kustomize config sink [DIR]
-
-  DIR:
-    Path to local directory.  If unspecified, sink will write to stdout as if it were a single file.
-
-`sink` writes its input to a directory
-
-### Examples
-
-    kustomize config source DIR/ | your-function | kustomize config sink DIR/

cmd/config/docs/commands/source.md

@@ -1,22 +0,0 @@
-## source
-
-[Alpha] Implement a Source by reading a local directory.
-
-### Synopsis
-
-[Alpha] Implement a Source by reading a local directory.
-
-    kustomize config source DIR...
-
-  DIR:
-    One or more paths to local directories.  Contents from directories will be concatenated.
-    If no directories are provided, source will read from stdin as if it were a single file.
-
-`source` emits configuration to act as input to a function
-
-### Examples
-
-    # emity configuration directory as input source to a function
-    kustomize config source DIR/
-
-    kustomize config source DIR/ | your-function | kustomize config sink DIR/

cmd/config/go.mod

@@ -9,8 +9,7 @@ require (
 	github.com/spf13/cobra v0.0.5
 	github.com/spf13/pflag v1.0.5
 	github.com/stretchr/testify v1.4.0
+	gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect
 	k8s.io/apimachinery v0.17.0
-	sigs.k8s.io/kustomize/kyaml v0.0.0
+	sigs.k8s.io/kustomize/kyaml v0.0.4
 )
-
-replace sigs.k8s.io/kustomize/kyaml v0.0.0 => ../../kyaml

cmd/config/go.sum

@@ -101,8 +101,6 @@ github.com/posener/complete/v2 v2.0.1-alpha.12/go.mod h1://JlL91cS2JV7rOl6LVHrRq
 github.com/posener/script v1.0.4 h1:nSuXW5ZdmFnQIueLB2s0qvs4oNsUloM1Zydzh75v42w=
 github.com/posener/script v1.0.4/go.mod h1:Rg3ijooqulo05aGLyGsHoLmIOUzHUVK19WVgrYBPU/E=
 github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
-github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0=
-github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
 github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
 github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
 github.com/spf13/cobra v0.0.5 h1:f0B+LkLX6DtmRH1isoNA9VTtNUK9K8xYd28JNNfOv/s=
@@ -164,5 +162,7 @@ k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8
 k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk=
 k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I=
 k8s.io/kube-openapi v0.0.0-20191107075043-30be4d16710a/go.mod h1:1TqjTSzOxsLGIKfj0lK8EeCP7K1iUG65v09OM0/WG5E=
+sigs.k8s.io/kustomize/kyaml v0.0.4 h1:kN8+jANVkVVdMCfgIcG42Yd4uy4Jd3wZaGx5DpF6MUE=
+sigs.k8s.io/kustomize/kyaml v0.0.4/go.mod h1:waxTrzQRK9i6/5fR5HNo8xa4YwvWn8t85vMnOGFEZik=
 sigs.k8s.io/structured-merge-diff v0.0.0-20190525122527-15d366b2352e/go.mod h1:wWxsB5ozmmv/SG7nM11ayaAW51xMvak/t1r0CSlcokI=
 sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=

cmd/config/internal/commands/annotate.go

@@ -1,103 +0,0 @@
-// Copyright 2019 The Kubernetes Authors.
-// SPDX-License-Identifier: Apache-2.0
-
-package commands
-
-import (
-	"strings"
-
-	"github.com/spf13/cobra"
-	"sigs.k8s.io/kustomize/cmd/config/internal/generateddocs/commands"
-	"sigs.k8s.io/kustomize/kyaml/errors"
-	"sigs.k8s.io/kustomize/kyaml/kio"
-	"sigs.k8s.io/kustomize/kyaml/yaml"
-)
-
-// NewAnnotateRunner returns a command runner.
-func NewAnnotateRunner(parent string) *AnnotateRunner {
-	r := &AnnotateRunner{}
-	c := &cobra.Command{
-		Use:     "annotate [DIR]",
-		Args:    cobra.MaximumNArgs(1),
-		Short:   commands.AnnotateShort,
-		Long:    commands.AnnotateLong,
-		Example: commands.AnnotateExamples,
-		RunE:    r.runE,
-	}
-	fixDocs(parent, c)
-	r.Command = c
-	c.Flags().StringVar(&r.Kind, "kind", "", "Resource kind to annotate")
-	c.Flags().StringVar(&r.ApiVersion, "apiVersion", "", "Resource apiVersion to annotate")
-	c.Flags().StringVar(&r.Name, "name", "", "Resource name to annotate")
-	c.Flags().StringVar(&r.Namespace, "namespace", "", "Resource namespace to annotate")
-	c.Flags().StringSliceVar(&r.Values, "kv", []string{}, "annotation as KEY=VALUE")
-	return r
-}
-
-func AnnotateCommand(parent string) *cobra.Command {
-	return NewAnnotateRunner(parent).Command
-}
-
-type AnnotateRunner struct {
-	Command    *cobra.Command
-	Values     []string
-	Kind       string
-	Name       string
-	ApiVersion string
-	Namespace  string
-	Path       string
-}
-
-func (r *AnnotateRunner) runE(c *cobra.Command, args []string) error {
-	var input []kio.Reader
-	var output []kio.Writer
-	if len(args) == 0 {
-		rw := &kio.ByteReadWriter{Reader: c.InOrStdin(), Writer: c.OutOrStdout()}
-		input = []kio.Reader{rw}
-		output = []kio.Writer{rw}
-	} else {
-		rw := &kio.LocalPackageReadWriter{PackagePath: args[0], NoDeleteFiles: true}
-		input = []kio.Reader{rw}
-		output = []kio.Writer{rw}
-	}
-	return handleError(c, kio.Pipeline{
-		Inputs:  input,
-		Filters: []kio.Filter{r},
-		Outputs: output,
-	}.Execute())
-}
-
-func (r *AnnotateRunner) Filter(nodes []*yaml.RNode) ([]*yaml.RNode, error) {
-	for i := range nodes {
-		n := nodes[i]
-		m, err := n.GetMeta()
-		if err != nil {
-			return nil, err
-		}
-		if r.Kind != "" && r.Kind != m.Kind {
-			continue
-		}
-		if r.ApiVersion != "" && r.ApiVersion != m.APIVersion {
-			continue
-		}
-		if r.Namespace != "" && r.Namespace != m.Namespace {
-			continue
-		}
-		if r.Name != "" && r.Name != m.Name {
-			continue
-		}
-
-		for i := range r.Values {
-			// split key, value pairs
-			kv := strings.SplitN(r.Values[i], "=", 2)
-			if len(kv) != 2 {
-				return nil, errors.Errorf("must specify --kv as KEY=VALUE: %s", r.Values[i])
-			}
-			if err := n.PipeE(yaml.SetAnnotation(kv[0], kv[1])); err != nil {
-				return nil, err
-			}
-		}
-
-	}
-	return nodes, nil
-}

cmd/config/internal/commands/annotate_test.go

@@ -1,488 +0,0 @@
-// Copyright 2019 The Kubernetes Authors.
-// SPDX-License-Identifier: Apache-2.0
-
-package commands
-
-import (
-	"bytes"
-	"io/ioutil"
-	"os"
-	"path/filepath"
-	"strings"
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-	"sigs.k8s.io/kustomize/kyaml/kio"
-)
-
-func TestAnnotateCommand(t *testing.T) {
-	var tests = []struct {
-		name     string
-		args     []string
-		expected string
-	}{
-		{
-			name:     "single value",
-			args:     []string{"--kv", "a=b"},
-			expected: expectedSingleValue,
-		},
-		{
-			name:     "multi value",
-			args:     []string{"--kv", "a=b", "--kv", "c=d"},
-			expected: expectedMultiValue,
-		},
-		{
-			name:     "filter kind",
-			args:     []string{"--kv", "a=b", "--kind", "Service"},
-			expected: expectedFilterKindService,
-		},
-		{
-			name:     "filter apiVersion",
-			args:     []string{"--kv", "a=b", "--apiVersion", "v1"},
-			expected: expectedFilterApiVersionV1,
-		},
-		{
-			name:     "filter name",
-			args:     []string{"--kv", "a=b", "--name", "bar"},
-			expected: expectedFilterNameBar,
-		},
-		{
-			name:     "filter namespace",
-			args:     []string{"--kv", "a=b", "--namespace", "bar"},
-			expected: expectedFilterNamespaceBar,
-		},
-	}
-	for i := range tests {
-		tt := tests[i]
-		t.Run(tt.name, func(t *testing.T) {
-			d := initTestDir(t)
-			defer os.RemoveAll(d)
-
-			a := NewAnnotateRunner("")
-			a.Command.SetArgs(append([]string{d}, tt.args...))
-			a.Command.SilenceUsage = true
-			a.Command.SilenceErrors = true
-
-			err := a.Command.Execute()
-			if !assert.NoError(t, err) {
-				t.FailNow()
-			}
-
-			actual := &bytes.Buffer{}
-			err = kio.Pipeline{
-				Inputs:  []kio.Reader{kio.LocalPackageReader{PackagePath: d}},
-				Outputs: []kio.Writer{kio.ByteWriter{Writer: actual, KeepReaderAnnotations: true}},
-			}.Execute()
-			if !assert.NoError(t, err) {
-				t.FailNow()
-			}
-			if !assert.Equal(t,
-				strings.TrimSpace(tt.expected),
-				strings.TrimSpace(actual.String())) {
-				t.FailNow()
-			}
-		})
-	}
-}
-
-func initTestDir(t *testing.T) string {
-	d, err := ioutil.TempDir("", "kustomize-annotate-test")
-	if !assert.NoError(t, err) {
-		t.FailNow()
-	}
-	err = ioutil.WriteFile(filepath.Join(d, "f1.yaml"), []byte(f1Input), 0600)
-	if !assert.NoError(t, err) {
-		defer os.RemoveAll(d)
-		t.FailNow()
-	}
-	err = ioutil.WriteFile(filepath.Join(d, "f2.yaml"), []byte(f2Input), 0600)
-	if !assert.NoError(t, err) {
-		defer os.RemoveAll(d)
-		t.FailNow()
-	}
-	return d
-}
-
-var (
-	f1Input = `
-kind: Deployment
-metadata:
-  labels:
-    app: nginx2
-  name: foo
-  annotations:
-    app: nginx2
-spec:
-  replicas: 1
----
-kind: Service
-metadata:
-  name: foo
-  annotations:
-    app: nginx
-spec:
-  selector:
-    app: nginx
-`
-
-	f2Input = `
-apiVersion: v1
-kind: Abstraction
-metadata:
-  name: foo
-  configFn:
-    container:
-      image: gcr.io/example/reconciler:v1
-  annotations:
-    config.kubernetes.io/local-config: "true"
-  namespace: bar
-spec:
-  replicas: 3
----
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  labels:
-    app: nginx
-  name: bar
-  annotations:
-    app: nginx
-  namespace: foo
-spec:
-  replicas: 3
-`
-
-	expectedSingleValue = `kind: Deployment
-metadata:
-  labels:
-    app: nginx2
-  name: foo
-  annotations:
-    app: nginx2
-    a: 'b'
-    config.kubernetes.io/index: '0'
-    config.kubernetes.io/path: 'f1.yaml'
-spec:
-  replicas: 1
----
-kind: Service
-metadata:
-  name: foo
-  annotations:
-    app: nginx
-    a: 'b'
-    config.kubernetes.io/index: '1'
-    config.kubernetes.io/path: 'f1.yaml'
-spec:
-  selector:
-    app: nginx
----
-apiVersion: v1
-kind: Abstraction
-metadata:
-  name: foo
-  configFn:
-    container:
-      image: gcr.io/example/reconciler:v1
-  annotations:
-    config.kubernetes.io/local-config: "true"
-    a: 'b'
-    config.kubernetes.io/index: '0'
-    config.kubernetes.io/path: 'f2.yaml'
-  namespace: bar
-spec:
-  replicas: 3
----
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  labels:
-    app: nginx
-  name: bar
-  annotations:
-    app: nginx
-    a: 'b'
-    config.kubernetes.io/index: '1'
-    config.kubernetes.io/path: 'f2.yaml'
-  namespace: foo
-spec:
-  replicas: 3
-`
-
-	expectedMultiValue = `kind: Deployment
-metadata:
-  labels:
-    app: nginx2
-  name: foo
-  annotations:
-    app: nginx2
-    a: 'b'
-    c: 'd'
-    config.kubernetes.io/index: '0'
-    config.kubernetes.io/path: 'f1.yaml'
-spec:
-  replicas: 1
----
-kind: Service
-metadata:
-  name: foo
-  annotations:
-    app: nginx
-    a: 'b'
-    c: 'd'
-    config.kubernetes.io/index: '1'
-    config.kubernetes.io/path: 'f1.yaml'
-spec:
-  selector:
-    app: nginx
----
-apiVersion: v1
-kind: Abstraction
-metadata:
-  name: foo
-  configFn:
-    container:
-      image: gcr.io/example/reconciler:v1
-  annotations:
-    config.kubernetes.io/local-config: "true"
-    a: 'b'
-    c: 'd'
-    config.kubernetes.io/index: '0'
-    config.kubernetes.io/path: 'f2.yaml'
-  namespace: bar
-spec:
-  replicas: 3
----
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  labels:
-    app: nginx
-  name: bar
-  annotations:
-    app: nginx
-    a: 'b'
-    c: 'd'
-    config.kubernetes.io/index: '1'
-    config.kubernetes.io/path: 'f2.yaml'
-  namespace: foo
-spec:
-  replicas: 3
-`
-
-	expectedFilterKindService = `kind: Deployment
-metadata:
-  labels:
-    app: nginx2
-  name: foo
-  annotations:
-    app: nginx2
-    config.kubernetes.io/index: '0'
-    config.kubernetes.io/path: 'f1.yaml'
-spec:
-  replicas: 1
----
-kind: Service
-metadata:
-  name: foo
-  annotations:
-    app: nginx
-    a: 'b'
-    config.kubernetes.io/index: '1'
-    config.kubernetes.io/path: 'f1.yaml'
-spec:
-  selector:
-    app: nginx
----
-apiVersion: v1
-kind: Abstraction
-metadata:
-  name: foo
-  configFn:
-    container:
-      image: gcr.io/example/reconciler:v1
-  annotations:
-    config.kubernetes.io/local-config: "true"
-    config.kubernetes.io/index: '0'
-    config.kubernetes.io/path: 'f2.yaml'
-  namespace: bar
-spec:
-  replicas: 3
----
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  labels:
-    app: nginx
-  name: bar
-  annotations:
-    app: nginx
-    config.kubernetes.io/index: '1'
-    config.kubernetes.io/path: 'f2.yaml'
-  namespace: foo
-spec:
-  replicas: 3
-`
-
-	expectedFilterApiVersionV1 = `kind: Deployment
-metadata:
-  labels:
-    app: nginx2
-  name: foo
-  annotations:
-    app: nginx2
-    config.kubernetes.io/index: '0'
-    config.kubernetes.io/path: 'f1.yaml'
-spec:
-  replicas: 1
----
-kind: Service
-metadata:
-  name: foo
-  annotations:
-    app: nginx
-    config.kubernetes.io/index: '1'
-    config.kubernetes.io/path: 'f1.yaml'
-spec:
-  selector:
-    app: nginx
----
-apiVersion: v1
-kind: Abstraction
-metadata:
-  name: foo
-  configFn:
-    container:
-      image: gcr.io/example/reconciler:v1
-  annotations:
-    config.kubernetes.io/local-config: "true"
-    a: 'b'
-    config.kubernetes.io/index: '0'
-    config.kubernetes.io/path: 'f2.yaml'
-  namespace: bar
-spec:
-  replicas: 3
----
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  labels:
-    app: nginx
-  name: bar
-  annotations:
-    app: nginx
-    config.kubernetes.io/index: '1'
-    config.kubernetes.io/path: 'f2.yaml'
-  namespace: foo
-spec:
-  replicas: 3
-`
-
-	expectedFilterNameBar = `kind: Deployment
-metadata:
-  labels:
-    app: nginx2
-  name: foo
-  annotations:
-    app: nginx2
-    config.kubernetes.io/index: '0'
-    config.kubernetes.io/path: 'f1.yaml'
-spec:
-  replicas: 1
----
-kind: Service
-metadata:
-  name: foo
-  annotations:
-    app: nginx
-    config.kubernetes.io/index: '1'
-    config.kubernetes.io/path: 'f1.yaml'
-spec:
-  selector:
-    app: nginx
----
-apiVersion: v1
-kind: Abstraction
-metadata:
-  name: foo
-  configFn:
-    container:
-      image: gcr.io/example/reconciler:v1
-  annotations:
-    config.kubernetes.io/local-config: "true"
-    config.kubernetes.io/index: '0'
-    config.kubernetes.io/path: 'f2.yaml'
-  namespace: bar
-spec:
-  replicas: 3
----
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  labels:
-    app: nginx
-  name: bar
-  annotations:
-    app: nginx
-    a: 'b'
-    config.kubernetes.io/index: '1'
-    config.kubernetes.io/path: 'f2.yaml'
-  namespace: foo
-spec:
-  replicas: 3
-`
-
-	expectedFilterNamespaceBar = `kind: Deployment
-metadata:
-  labels:
-    app: nginx2
-  name: foo
-  annotations:
-    app: nginx2
-    config.kubernetes.io/index: '0'
-    config.kubernetes.io/path: 'f1.yaml'
-spec:
-  replicas: 1
----
-kind: Service
-metadata:
-  name: foo
-  annotations:
-    app: nginx
-    config.kubernetes.io/index: '1'
-    config.kubernetes.io/path: 'f1.yaml'
-spec:
-  selector:
-    app: nginx
----
-apiVersion: v1
-kind: Abstraction
-metadata:
-  name: foo
-  configFn:
-    container:
-      image: gcr.io/example/reconciler:v1
-  annotations:
-    config.kubernetes.io/local-config: "true"
-    a: 'b'
-    config.kubernetes.io/index: '0'
-    config.kubernetes.io/path: 'f2.yaml'
-  namespace: bar
-spec:
-  replicas: 3
----
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  labels:
-    app: nginx
-  name: bar
-  annotations:
-    app: nginx
-    config.kubernetes.io/index: '1'
-    config.kubernetes.io/path: 'f2.yaml'
-  namespace: foo
-spec:
-  replicas: 3
-`
-)

cmd/config/internal/commands/cat.go

@@ -121,13 +121,6 @@ func (r *CatRunner) runE(c *cobra.Command, args []string) error {
 		out = o
 	}
 
-	// remove this annotation explicitly, the ByteWriter won't clear it by
-	// default because it doesn't set it
-	clear := []string{"config.kubernetes.io/path"}
-	if r.KeepAnnotations {
-		clear = nil
-	}
-
 	var outputs []kio.Writer
 	outputs = append(outputs, kio.ByteWriter{
 		Writer:                out,
@@ -136,7 +129,6 @@ func (r *CatRunner) runE(c *cobra.Command, args []string) error {
 		WrappingAPIVersion:    r.WrapApiVersion,
 		FunctionConfig:        functionConfig,
 		Style:                 yaml.GetStyle(r.Styles...),
-		ClearAnnotations:      clear,
 	})
 
 	return handleError(c, kio.Pipeline{Inputs: inputs, Filters: fltr, Outputs: outputs}.Execute())

cmd/config/internal/commands/cat_test.go

@@ -90,6 +90,8 @@ metadata:
   name: foo
   annotations:
     app: nginx2
+    config.kubernetes.io/package: '.'
+    config.kubernetes.io/path: 'f1.yaml'
 spec:
   replicas: 1
 ---
@@ -98,6 +100,8 @@ metadata:
   name: foo
   annotations:
     app: nginx
+    config.kubernetes.io/package: '.'
+    config.kubernetes.io/path: 'f1.yaml'
 spec:
   selector:
     app: nginx
@@ -110,6 +114,8 @@ metadata:
     app: nginx
   annotations:
     app: nginx
+    config.kubernetes.io/package: '.'
+    config.kubernetes.io/path: 'f2.yaml'
 spec:
   replicas: 3
 `, b.String()) {
@@ -190,6 +196,8 @@ metadata:
   name: foo
   annotations:
     app: nginx2
+    config.kubernetes.io/package: '.'
+    config.kubernetes.io/path: 'f1.yaml'
 spec:
   replicas: 1
 ---
@@ -198,6 +206,8 @@ metadata:
   name: foo
   annotations:
     app: nginx
+    config.kubernetes.io/package: '.'
+    config.kubernetes.io/path: 'f1.yaml'
 spec:
   selector:
     app: nginx
@@ -208,6 +218,8 @@ metadata:
   name: foo
   annotations:
     config.kubernetes.io/local-config: "true"
+    config.kubernetes.io/package: '.'
+    config.kubernetes.io/path: 'f2.yaml'
   configFn:
     container:
       image: gcr.io/example/image:version
@@ -221,6 +233,8 @@ metadata:
   name: bar
   annotations:
     app: nginx
+    config.kubernetes.io/package: '.'
+    config.kubernetes.io/path: 'f2.yaml'
 spec:
   replicas: 3
 `, b.String()) {
@@ -300,6 +314,8 @@ metadata:
   name: foo
   annotations:
     config.kubernetes.io/local-config: "true"
+    config.kubernetes.io/package: '.'
+    config.kubernetes.io/path: 'f2.yaml'
   configFn:
     container:
       image: gcr.io/example/reconciler:v1
@@ -398,6 +414,8 @@ metadata:
   name: foo
   annotations:
     app: nginx2
+    config.kubernetes.io/package: '.'
+    config.kubernetes.io/path: 'f1.yaml'
 spec:
   replicas: 1
 ---
@@ -406,6 +424,8 @@ metadata:
   name: foo
   annotations:
     app: nginx
+    config.kubernetes.io/package: '.'
+    config.kubernetes.io/path: 'f1.yaml'
 spec:
   selector:
     app: nginx
@@ -418,6 +438,8 @@ metadata:
     app: nginx
   annotations:
     app: nginx
+    config.kubernetes.io/package: '.'
+    config.kubernetes.io/path: 'f2.yaml'
 spec:
   replicas: 3
 `, string(actual)) {
@@ -514,6 +536,8 @@ metadata:
   name: foo
   annotations:
     app: nginx2
+    config.kubernetes.io/package: '.'
+    config.kubernetes.io/path: 'f1.yaml'
 spec:
   replicas: 1
 ---
@@ -522,6 +546,8 @@ metadata:
   name: foo
   annotations:
     app: nginx
+    config.kubernetes.io/package: '.'
+    config.kubernetes.io/path: 'f1.yaml'
 spec:
   selector:
     app: nginx
@@ -534,6 +560,8 @@ metadata:
     app: nginx
   annotations:
     app: nginx
+    config.kubernetes.io/package: '.'
+    config.kubernetes.io/path: 'f2.yaml'
 spec:
   replicas: 3
 `, string(actual)) {

cmd/config/internal/commands/cmdcreatesetter.go

@@ -34,10 +34,9 @@ func NewCreateSetterRunner(parent string) *CreateSetterRunner {
 		"kind of the Resource on which to create the setter.")
 	set.Flags().StringVar(&r.Set.SetPartialField.Type, "type", "",
 		"valid OpenAPI field type -- e.g. integer,boolean,string.")
-	set.Flags().BoolVar(&r.Set.SetPartialField.Partial, "partial", false,
-		"create a partial setter for only part of the field value.")
 	fixDocs(parent, set)
 	set.MarkFlagRequired("type")
+	set.MarkFlagRequired("field")
 	r.Command = set
 	return r
 }

cmd/config/internal/commands/cmdlistsetters.go

@@ -1,47 +0,0 @@
-// Copyright 2019 The Kubernetes Authors.
-// SPDX-License-Identifier: Apache-2.0
-
-package commands
-
-import (
-	"github.com/spf13/cobra"
-	"sigs.k8s.io/kustomize/cmd/config/internal/generateddocs/commands"
-	"sigs.k8s.io/kustomize/kyaml/setters"
-)
-
-// NewListSettersRunner returns a command runner.
-func NewListSettersRunner(parent string) *ListSettersRunner {
-	r := &ListSettersRunner{}
-	c := &cobra.Command{
-		Use:     "list-setters DIR [NAME]",
-		Args:    cobra.RangeArgs(1, 2),
-		Short:   commands.ListSettersShort,
-		Long:    commands.ListSettersLong,
-		Example: commands.ListSettersExamples,
-		PreRunE: r.preRunE,
-		RunE:    r.runE,
-	}
-	fixDocs(parent, c)
-	r.Command = c
-	return r
-}
-
-func ListSettersCommand(parent string) *cobra.Command {
-	return NewListSettersRunner(parent).Command
-}
-
-type ListSettersRunner struct {
-	Command *cobra.Command
-	Lookup  setters.LookupSetters
-}
-
-func (r *ListSettersRunner) preRunE(c *cobra.Command, args []string) error {
-	if len(args) > 1 {
-		r.Lookup.Name = args[1]
-	}
-	return nil
-}
-
-func (r *ListSettersRunner) runE(c *cobra.Command, args []string) error {
-	return handleError(c, lookup(r.Lookup, c, args))
-}

cmd/config/internal/commands/cmdset.go

@@ -5,7 +5,6 @@ package commands
 
 import (
 	"fmt"
-	"os"
 
 	"github.com/olekukonko/tablewriter"
 	"github.com/spf13/cobra"
@@ -28,10 +27,6 @@ func NewSetRunner(parent string) *SetRunner {
 	}
 	fixDocs(parent, c)
 	r.Command = c
-	c.Flags().StringVar(&r.Perform.SetBy, "set-by", "",
-		"annotate the field with who set it")
-	c.Flags().StringVar(&r.Perform.Description, "description", "",
-		"annotate the field with a description of its value")
 
 	return r
 }
@@ -64,14 +59,14 @@ func (r *SetRunner) runE(c *cobra.Command, args []string) error {
 		return handleError(c, r.perform(c, args))
 	}
 
-	return handleError(c, lookup(r.Lookup, c, args))
+	return handleError(c, r.lookup(c, args))
 }
 
-func lookup(l setters.LookupSetters, c *cobra.Command, args []string) error {
+func (r *SetRunner) lookup(c *cobra.Command, args []string) error {
 	// lookup the setters
 	err := kio.Pipeline{
 		Inputs:  []kio.Reader{&kio.LocalPackageReader{PackagePath: args[0]}},
-		Filters: []kio.Filter{&l},
+		Filters: []kio.Filter{&r.Lookup},
 	}.Execute()
 	if err != nil {
 		return err
@@ -87,8 +82,8 @@ func lookup(l setters.LookupSetters, c *cobra.Command, args []string) error {
 	table.SetHeader([]string{
 		"NAME", "DESCRIPTION", "VALUE", "TYPE", "COUNT", "SETBY",
 	})
-	for i := range l.SetterCounts {
-		s := l.SetterCounts[i]
+	for i := range r.Lookup.SetterCounts {
+		s := r.Lookup.SetterCounts[i]
 		v := s.Value
 		if s.Value == "" {
 			v = s.Value
@@ -103,11 +98,6 @@ func lookup(l setters.LookupSetters, c *cobra.Command, args []string) error {
 		})
 	}
 	table.Render()
-
-	if len(l.SetterCounts) == 0 {
-		// exit non-0 if no matching setters are found
-		os.Exit(1)
-	}
 	return nil
 }
 

cmd/config/internal/commands/fmt.go

@@ -31,8 +31,6 @@ formatting substitution verbs {'%n': 'metadata.name', '%s': 'metadata.namespace'
 		`if true, keep index and filename annotations set on Resources.`)
 	c.Flags().BoolVar(&r.Override, "override", false,
 		`if true, override existing filepath annotations.`)
-	c.Flags().BoolVar(&r.UseSchema, "use-schema", false,
-		`if true, uses openapi resource schema to format resources.`)
 	r.Command = c
 	return r
 }
@@ -48,7 +46,6 @@ type FmtRunner struct {
 	SetFilenames    bool
 	KeepAnnotations bool
 	Override        bool
-	UseSchema       bool
 }
 
 func (r *FmtRunner) preRunE(c *cobra.Command, args []string) error {
@@ -59,9 +56,7 @@ func (r *FmtRunner) preRunE(c *cobra.Command, args []string) error {
 }
 
 func (r *FmtRunner) runE(c *cobra.Command, args []string) error {
-	f := []kio.Filter{filters.FormatFilter{
-		UseSchema: r.UseSchema,
-	}}
+	f := []kio.Filter{filters.FormatFilter{}}
 
 	// format with file names
 	if r.SetFilenames {

cmd/config/internal/commands/grep_test.go

@@ -76,6 +76,7 @@ metadata:
   annotations:
     app: nginx2
     config.kubernetes.io/index: '0'
+    config.kubernetes.io/package: '.'
     config.kubernetes.io/path: 'f1.yaml'
 spec:
   replicas: 1
@@ -86,6 +87,7 @@ metadata:
   annotations:
     app: nginx
     config.kubernetes.io/index: '1'
+    config.kubernetes.io/package: '.'
     config.kubernetes.io/path: 'f1.yaml'
 spec:
   selector:

cmd/config/internal/commands/merge3.go

@@ -25,8 +25,6 @@ func GetMerge3Runner(name string) *Merge3Runner {
 		"Path to updated package")
 	c.Flags().StringVar(&r.toDir, "to", "",
 		"Path to destination package")
-	c.Flags().BoolVar(&r.path, "path-merge-key", false,
-		"Use the path as part of the merge key when merging resources")
 
 	r.Command = c
 	return r
@@ -42,7 +40,6 @@ type Merge3Runner struct {
 	ancestor string
 	fromDir  string
 	toDir    string
-	path     bool
 }
 
 func (r *Merge3Runner) runE(c *cobra.Command, args []string) error {
@@ -50,7 +47,6 @@ func (r *Merge3Runner) runE(c *cobra.Command, args []string) error {
 		OriginalPath: r.ancestor,
 		UpdatedPath:  r.fromDir,
 		DestPath:     r.toDir,
-		MergeOnPath:  r.path,
 	}.Merge()
 	if err != nil {
 		return err

cmd/config/internal/commands/run-fns.go

@@ -4,27 +4,22 @@
 package commands
 
 import (
-	"fmt"
-	"io"
-	"strings"
-
 	"github.com/spf13/cobra"
 	"sigs.k8s.io/kustomize/cmd/config/internal/generateddocs/commands"
-	"sigs.k8s.io/kustomize/kyaml/errors"
 	"sigs.k8s.io/kustomize/kyaml/runfn"
-	"sigs.k8s.io/kustomize/kyaml/yaml"
 )
 
 // GetCatRunner returns a RunFnRunner.
 func GetRunFnRunner(name string) *RunFnRunner {
 	r := &RunFnRunner{}
 	c := &cobra.Command{
-		Use:     "run [DIR]",
+		Use:     "run DIR",
+		Aliases: []string{"run-fns"},
 		Short:   commands.RunFnsShort,
 		Long:    commands.RunFnsLong,
 		Example: commands.RunFnsExamples,
 		RunE:    r.runE,
-		PreRunE: r.preRunE,
+		Args:    cobra.ExactArgs(1),
 	}
 	fixDocs(name, c)
 	c.Flags().BoolVar(&r.IncludeSubpackages, "include-subpackages", true,
@@ -32,14 +27,11 @@ func GetRunFnRunner(name string) *RunFnRunner {
 	r.Command = c
 	r.Command.Flags().BoolVar(
 		&r.DryRun, "dry-run", false, "print results to stdout")
-	r.Command.Flags().BoolVar(
-		&r.GlobalScope, "global-scope", false, "set global scope for functions.")
 	r.Command.Flags().StringSliceVar(
 		&r.FnPaths, "fn-path", []string{},
-		"read functions from these directories instead of the configuration directory.")
-	r.Command.Flags().StringVar(
-		&r.Image, "image", "",
-		"run this image as a function instead of discovering them.")
+		"directories containing functions without configuration")
+	r.Command.AddCommand(XArgsCommand())
+	r.Command.AddCommand(WrapCommand())
 	return r
 }
 
@@ -52,144 +44,13 @@ type RunFnRunner struct {
 	IncludeSubpackages bool
 	Command            *cobra.Command
 	DryRun             bool
-	GlobalScope        bool
 	FnPaths            []string
-	Image              string
-	RunFns             runfn.RunFns
 }
 
 func (r *RunFnRunner) runE(c *cobra.Command, args []string) error {
-	return handleError(c, r.RunFns.Execute())
-}
-
-// getFunctions parses the commandline flags and arguments into explicit
-// Functions to run.
-func (r *RunFnRunner) getFunctions(c *cobra.Command, args, dataItems []string) (
-	[]*yaml.RNode, error) {
-	// if image isn't specified, then Functions is empty
-	if r.Image == "" {
-		return nil, nil
-	}
-
-	// create the function spec to set as an annotation
-	fn, err := yaml.Parse(`container: {}`)
-	if err != nil {
-		return nil, err
-	}
-	// TODO: add support network, volumes, etc based on flag values
-	err = fn.PipeE(
-		yaml.Lookup("container"),
-		yaml.SetField("image", yaml.NewScalarRNode(r.Image)))
-	if err != nil {
-		return nil, err
-	}
-
-	// create the function config
-	rc, err := yaml.Parse(`
-metadata:
-  name: function-input
-data: {}
-`)
-	if err != nil {
-		return nil, err
-	}
-
-	// set the function annotation on the function config so it
-	// is parsed by RunFns
-	value, err := fn.String()
-	if err != nil {
-		return nil, err
-	}
-	err = rc.PipeE(
-		yaml.LookupCreate(yaml.MappingNode, "metadata", "annotations"),
-		yaml.SetField("config.kubernetes.io/function", yaml.NewScalarRNode(value)))
-	if err != nil {
-		return nil, err
-	}
-
-	// default the function config kind to ConfigMap, this may be overridden
-	var kind = "ConfigMap"
-	var version = "v1"
-
-	// populate the function config with data.  this is a convention for functions
-	// to be more commandline friendly
-	if len(dataItems) > 0 {
-		dataField, err := rc.Pipe(yaml.Lookup("data"))
-		if err != nil {
-			return nil, err
-		}
-		for i, s := range dataItems {
-			kv := strings.SplitN(s, "=", 2)
-			if i == 0 && len(kv) == 1 {
-				// first argument may be the kind
-				kind = s
-				continue
-			}
-			if len(kv) != 2 {
-				return nil, fmt.Errorf("args must have keys and values separated by =")
-			}
-			err := dataField.PipeE(yaml.SetField(kv[0], yaml.NewScalarRNode(kv[1])))
-			if err != nil {
-				return nil, err
-			}
-		}
-	}
-	err = rc.PipeE(yaml.SetField("kind", yaml.NewScalarRNode(kind)))
-	if err != nil {
-		return nil, err
-	}
-	err = rc.PipeE(yaml.SetField("apiVersion", yaml.NewScalarRNode(version)))
-	if err != nil {
-		return nil, err
-	}
-	return []*yaml.RNode{rc}, nil
-}
-
-func (r *RunFnRunner) preRunE(c *cobra.Command, args []string) error {
-	if c.ArgsLenAtDash() >= 0 && r.Image == "" {
-		return errors.Errorf("must specify --image")
-	}
-
-	var dataItems []string
-	if c.ArgsLenAtDash() >= 0 {
-		dataItems = args[c.ArgsLenAtDash():]
-		args = args[:c.ArgsLenAtDash()]
-	}
-	if len(args) > 1 {
-		return errors.Errorf("0 or 1 arguments supported, function arguments go after '--'")
-	}
-
-	fns, err := r.getFunctions(c, args, dataItems)
-	if err != nil {
-		return err
-	}
-
-	// set the output to stdout if in dry-run mode or no arguments are specified
-	var output io.Writer
-	var input io.Reader
-	if len(args) == 0 {
-		output = c.OutOrStdout()
-		input = c.InOrStdin()
-	} else if r.DryRun {
-		output = c.OutOrStdout()
-	}
-
-	// set the path if specified as an argument
-	var path string
-	if len(args) == 1 {
-		// argument is the directory
-		path = args[0]
-	}
-
-	r.RunFns = runfn.RunFns{
-		FunctionPaths: r.FnPaths,
-		GlobalScope:   r.GlobalScope,
-		Functions:     fns,
-		Output:        output,
-		Input:         input,
-		Path:          path,
-	}
-
-	// don't consider args for the function
-	return nil
+	rec := runfn.RunFns{Path: args[0], FunctionPaths: r.FnPaths}
+	if r.DryRun {
+		rec.Output = c.OutOrStdout()
+	}
+	return handleError(c, rec.Execute())
 }

cmd/config/internal/commands/run_test.go

@@ -1,232 +0,0 @@
-// Copyright 2019 The Kubernetes Authors.
-// SPDX-License-Identifier: Apache-2.0
-
-package commands
-
-import (
-	"io"
-	"os"
-	"strings"
-	"testing"
-
-	"github.com/spf13/cobra"
-	"github.com/stretchr/testify/assert"
-)
-
-// TestRunFnCommand_preRunE verifies that preRunE correctly parses the commandline
-// flags and arguments into the RunFns structure to be executed.
-func TestRunFnCommand_preRunE(t *testing.T) {
-	tests := []struct {
-		name          string
-		args          []string
-		expected      string
-		err           string
-		path          string
-		input         io.Reader
-		output        io.Writer
-		functionPaths []string
-	}{
-		{
-			name: "config map",
-			args: []string{"run", "dir", "--image", "foo:bar", "--", "a=b", "c=d", "e=f"},
-			path: "dir",
-			expected: `
-metadata:
-  name: function-input
-  annotations:
-    config.kubernetes.io/function: |
-      container: {image: 'foo:bar'}
-data: {a: b, c: d, e: f}
-kind: ConfigMap
-apiVersion: v1
-`,
-		},
-		{
-			name:   "config map stdin / stdout",
-			args:   []string{"run", "--image", "foo:bar", "--", "a=b", "c=d", "e=f"},
-			input:  os.Stdin,
-			output: os.Stdout,
-			expected: `
-metadata:
-  name: function-input
-  annotations:
-    config.kubernetes.io/function: |
-      container: {image: 'foo:bar'}
-data: {a: b, c: d, e: f}
-kind: ConfigMap
-apiVersion: v1
-`,
-		},
-		{
-			name:   "config map dry-run",
-			args:   []string{"run", "dir", "--image", "foo:bar", "--dry-run", "--", "a=b", "c=d", "e=f"},
-			output: os.Stdout,
-			path:   "dir",
-			expected: `
-metadata:
-  name: function-input
-  annotations:
-    config.kubernetes.io/function: |
-      container: {image: 'foo:bar'}
-data: {a: b, c: d, e: f}
-kind: ConfigMap
-apiVersion: v1
-`,
-		},
-		{
-			name: "config map no args",
-			args: []string{"run", "dir", "--image", "foo:bar"},
-			path: "dir",
-			expected: `
-metadata:
-  name: function-input
-  annotations:
-    config.kubernetes.io/function: |
-      container: {image: 'foo:bar'}
-data: {}
-kind: ConfigMap
-apiVersion: v1
-`,
-		},
-		{
-			name: "custom kind",
-			args: []string{"run", "dir", "--image", "foo:bar", "--", "Foo", "g=h"},
-			path: "dir",
-			expected: `
-metadata:
-  name: function-input
-  annotations:
-    config.kubernetes.io/function: |
-      container: {image: 'foo:bar'}
-data: {g: h}
-kind: Foo
-apiVersion: v1
-`,
-		},
-		{
-			name: "custom kind '=' in data",
-			args: []string{"run", "dir", "--image", "foo:bar", "--", "Foo", "g=h", "i=j=k"},
-			path: "dir",
-			expected: `
-metadata:
-  name: function-input
-  annotations:
-    config.kubernetes.io/function: |
-      container: {image: 'foo:bar'}
-data: {g: h, i: j=k}
-kind: Foo
-apiVersion: v1
-`,
-		},
-		{
-			name:          "function paths",
-			args:          []string{"run", "dir", "--fn-path", "path1", "--fn-path", "path2"},
-			path:          "dir",
-			functionPaths: []string{"path1", "path2"},
-		},
-		{
-			name: "custom kind with function paths",
-			args: []string{
-				"run", "dir", "--fn-path", "path", "--image", "foo:bar", "--", "Foo", "g=h", "i=j=k"},
-			path:          "dir",
-			functionPaths: []string{"path"},
-			expected: `
-metadata:
-  name: function-input
-  annotations:
-    config.kubernetes.io/function: |
-      container: {image: 'foo:bar'}
-data: {g: h, i: j=k}
-kind: Foo
-apiVersion: v1
-`,
-		},
-		{
-			name: "config map multi args",
-			args: []string{"run", "dir", "dir2", "--image", "foo:bar", "--", "a=b", "c=d", "e=f"},
-			err:  "0 or 1 arguments supported",
-		},
-		{
-			name: "config map not image",
-			args: []string{"run", "dir", "--", "a=b", "c=d", "e=f"},
-			err:  "must specify --image",
-		},
-		{
-			name: "config map bad data",
-			args: []string{"run", "dir", "--image", "foo:bar", "--", "a=b", "c", "e=f"},
-			err:  "must have keys and values separated by",
-		},
-	}
-
-	for i := range tests {
-		tt := tests[i]
-		t.Run(tt.name, func(t *testing.T) {
-			r := GetRunFnRunner("kustomize")
-			// Don't run the actual command
-			r.Command.Run = nil
-			r.Command.RunE = func(cmd *cobra.Command, args []string) error { return nil }
-			r.Command.SilenceErrors = true
-			r.Command.SilenceUsage = true
-
-			// hack due to https://github.com/spf13/cobra/issues/42
-			root := &cobra.Command{Use: "root"}
-			root.AddCommand(r.Command)
-			root.SetArgs(tt.args)
-
-			// error case
-			err := r.Command.Execute()
-			if tt.err != "" {
-				if !assert.Error(t, err) {
-					t.FailNow()
-				}
-				if !assert.Contains(t, err.Error(), tt.err) {
-					t.FailNow()
-				}
-				// don't check anything else in error case
-				return
-			}
-
-			// non-error case
-			if !assert.NoError(t, err) {
-				t.FailNow()
-			}
-
-			// check if Input was set
-			if !assert.Equal(t, tt.input, r.RunFns.Input) {
-				t.FailNow()
-			}
-
-			// check if Output was set
-			if !assert.Equal(t, tt.output, r.RunFns.Output) {
-				t.FailNow()
-			}
-
-			// check if Path was set
-			if !assert.Equal(t, tt.path, r.RunFns.Path) {
-				t.FailNow()
-			}
-
-			// check if FunctionPaths were set
-			if tt.functionPaths == nil {
-				// make Equal work against flag default
-				tt.functionPaths = []string{}
-			}
-			if !assert.Equal(t, tt.functionPaths, r.RunFns.FunctionPaths) {
-				t.FailNow()
-			}
-
-			// check if Functions were set
-			if tt.expected != "" {
-				if !assert.Len(t, r.RunFns.Functions, 1) {
-					t.FailNow()
-				}
-				actual := strings.TrimSpace(r.RunFns.Functions[0].MustString())
-				if !assert.Equal(t, strings.TrimSpace(tt.expected), actual) {
-					t.FailNow()
-				}
-			}
-
-		})
-	}
-
-}

cmd/config/internal/commands/sink.go

@@ -1,53 +0,0 @@
-// Copyright 2019 The Kubernetes Authors.
-// SPDX-License-Identifier: Apache-2.0
-
-package commands
-
-import (
-	"github.com/spf13/cobra"
-	"sigs.k8s.io/kustomize/cmd/config/internal/generateddocs/commands"
-	"sigs.k8s.io/kustomize/kyaml/kio"
-	"sigs.k8s.io/kustomize/kyaml/kio/kioutil"
-)
-
-// GetSinkRunner returns a command for Sink.
-func GetSinkRunner(name string) *SinkRunner {
-	r := &SinkRunner{}
-	c := &cobra.Command{
-		Use:     "sink DIR",
-		Short:   commands.SinkShort,
-		Long:    commands.SinkLong,
-		Example: commands.SinkExamples,
-		RunE:    r.runE,
-		Args:    cobra.MaximumNArgs(1),
-	}
-	fixDocs(name, c)
-	r.Command = c
-	return r
-}
-
-func SinkCommand(name string) *cobra.Command {
-	return GetSinkRunner(name).Command
-}
-
-// SinkRunner contains the run function
-type SinkRunner struct {
-	Command *cobra.Command
-}
-
-func (r *SinkRunner) runE(c *cobra.Command, args []string) error {
-	var outputs []kio.Writer
-	if len(args) == 1 {
-		outputs = []kio.Writer{&kio.LocalPackageWriter{PackagePath: args[0]}}
-	} else {
-		outputs = []kio.Writer{&kio.ByteWriter{
-			Writer:           c.OutOrStdout(),
-			ClearAnnotations: []string{kioutil.PathAnnotation}},
-		}
-	}
-
-	err := kio.Pipeline{
-		Inputs:  []kio.Reader{&kio.ByteReader{Reader: c.InOrStdin()}},
-		Outputs: outputs}.Execute()
-	return handleError(c, err)
-}

cmd/config/internal/commands/sink_test.go

@@ -1,251 +0,0 @@
-// Copyright 2019 The Kubernetes Authors.
-// SPDX-License-Identifier: Apache-2.0
-
-package commands_test
-
-import (
-	"bytes"
-	"io/ioutil"
-	"os"
-	"path/filepath"
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-	"sigs.k8s.io/kustomize/cmd/config/internal/commands"
-)
-
-func TestSinkCommand(t *testing.T) {
-	d, err := ioutil.TempDir("", "kustomize-source-test")
-	if !assert.NoError(t, err) {
-		t.FailNow()
-	}
-	defer os.RemoveAll(d)
-
-	r := commands.GetSinkRunner("")
-	r.Command.SetIn(bytes.NewBufferString(`apiVersion: config.kubernetes.io/v1alpha1
-kind: ResourceList
-items:
-- kind: Deployment
-  metadata:
-    labels:
-      app: nginx2
-    name: foo
-    annotations:
-      app: nginx2
-      config.kubernetes.io/index: '0'
-      config.kubernetes.io/path: 'f1.yaml'
-  spec:
-    replicas: 1
-- kind: Service
-  metadata:
-    name: foo
-    annotations:
-      app: nginx
-      config.kubernetes.io/index: '1'
-      config.kubernetes.io/path: 'f1.yaml'
-  spec:
-    selector:
-      app: nginx
-- apiVersion: v1
-  kind: Abstraction
-  metadata:
-    name: foo
-    annotations:
-      config.kubernetes.io/function: |
-        container:
-          image: gcr.io/example/reconciler:v1
-      config.kubernetes.io/local-config: "true"
-      config.kubernetes.io/index: '0'
-      config.kubernetes.io/path: 'f2.yaml'
-  spec:
-    replicas: 3
-- apiVersion: apps/v1
-  kind: Deployment
-  metadata:
-    labels:
-      app: nginx
-    name: bar
-    annotations:
-      app: nginx
-      config.kubernetes.io/index: '1'
-      config.kubernetes.io/path: 'f2.yaml'
-  spec:
-    replicas: 3
-`))
-	r.Command.SetArgs([]string{d})
-	if !assert.NoError(t, r.Command.Execute()) {
-		t.FailNow()
-	}
-
-	actual, err := ioutil.ReadFile(filepath.Join(d, "f1.yaml"))
-	if !assert.NoError(t, err) {
-		t.FailNow()
-	}
-	expected := `kind: Deployment
-metadata:
-  labels:
-    app: nginx2
-  name: foo
-  annotations:
-    app: nginx2
-spec:
-  replicas: 1
----
-kind: Service
-metadata:
-  name: foo
-  annotations:
-    app: nginx
-spec:
-  selector:
-    app: nginx
-`
-	if !assert.Equal(t, expected, string(actual)) {
-		t.FailNow()
-	}
-
-	actual, err = ioutil.ReadFile(filepath.Join(d, "f2.yaml"))
-	if !assert.NoError(t, err) {
-		t.FailNow()
-	}
-	expected = `apiVersion: v1
-kind: Abstraction
-metadata:
-  name: foo
-  annotations:
-    config.kubernetes.io/function: |
-      container:
-        image: gcr.io/example/reconciler:v1
-    config.kubernetes.io/local-config: "true"
-spec:
-  replicas: 3
----
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  labels:
-    app: nginx
-  name: bar
-  annotations:
-    app: nginx
-spec:
-  replicas: 3
-`
-	if !assert.Equal(t, expected, string(actual)) {
-		t.FailNow()
-	}
-}
-
-func TestSinkCommand_Stdout(t *testing.T) {
-	d, err := ioutil.TempDir("", "kustomize-source-test")
-	if !assert.NoError(t, err) {
-		t.FailNow()
-	}
-	defer os.RemoveAll(d)
-
-	// fmt the files
-	out := &bytes.Buffer{}
-	r := commands.GetSinkRunner("")
-	r.Command.SetIn(bytes.NewBufferString(`apiVersion: config.kubernetes.io/v1alpha1
-kind: ResourceList
-items:
-- kind: Deployment
-  metadata:
-    labels:
-      app: nginx2
-    name: foo
-    annotations:
-      app: nginx2
-      config.kubernetes.io/index: '0'
-      config.kubernetes.io/path: 'f1.yaml'
-  spec:
-    replicas: 1
-- kind: Service
-  metadata:
-    name: foo
-    annotations:
-      app: nginx
-      config.kubernetes.io/index: '1'
-      config.kubernetes.io/path: 'f1.yaml'
-  spec:
-    selector:
-      app: nginx
-- apiVersion: v1
-  kind: Abstraction
-  metadata:
-    name: foo
-    annotations:
-      config.kubernetes.io/function: |
-        container:
-          image: gcr.io/example/reconciler:v1
-      config.kubernetes.io/local-config: "true"
-      config.kubernetes.io/index: '0'
-      config.kubernetes.io/path: 'f2.yaml'
-  spec:
-    replicas: 3
-- apiVersion: apps/v1
-  kind: Deployment
-  metadata:
-    labels:
-      app: nginx
-    name: bar
-    annotations:
-      app: nginx
-      config.kubernetes.io/index: '1'
-      config.kubernetes.io/path: 'f2.yaml'
-  spec:
-    replicas: 3
-`))
-
-	r.Command.SetOut(out)
-	r.Command.SetArgs([]string{})
-	if !assert.NoError(t, r.Command.Execute()) {
-		t.FailNow()
-	}
-
-	expected := `kind: Deployment
-metadata:
-  labels:
-    app: nginx2
-  name: foo
-  annotations:
-    app: nginx2
-spec:
-  replicas: 1
----
-kind: Service
-metadata:
-  name: foo
-  annotations:
-    app: nginx
-spec:
-  selector:
-    app: nginx
----
-apiVersion: v1
-kind: Abstraction
-metadata:
-  name: foo
-  annotations:
-    config.kubernetes.io/function: |
-      container:
-        image: gcr.io/example/reconciler:v1
-    config.kubernetes.io/local-config: "true"
-spec:
-  replicas: 3
----
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  labels:
-    app: nginx
-  name: bar
-  annotations:
-    app: nginx
-spec:
-  replicas: 3
-`
-	if !assert.Equal(t, expected, out.String()) {
-		t.FailNow()
-	}
-}

cmd/config/internal/commands/source.go

@@ -1,82 +0,0 @@
-// Copyright 2019 The Kubernetes Authors.
-// SPDX-License-Identifier: Apache-2.0
-
-package commands
-
-import (
-	"fmt"
-
-	"github.com/spf13/cobra"
-	"sigs.k8s.io/kustomize/cmd/config/internal/generateddocs/commands"
-	"sigs.k8s.io/kustomize/kyaml/kio"
-	"sigs.k8s.io/kustomize/kyaml/yaml"
-)
-
-// GetSourceRunner returns a command for Source.
-func GetSourceRunner(name string) *SourceRunner {
-	r := &SourceRunner{}
-	c := &cobra.Command{
-		Use:     "source DIR",
-		Short:   commands.SourceShort,
-		Long:    commands.SourceLong,
-		Example: commands.SourceExamples,
-		RunE:    r.runE,
-	}
-	fixDocs(name, c)
-	c.Flags().StringVar(&r.WrapKind, "wrap-kind", kio.ResourceListKind,
-		"output using this format.")
-	c.Flags().StringVar(&r.WrapApiVersion, "wrap-version", kio.ResourceListAPIVersion,
-		"output using this format.")
-	c.Flags().StringVar(&r.FunctionConfig, "function-config", "",
-		"path to function config.")
-	r.Command = c
-	_ = c.MarkFlagFilename("function-config", "yaml", "json", "yml")
-	return r
-}
-
-func SourceCommand(name string) *cobra.Command {
-	return GetSourceRunner(name).Command
-}
-
-// SourceRunner contains the run function
-type SourceRunner struct {
-	WrapKind       string
-	WrapApiVersion string
-	FunctionConfig string
-	Command        *cobra.Command
-}
-
-func (r *SourceRunner) runE(c *cobra.Command, args []string) error {
-	// if there is a function-config specified, emit it
-	var functionConfig *yaml.RNode
-	if r.FunctionConfig != "" {
-		configs, err := kio.LocalPackageReader{PackagePath: r.FunctionConfig}.Read()
-		if err != nil {
-			return err
-		}
-		if len(configs) != 1 {
-			return fmt.Errorf("expected exactly 1 functionConfig, found %d", len(configs))
-		}
-		functionConfig = configs[0]
-	}
-
-	var outputs []kio.Writer
-	outputs = append(outputs, kio.ByteWriter{
-		Writer:                c.OutOrStdout(),
-		KeepReaderAnnotations: true,
-		WrappingKind:          r.WrapKind,
-		WrappingAPIVersion:    r.WrapApiVersion,
-		FunctionConfig:        functionConfig,
-	})
-
-	var inputs []kio.Reader
-	for _, a := range args {
-		inputs = append(inputs, kio.LocalPackageReader{PackagePath: a})
-	}
-	if len(inputs) == 0 {
-		inputs = []kio.Reader{&kio.ByteReader{Reader: c.InOrStdin()}}
-	}
-
-	err := kio.Pipeline{Inputs: inputs, Outputs: outputs}.Execute()
-	return handleError(c, err)
-}

cmd/config/internal/commands/source_test.go

@@ -1,200 +0,0 @@
-// Copyright 2019 The Kubernetes Authors.
-// SPDX-License-Identifier: Apache-2.0
-
-package commands_test
-
-import (
-	"bytes"
-	"io/ioutil"
-	"os"
-	"path/filepath"
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-	"sigs.k8s.io/kustomize/cmd/config/internal/commands"
-)
-
-func TestSourceCommand(t *testing.T) {
-	d, err := ioutil.TempDir("", "kustomize-source-test")
-	if !assert.NoError(t, err) {
-		return
-	}
-	defer os.RemoveAll(d)
-
-	err = ioutil.WriteFile(filepath.Join(d, "f1.yaml"), []byte(`
-kind: Deployment
-metadata:
-  labels:
-    app: nginx2
-  name: foo
-  annotations:
-    app: nginx2
-spec:
-  replicas: 1
----
-kind: Service
-metadata:
-  name: foo
-  annotations:
-    app: nginx
-spec:
-  selector:
-    app: nginx
-`), 0600)
-	if !assert.NoError(t, err) {
-		return
-	}
-	err = ioutil.WriteFile(filepath.Join(d, "f2.yaml"), []byte(`
-apiVersion: v1
-kind: Abstraction
-metadata:
-  name: foo
-  annotations:
-    config.kubernetes.io/function: |
-      container:
-        image: gcr.io/example/reconciler:v1
-    config.kubernetes.io/local-config: "true"
-spec:
-  replicas: 3
----
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  labels:
-    app: nginx
-  name: bar
-  annotations:
-    app: nginx
-spec:
-  replicas: 3
-`), 0600)
-	if !assert.NoError(t, err) {
-		return
-	}
-
-	// fmt the files
-	b := &bytes.Buffer{}
-	r := commands.GetSourceRunner("")
-	r.Command.SetArgs([]string{d})
-	r.Command.SetOut(b)
-	if !assert.NoError(t, r.Command.Execute()) {
-		return
-	}
-
-	if !assert.Equal(t, `apiVersion: config.kubernetes.io/v1alpha1
-kind: ResourceList
-items:
-- kind: Deployment
-  metadata:
-    labels:
-      app: nginx2
-    name: foo
-    annotations:
-      app: nginx2
-      config.kubernetes.io/index: '0'
-      config.kubernetes.io/path: 'f1.yaml'
-  spec:
-    replicas: 1
-- kind: Service
-  metadata:
-    name: foo
-    annotations:
-      app: nginx
-      config.kubernetes.io/index: '1'
-      config.kubernetes.io/path: 'f1.yaml'
-  spec:
-    selector:
-      app: nginx
-- apiVersion: v1
-  kind: Abstraction
-  metadata:
-    name: foo
-    annotations:
-      config.kubernetes.io/function: |
-        container:
-          image: gcr.io/example/reconciler:v1
-      config.kubernetes.io/local-config: "true"
-      config.kubernetes.io/index: '0'
-      config.kubernetes.io/path: 'f2.yaml'
-  spec:
-    replicas: 3
-- apiVersion: apps/v1
-  kind: Deployment
-  metadata:
-    labels:
-      app: nginx
-    name: bar
-    annotations:
-      app: nginx
-      config.kubernetes.io/index: '1'
-      config.kubernetes.io/path: 'f2.yaml'
-  spec:
-    replicas: 3
-`, b.String()) {
-		return
-	}
-}
-
-func TestSourceCommand_Stdin(t *testing.T) {
-	d, err := ioutil.TempDir("", "kustomize-source-test")
-	if !assert.NoError(t, err) {
-		return
-	}
-	defer os.RemoveAll(d)
-
-	in := bytes.NewBufferString(`
-kind: Deployment
-metadata:
-  labels:
-    app: nginx2
-  name: foo
-  annotations:
-    app: nginx2
-spec:
-  replicas: 1
----
-kind: Service
-metadata:
-  name: foo
-  annotations:
-    app: nginx
-spec:
-  selector:
-    app: nginx
-`)
-
-	out := &bytes.Buffer{}
-	r := commands.GetSourceRunner("")
-	r.Command.SetArgs([]string{})
-	r.Command.SetIn(in)
-	r.Command.SetOut(out)
-	if !assert.NoError(t, r.Command.Execute()) {
-		return
-	}
-
-	if !assert.Equal(t, `apiVersion: config.kubernetes.io/v1alpha1
-kind: ResourceList
-items:
-- kind: Deployment
-  metadata:
-    labels:
-      app: nginx2
-    name: foo
-    annotations:
-      app: nginx2
-      config.kubernetes.io/index: '0'
-  spec:
-    replicas: 1
-- kind: Service
-  metadata:
-    name: foo
-    annotations:
-      app: nginx
-      config.kubernetes.io/index: '1'
-  spec:
-    selector:
-      app: nginx
-`, out.String()) {
-		return
-	}
-}

cmd/config/internal/commands/tree_test.go

@@ -104,6 +104,7 @@ metadata:
   namespace: default
   annotations:
     app: nginx2
+    config.kubernetes.io/package: .
     config.kubernetes.io/path: f1.yaml
 spec:
   replicas: 1
@@ -117,6 +118,7 @@ metadata:
   namespace: default
   annotations:
     app: nginx2
+    config.kubernetes.io/package: .
     config.kubernetes.io/path: f1.yaml
 spec:
   replicas: 1
@@ -130,6 +132,7 @@ metadata:
   namespace: default
   annotations:
     app: nginx2
+    config.kubernetes.io/package: .
     config.kubernetes.io/path: f1.yaml
 spec:
   replicas: 1
@@ -143,6 +146,7 @@ metadata:
   namespace: default2
   annotations:
     app: nginx2
+    config.kubernetes.io/package: .
     config.kubernetes.io/path: f1.yaml
 spec:
   replicas: 1
@@ -156,6 +160,7 @@ metadata:
   namespace: default
   annotations:
     app: nginx3
+    config.kubernetes.io/package: .
     config.kubernetes.io/path: f1.yaml
 spec:
   replicas: 1
@@ -166,7 +171,8 @@ metadata:
     app: nginx
   annotations:
     app: nginx
-    config.kubernetes.io/path: bar-package/f2.yaml
+    config.kubernetes.io/package: bar-package
+    config.kubernetes.io/path: f2.yaml
   name: bar
 spec:
   replicas: 3
@@ -177,6 +183,7 @@ metadata:
   namespace: default
   annotations:
     app: nginx
+    config.kubernetes.io/package: .
     config.kubernetes.io/path: f1.yaml
 spec:
   selector:

cmd/config/internal/generateddocs/api/docs.go

@@ -4,6 +4,288 @@
 // Code generated by "mdtogo"; DO NOT EDIT.
 package api
 
+var ConfigFnLong = `# Configuration Functions API Semantics
+
+  Configuration Functions are functions packaged as executables in containers which enable
+  **shift-left practices**.  They configure applications and infrastructure through
+  Kubernetes style Resource Configuration, but run locally pre-commit.
+
+  Configuration functions enable shift-left practices (client-side) through:
+
+  - Pre-commit / delivery validation and linting of configuration
+    - e.g. Fail if any containers don't have PodSecurityPolicy or CPU / Memory limits
+  - Implementation of abstractions as client actuated APIs (e.g. templating)
+    - e.g. Create a client-side *"CRD"* for generating configuration checked into git
+  - Aspect Orient configuration / Injection of cross-cutting configuration
+    - e.g. T-Shirt size containers by annotating Resources with ` + "`" + `small` + "`" + `, ` + "`" + `medium` + "`" + `, ` + "`" + `large` + "`" + `
+      and inject the cpu and memory resources into containers accordingly.
+    - e.g. Inject ` + "`" + `init` + "`" + ` and ` + "`" + `side-car` + "`" + ` containers into Resources based off of Resource
+      Type, annotations, etc.
+
+  Performing these on the client rather than the server enables:
+
+  - Configuration to be reviewed prior to being sent to the API server
+  - Configuration to be validated as part of the CD pipeline
+  - Configuration for Resources to validated holistically rather than individually
+    per-Resource -- e.g. ensure the ` + "`" + `Service.selector` + "`" + ` and ` + "`" + `Deployment.spec.template` + "`" + ` labels
+    match.
+    - MutatingWebHooks are scoped to a single Resource instance at a time.
+  - Low-level tweaks to the output of high-level abstractions -- e.g. add an ` + "`" + `init container` + "`" + `
+    to a client *"CRD"* Resource after it was generated.
+  - Composition and layering of multiple functions together
+    - Compose generation, injection, validation together
+
+  Configuration Functions are implemented as executable programs published in containers which:
+
+  - Accept as input (stdin):
+    - A list of Resource Configuration
+    - A Function Configuration (to configure the function itself)
+  - Emit as output (stdout + exit):
+    - A list of Resource Configuration
+    - An exit code for success / failure
+
+### Function Specification
+
+  - Functions **SHOULD** be published as container images containing a ` + "`" + `CMD` + "`" + ` invoking an executable.
+  - Functions **MUST** accept input on STDIN a ` + "`" + `ResourceList` + "`" + ` containing the Resources and
+    ` + "`" + `functionConfig` + "`" + `.
+  - Functions **MUST** emit output on STDOUT a ` + "`" + `ResourceList` + "`" + ` containing the modified
+    Resources.
+  - Functions **MUST** exit non-0 on failure, and exit 0 on success.
+  - Functions **MAY** emit output on STDERR with error messaging.
+  - Functions performing validation **SHOULD** exit failure and emit error messaging
+    on a validation failure.
+  - Functions generating Resources **SHOULD** retain non-conflicting changes on the
+    generated Resources -- e.g. 1. the function generates a Deployment, but doesn't
+    specify ` + "`" + `cpu` + "`" + `, 2. the user sets the ` + "`" + `cpu` + "`" + ` on the generated Resource, 3. the
+    function should keep the ` + "`" + `cpu` + "`" + ` when regenerating the Resource a second time.
+  - Functions **SHOULD** be usable outside ` + "`" + `kustomize config run` + "`" + ` -- e.g. though pipeline
+    mechanisms such as Tekton.
+
+#### Input Format
+
+  Functions must accept on STDIN:
+
+  ` + "`" + `ResourceList` + "`" + `:
+  - contains ` + "`" + `items` + "`" + ` field, same as ` + "`" + `List.items` + "`" + `
+  - contains ` + "`" + `functionConfig` + "`" + ` field -- a single item with the configuration for the function itself
+
+  Example ` + "`" + `ResourceList` + "`" + ` Input:
+
+    apiVersion: config.kubernetes.io/v1alpha1
+    kind: ResourceList
+    functionConfig:
+      apiVersion: example.com/v1beta1
+      kind: Nginx
+      metadata:
+        name: my-instance
+        annotations:
+          config.kubernetes.io/local-config: "true"
+      spec:
+        replicas: 5
+    items:
+    -  apiVersion: apps/v1
+       kind: Deployment
+       metadata:
+         name: my-instance
+       spec:
+         replicas: 3
+         ...
+    - apiVersion: v1
+      kind: Service
+      metadata:
+        name: my-instance
+      spec:
+        ...
+
+#### Output Format
+
+  Functions must emit on STDOUT:
+
+  ` + "`" + `ResourceList` + "`" + `:
+  - contains ` + "`" + `items` + "`" + ` field, same as ` + "`" + `List.items` + "`" + `
+
+  Example ` + "`" + `ResourceList` + "`" + ` Output:
+
+    apiVersion: config.kubernetes.io/v1alpha1
+    kind: ResourceList
+    items:
+    -  apiVersion: apps/v1
+       kind: Deployment
+       metadata:
+         name: my-instance
+       spec:
+         replicas: 5
+         ...
+    - apiVersion: v1
+      kind: Service
+      metadata:
+        name: my-instance
+      spec:
+        ...
+
+#### Container Environment
+
+  When run by ` + "`" + `kustomize config run` + "`" + `, functions are run in containers with the
+  following environment:
+
+  - Network: ` + "`" + `none` + "`" + `
+  - User: ` + "`" + `nobody` + "`" + `
+  - Security Options: ` + "`" + `no-new-privileges` + "`" + `
+  - Volumes: the volume containing the ` + "`" + `functionConfig` + "`" + ` yaml is mounted under ` + "`" + `/local` + "`" + ` as ` + "`" + `ro` + "`" + `
+
+### Example Function Implementation
+
+  Following is an example for implementing an nginx abstraction using a config
+  function.
+
+#### ` + "`" + `nginx-template.sh` + "`" + `
+
+  ` + "`" + `nginx-template.sh` + "`" + ` is a simple bash script which uses a *heredoc* as a templating solution
+  for generating Resources from the functionConfig input fields.
+
+  The script wraps itself using ` + "`" + `config run wrap -- $0` + "`" + ` which will:
+
+  1. Parse the ` + "`" + `ResourceList.functionConfig` + "`" + ` (provided to the container stdin) into env vars
+  2. Merge the stdout into the original list of Resources
+  3. Defaults filenames for newly generated Resources (if they are not set as annotations)
+     to ` + "`" + `config/NAME_KIND.yaml` + "`" + `
+  4. Format the output
+
+    #!/bin/bash
+    # script must run wrapped by ` + "`" + `kustomize config run wrap` + "`" + `
+    # for parsing input the functionConfig into env vars
+    if [ -z ${WRAPPED} ]; then
+      export WRAPPED=true
+      config run wrap -- $0
+      exit $?
+    fi
+
+    cat <<End-of-message
+    apiVersion: v1
+    kind: Service
+    metadata:
+      name: ${NAME}
+      labels:
+        app: nginx
+        instance: ${NAME}
+    spec:
+      ports:
+      - port: 80
+        targetPort: 80
+        name: http
+      selector:
+        app: nginx
+        instance: ${NAME}
+    ---
+    apiVersion: apps/v1
+    kind: Deployment
+    metadata:
+      name: ${NAME}
+      labels:
+        app: nginx
+        instance: ${NAME}
+    spec:
+      replicas: ${REPLICAS}
+      selector:
+        matchLabels:
+          app: nginx
+          instance: ${NAME}
+      template:
+        metadata:
+          labels:
+            app: nginx
+            instance: ${NAME}
+        spec:
+          containers:
+          - name: nginx
+            image: nginx:1.7.9
+            ports:
+            - containerPort: 80
+    End-of-message
+
+#### ` + "`" + `Dockerfile` + "`" + `
+
+  ` + "`" + `Dockerfile` + "`" + ` installs ` + "`" + `kustomize config` + "`" + ` and copies the script into the container image.
+
+    FROM golang:1.13-stretch
+    RUN go get sigs.k8s.io/kustomize/cmd/config
+    RUN mv /go/bin/config /usr/bin/config
+    COPY nginx-template.sh /usr/bin/nginx-template.sh
+    CMD ["nginx-template.sh]
+
+### Example Function Usage
+
+Following is an example of running the ` + "`" + `kustomize config run` + "`" + ` using the preceding API.
+
+#### ` + "`" + `nginx.yaml` + "`" + ` (Input)
+
+  ` + "`" + `dir/nginx.yaml` + "`" + ` contains a reference to the Function.  The contents of ` + "`" + `nginx.yaml` + "`" + `
+  are passed to the Function through the ` + "`" + `ResourceList.functionConfig` + "`" + ` field.
+
+    apiVersion: example.com/v1beta1
+    kind: Nginx
+    metadata:
+      name: my-instance
+      annotations:
+        config.kubernetes.io/local-config: "true"
+      configFn:
+        container:
+          image: gcr.io/example-functions/nginx-template:v1.0.0
+    spec:
+      replicas: 5
+
+  - ` + "`" + `configFn.container.image` + "`" + `: the image to use for this API
+  - ` + "`" + `annotations[config.kubernetes.io/local-config]` + "`" + `: mark this as not a Resource that should
+    be applied
+
+#### ` + "`" + `kustomize config run dir/` + "`" + ` (Output)
+
+  ` + "`" + `dir/my-instance_deployment.yaml` + "`" + ` contains the Deployment:
+
+    apiVersion: apps/v1
+    kind: Deployment
+    metadata:
+      name: my-instance
+      labels:
+        app: nginx
+        instance: my-instance
+    spec:
+      replicas: 5
+      selector:
+        matchLabels:
+          app: nginx
+          instance: my-instance
+      template:
+        metadata:
+          labels:
+            app: nginx
+            instance: my-instance
+        spec:
+          containers:
+          - name: nginx
+            image: nginx:1.7.9
+            ports:
+            - containerPort: 80
+
+  ` + "`" + `dir/my-instance_service.yaml` + "`" + ` contains the Service:
+
+    apiVersion: v1
+    kind: Service
+    metadata:
+      name: my-instance
+      labels:
+        app: nginx
+        instance: my-instance
+    spec:
+      ports:
+      - port: 80
+        targetPort: 80
+        name: http
+      selector:
+        app: nginx
+        instance: my-instance`
+
 var ConfigIoLong = `# Configuration IO API Semantics
 
 Resource Configuration may be read / written from / to sources such as directories,
@@ -33,7 +315,7 @@ Example:
 ### ` + "`" + `config.kubernetes.io/index` + "`" + `
 
 Records the index of a Resource in file. In a multi-object YAML file, Resources are separated
-by three dashes (` + "`" + `---` + "`" + `), and the index represents the position of the Resource starting from zero.
+by three dashes (` + "`" + `---` + "`" + `), and the index represents the positon of the Resource starting from zero.
 
 This annotation **SHOULD** be set when reading Resources from files.
 It **SHOULD** be unset when writing Resources to files.
@@ -64,355 +346,6 @@ Example:
 	  annotations:
 	    config.kubernetes.io/local-config: "true"`
 
-var FunctionsImplShort = `Following is an example for implementing an nginx abstraction using a configuration`
-var FunctionsImplLong = `# Running Configuration Functions using kustomize CLI
-
-Configuration functions can be implemented using any toolchain and invoked using any
-container workflow orchestrator including Tekton, Cloud Build, or run directly using ` + "`" + `docker run` + "`" + `.
-
-Run ` + "`" + `config help docs-fn-spec` + "`" + ` to see the Configuration Functions Specification.
-
-` + "`" + `kustomize config run` + "`" + ` is an example orchestrator for invoking Configuration Functions. This
-document describes how to implement and invoke an example function.
-
-function.
-
-### ` + "`" + `nginx-template.sh` + "`" + `
-
-` + "`" + `nginx-template.sh` + "`" + ` is a simple bash script which uses a _heredoc_ as a templating solution
-for generating Resources from the functionConfig input fields.
-
-The script wraps itself using ` + "`" + `config run wrap -- $0` + "`" + ` which will:
-
-1. Parse the ` + "`" + `ResourceList.functionConfig` + "`" + ` (provided to the container stdin) into env vars
-2. Merge the stdout into the original list of Resources
-3. Defaults filenames for newly generated Resources (if they are not set as annotations)
-   to ` + "`" + `config/NAME_KIND.yaml` + "`" + `
-4. Format the output
-
-	#!/bin/bash
-	# script must run wrapped by "kustomize config run wrap"
-	# for parsing input the functionConfig into env vars
-	if [ -z ${WRAPPED} ]; then
-	  export WRAPPED=true
-	  config run wrap -- $0
-	  exit $?
-	fi
-	
-	cat <<End-of-message
-	apiVersion: v1
-	kind: Service
-	metadata:
-	  name: ${NAME}
-	  labels:
-	    app: nginx
-	    instance: ${NAME}
-	spec:
-	  ports:
-	  - port: 80
-	    targetPort: 80
-	    name: http
-	  selector:
-	    app: nginx
-	    instance: ${NAME}
-	---
-	apiVersion: apps/v1
-	kind: Deployment
-	metadata:
-	  name: ${NAME}
-	  labels:
-	    app: nginx
-	    instance: ${NAME}
-	spec:
-	  replicas: ${REPLICAS}
-	  selector:
-	    matchLabels:
-	      app: nginx
-	      instance: ${NAME}
-	  template:
-	    metadata:
-	      labels:
-	        app: nginx
-	        instance: ${NAME}
-	    spec:
-	      containers:
-	      - name: nginx
-	        image: nginx:1.7.9
-	        ports:
-	        - containerPort: 80
-	End-of-message
-
-### Dockerfile
-
-` + "`" + `Dockerfile` + "`" + ` installs ` + "`" + `kustomize config` + "`" + ` and copies the script into the container image.
-
-	FROM golang:1.13-stretch
-	RUN go get sigs.k8s.io/kustomize/cmd/config
-	RUN mv /go/bin/config /usr/bin/config
-	COPY nginx-template.sh /usr/bin/nginx-template.sh
-	CMD ["nginx-template.sh]
-
-## Example Function Usage
-
-Following is an example of running the ` + "`" + `kustomize config run` + "`" + ` using the preceding API.
-
-When run by ` + "`" + `kustomize config run` + "`" + `, functions are run in containers with the
-following environment:
-
-- Network: ` + "`" + `none` + "`" + `
-- User: ` + "`" + `nobody` + "`" + `
-- Security Options: ` + "`" + `no-new-privileges` + "`" + `
-- Volumes: the volume containing the ` + "`" + `functionConfig` + "`" + ` yaml is mounted under ` + "`" + `/local` + "`" + ` as ` + "`" + `ro` + "`" + `
-
-### Input
-
-` + "`" + `dir/nginx.yaml` + "`" + ` contains a reference to the Function. The contents of ` + "`" + `nginx.yaml` + "`" + `
-are passed to the Function through the ` + "`" + `ResourceList.functionConfig` + "`" + ` field.
-
-	apiVersion: example.com/v1beta1
-	kind: Nginx
-	metadata:
-	  name: my-instance
-	  annotations:
-	    config.kubernetes.io/local-config: "true"
-	    config.k8s.io/function: |
-	      container:
-	        image: gcr.io/example-functions/nginx-template:v1.0.0
-	spec:
-	  replicas: 5
-
-- ` + "`" + `annotations[config.k8s.io/function].container.image` + "`" + `: the image to use for this API
-- ` + "`" + `annotations[config.kubernetes.io/local-config]` + "`" + `: mark this as not a Resource that should
-  be applied
-
-### Output
-
-The function is invoked using byrunning ` + "`" + `kustomize config run dir/` + "`" + `.
-
-` + "`" + `dir/my-instance_deployment.yaml` + "`" + ` contains the Deployment:
-
-	apiVersion: apps/v1
-	kind: Deployment
-	metadata:
-	  name: my-instance
-	  labels:
-	    app: nginx
-	    instance: my-instance
-	spec:
-	  replicas: 5
-	  selector:
-	    matchLabels:
-	      app: nginx
-	      instance: my-instance
-	  template:
-	    metadata:
-	      labels:
-	        app: nginx
-	        instance: my-instance
-	    spec:
-	      containers:
-	        - name: nginx
-	          image: nginx:1.7.9
-	          ports:
-	            - containerPort: 80
-
-` + "`" + `dir/my-instance_service.yaml` + "`" + ` contains the Service:
-
-	apiVersion: v1
-	kind: Service
-	metadata:
-	  name: my-instance
-	  labels:
-	    app: nginx
-	    instance: my-instance
-	spec:
-	  ports:
-	    - port: 80
-	      targetPort: 80
-	      name: http
-	  selector:
-	    app: nginx
-	    instance: my-instance`
-
-var FunctionsSpecShort = `_Configuration functions_ enable shift-left practices (client-side) through:`
-var FunctionsSpecLong = `# Configuration Functions Specification
-
-This document specifies a standard for client-side functions that operate on
-Kubernetes declarative configurations. This standard enables creating
-small, interoperable, and language-independent executable programs packaged as
-containers that can be chained together as part of a configuration management pipeline.
-The end result of such a pipeline are fully rendered configurations that can then be
-applied to a control plane (e.g. Using ‘kubectl apply’ for Kubernetes control plane).
-As such, although this document references Kubernetes Resource Model and API conventions,
-it is completely decoupled from Kuberentes API machinery and does not depend on any
-in-cluster components.
-
-This document references terms described in [Kubernetes API Conventions][1].
-
-The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", "SHOULD",
-"SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this document are to be
-interpreted as described in [RFC 2119][2].
-
-
-- Pre-commit / delivery validation and linting of configuration
-  - e.g. Fail if any containers don't have PodSecurityPolicy or CPU / Memory limits
-- Implementation of abstractions as client actuated APIs (e.g. templating)
-  - e.g. Create a client-side _"CRD"_ for generating configuration checked into git
-- Aspect Orient configuration / Injection of cross-cutting configuration
-  - e.g. T-Shirt size containers by annotating Resources with ` + "`" + `small` + "`" + `, ` + "`" + `medium` + "`" + `, ` + "`" + `large` + "`" + `
-    and inject the cpu and memory resources into containers accordingly.
-  - e.g. Inject ` + "`" + `init` + "`" + ` and ` + "`" + `side-car` + "`" + ` containers into Resources based off of Resource
-    Type, annotations, etc.
-
-Performing these on the client rather than the server enables:
-
-- Configuration to be reviewed prior to being sent to the API server
-- Configuration to be validated as part of the CI?CD pipeline
-- Configuration for Resources to validated holistically rather than individually
-  per-Resource
-  - e.g. ensure the ` + "`" + `Service.selector` + "`" + ` and ` + "`" + `Deployment.spec.template` + "`" + ` labels
-    match.
-  - e.g. MutatingWebHooks are scoped to a single Resource instance at a time.
-- Low-level tweaks to the output of high-level abstractions
-  - e.g. add an ` + "`" + `init container` + "`" + ` to a client _"CRD"_ Resource after it was generated.
-- Composition and layering of multiple functions together
-  - Compose generation, injection, validation together
-
-## Spec
-
-### Input Type
-
-A function MUST accept as input a single [Kubernetes List type][3].
-The ` + "`" + `items` + "`" + ` field in the input will contain a sequence of [Object types][3].
-A function MAY not support [Simple types][3] and List types.
-
-An example using ` + "`" + `v1/ConfigMapList` + "`" + ` as input:
-
-	apiVersion: v1
-	kind: ConfigMapList
-	items:
-	  - apiVersion: v1
-	    kind: ConfigMap
-	    metadata:
-	      name: config1
-	    data:
-	      p1: v1
-	      p2: v2
-	  - apiVersion: v1
-	    kind: ConfigMap
-	    metadata:
-	      name: config2
-
-An example using ` + "`" + `v1/List` + "`" + ` as input:
-
-	apiVersion: v1
-	kind: List
-	items:
-	  spec:
-	   - apiVersion: foo-corp.com/v1
-	    kind: FulfillmentCenter
-	    metadata:
-	      name: staging
-	    address: "100 Main St."
-	  - apiVersion: rbac.authorization.k8s.io/v1
-	    kind: ClusterRole
-	    metadata:
-	      name: namespace-reader
-	    rules:
-	      - resources:
-	          - namespaces
-	        apiGroups:
-	          - ""
-	        verbs:
-	          - get
-	          - watch
-	          - list
-
-In addition, a function MUST accept as input a List of kind ` + "`" + `ResourceList` + "`" + ` where the
-` + "`" + `functionConfig` + "`" + ` field, if present, will contain the invocation-specific configuration passed to the function
-by the orchestrator.
-Functions MAY consider this field optional so that they can be triggered in an ad-hoc fashion.
-
-An example using ` + "`" + `config.kubernetes.io/v1beta1/ResourceList` + "`" + ` as input:
-
-	apiVersion: config.kubernetes.io/v1beta1
-	kind: ResourceList
-	functionConfig:
-	  apiVersion: foo-corp.com/v1
-	  kind: FulfillmentCenter
-	  metadata:
-	    name: staging
-	    metadata:
-	      annotations:
-	        config.k8s.io/function: |
-	          container:
-	            image: gcr.io/example/foo:v1.0.0
-	  spec:
-	    address: "100 Main St."
-	items:
-	  - apiVersion: rbac.authorization.k8s.io/v1
-	    kind: ClusterRole
-	    metadata:
-	      name: namespace-reader
-	    rules:
-	      - resources:
-	          - namespaces
-	        apiGroups:
-	          - ""
-	        verbs:
-	          - get
-	          - watch
-	          - list
-
-Here ` + "`" + `FulfillmentCenter` + "`" + ` kind with name ` + "`" + `staging` + "`" + ` is passed as the invocation-specific configuration
-to the function.
-
-### Output Type
-
-A function’s output MUST be the same as the input specification above
--- i.e. ` + "`" + `ResourceList` + "`" + ` or ` + "`" + `List` + "`" + `.
-This is necessary to enable chaining two or more functions together in a pipeline.
-The serialization format of the output SHOULD match that of its input on each invocation
--- e.g. if the input was a ` + "`" + `ResourceList` + "`" + `, the output should also be a ` + "`" + `ResourceList` + "`" + `.
-
-### Serialization Format
-
-A function MUST support YAML as a serialization format for the input and output.
-A function MUST use utf8 encoding (as YAML is a superset of JSON, JSON will also be supported
-by any conforming function).
-
-### Operations
-
-A function MAY Create, Update, or Delete any number of items in the ` + "`" + `items` + "`" + ` field and output the
-resultant list.
-
-A function MAY modify annotations with prefix ` + "`" + `config.kubernetes.io` + "`" + `, but must be careful about
-doing so since they’re used for orchestration purposes and will likely impact subsequent functions
-in the pipeline.
-
-A function SHOULD preserve comments when input serialization format is YAML.
-This allows for human authoring of configuration to coexist with changes made by functions.
-
-### Containerization
-
-A function MUST be implemented as a container.
-
-A function container MUST be capable of running as a non-root user if it does not require
-access to host filesystem or makes network calls.
-
-### stdin/stdout/stderr and Exit Codes
-
-A function MUST accept input from stdin and emit output to stdout.
-
-Any error messages MUST be emitted to stderr.
-
-An exit code of zero indicates function execution was successful.
-A non-zero exit code indicates a failure.
-
-[1]: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md
-[2]: https://tools.ietf.org/html/rfc2119
-[3]: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#types-kinds`
-
 var Merge2Long = `# Merge (2-way)
 
   2-way merges fields from a source to a destination, overriding the destination fields

cmd/config/internal/generateddocs/commands/docs.go

@@ -4,20 +4,6 @@
 // Code generated by "mdtogo"; DO NOT EDIT.
 package commands
 
-var AnnotateShort = `[Alpha] Set an annotation on Resources.`
-var AnnotateLong = `
-[Alpha]  Set an annotation on Resources.
-
-  DIR:
-    Path to local directory.
-`
-var AnnotateExamples = `
-    kustomize config annotate my-dir/ --kv foo=bar
-
-    kustomize config annotate my-dir/ --kv foo=bar --kv a=b
-
-    kustomize config annotate my-dir/ --kv foo=bar --kind Deployment --name foo`
-
 var CatShort = `[Alpha] Print Resource Config from a local directory.`
 var CatLong = `
 [Alpha]  Print Resource Config from a local directory.
@@ -156,25 +142,6 @@ var GrepExamples = `
     # look for Resources matching a specific container image
     kustomize config grep "spec.template.spec.containers[name=nginx].image=nginx:1\.7\.9" my-dir/ | kustomize config tree`
 
-var ListSettersShort = `[Alpha] List setters for Resources.`
-var ListSettersLong = `
-List setters for Resources.
-
-  DIR
-
-    A directory containing Resource configuration.
-
-  NAME
-
-    Optional.  The name of the setter to display.
-`
-var ListSettersExamples = `
-  Show setters:
-
-    $ config set DIR/
-        NAME      DESCRIPTION   VALUE     TYPE     COUNT   SETBY  
-    name-prefix   ''            PREFIX    string   2`
-
 var MergeShort = `[Alpha] Merge Resource configuration files`
 var MergeLong = `
 [Alpha] Merge Resource configuration files
@@ -234,8 +201,8 @@ order they appear in the file).
 
 #### Config Functions:
 
-  Config functions are specified as Kubernetes types containing a metadata.annotations.[config.kubernetes.io/function]
-  field specifying an image for the container to run.  This image tells run how to invoke the container.
+  Config functions are specified as Kubernetes types containing a metadata.configFn.container.image
+  field.  This field tells run how to invoke the container.
 
   Example config function:
 
@@ -243,17 +210,17 @@ order they appear in the file).
 	apiVersion: fn.example.com/v1beta1
 	kind: ExampleFunctionKind
 	metadata:
+	  configFn:
+	    container:
+	      # function is invoked as a container running this image
+	      image: gcr.io/example/examplefunction:v1.0.1
 	  annotations:
-	    config.kubernetes.io/function: |
-	      container:
-	        # function is invoked as a container running this image
-	        image: gcr.io/example/examplefunction:v1.0.1
 	    config.kubernetes.io/local-config: "true" # tools should ignore this
 	spec:
 	  configField: configValue
 
   In the preceding example, 'kustomize config run example/' would identify the function by
-  the metadata.annotations.[config.kubernetes.io/function] field.  It would then write all Resources in the directory to
+  the metadata.configFn field.  It would then write all Resources in the directory to
   a container stdin (running the gcr.io/example/examplefunction:v1.0.1 image).  It
   would then write the container stdout back to example/, replacing the directory
   file contents.
@@ -319,19 +286,19 @@ var SetExamples = `
   List setters: Show the possible setters
 
     $ config set DIR/
-        NAME      DESCRIPTION   VALUE     TYPE     COUNT   SETBY  
+    NAME          DESCRIPTION    VALUE     TYPE    COUNT   OWNER
     name-prefix   ''            PREFIX    string   2
 
-  Perform set: set a new value, owner and description
+  Perform substitution: set a new value, owner and description
 
     $ kustomize config set DIR/ name-prefix "test" --description "test environment" --set-by "dev"
-    set 2 values
+    performed 2 substitutions
 
-  List setters: Show the new values
+  Show substitutions: Show the new values
 
-    $ config set DIR/
-        NAME      DESCRIPTION         VALUE     TYPE     COUNT     SETBY 
-    name-prefix   'test environment'   test     string   2          dev
+    $ config set dir
+    NAME       DESCRIPTION       VALUE    TYPE    COUNT   SUBSTITUTED   OWNER
+    prefix   'test environment'   test   string   2       true          dev
 
   New Resource YAML:
 
@@ -346,38 +313,6 @@ var SetExamples = `
         name: test-app2 # {"description":"test environment","type":"string","x-kustomize":{"setBy":"dev","partialFieldSetters":[{"name":"name-prefix","value":"test"}]}}
     ...`
 
-var SinkShort = `[Alpha] Implement a Sink by writing input to a local directory.`
-var SinkLong = `
-[Alpha] Implement a Sink by writing input to a local directory.
-
-    kustomize config sink [DIR]
-
-  DIR:
-    Path to local directory.  If unspecified, sink will write to stdout as if it were a single file.
-
-` + "`" + `sink` + "`" + ` writes its input to a directory
-`
-var SinkExamples = `
-    kustomize config source DIR/ | your-function | kustomize config sink DIR/`
-
-var SourceShort = `[Alpha] Implement a Source by reading a local directory.`
-var SourceLong = `
-[Alpha] Implement a Source by reading a local directory.
-
-    kustomize config source DIR...
-
-  DIR:
-    One or more paths to local directories.  Contents from directories will be concatenated.
-    If no directories are provided, source will read from stdin as if it were a single file.
-
-` + "`" + `source` + "`" + ` emits configuration to act as input to a function
-`
-var SourceExamples = `
-    # emity configuration directory as input source to a function
-    kustomize config source DIR/
-
-    kustomize config source DIR/ | your-function | kustomize config sink DIR/`
-
 var TreeShort = `[Alpha] Display Resource structure from a directory or stdin.`
 var TreeLong = `
 [Alpha] Display Resource structure from a directory or stdin.

cmd/config/main.go

@@ -9,7 +9,6 @@ package main
 import (
 	"os"
 
-	"sigs.k8s.io/kustomize/cmd/config/complete"
 	"sigs.k8s.io/kustomize/cmd/config/configcobra"
 	"sigs.k8s.io/kustomize/kyaml/commandutil"
 )
@@ -17,10 +16,7 @@ import (
 func main() {
 	// enable the config commands
 	os.Setenv(commandutil.EnableAlphaCommmandsEnvName, "true")
-	cmd := configcobra.NewConfigCommand("")
-	complete.Complete(cmd).Complete("config")
-
-	if err := cmd.Execute(); err != nil {
+	if err := configcobra.NewConfigCommand("").Execute(); err != nil {
 		os.Exit(1)
 	}
 }

cmd/kubectl/go.mod

@@ -3,21 +3,12 @@ module sigs.k8s.io/kustomize/cmd/kubectl
 go 1.13
 
 require (
-	github.com/go-errors/errors v1.0.1
 	github.com/spf13/cobra v0.0.5
-	github.com/stretchr/testify v1.4.0
-	k8s.io/api v0.17.0
-	k8s.io/apimachinery v0.17.0
 	k8s.io/cli-runtime v0.17.0
 	k8s.io/client-go v0.17.0
 	k8s.io/component-base v0.17.0 // indirect
 	k8s.io/kubectl v0.0.0-20191219154910-1528d4eea6dd
-	sigs.k8s.io/controller-runtime v0.4.0
-	sigs.k8s.io/kustomize/kstatus v0.0.1
-	sigs.k8s.io/kustomize/kyaml v0.0.2
+	sigs.k8s.io/kustomize/kyaml v0.0.0
 )
 
-replace (
-	sigs.k8s.io/kustomize/kstatus v0.0.1 => ../../kstatus
-	sigs.k8s.io/kustomize/kyaml v0.0.0 => ../../kyaml
-)
+replace sigs.k8s.io/kustomize/kyaml v0.0.0 => ../../kyaml

cmd/kubectl/go.sum

@@ -23,7 +23,6 @@ github.com/MakeNowJust/heredoc v0.0.0-20170808103936-bb23615498cd h1:sjQovDkwrZp
 github.com/MakeNowJust/heredoc v0.0.0-20170808103936-bb23615498cd/go.mod h1:64YHyfSL2R96J44Nlwm39UHepQbyR5q10x7iYa1ks2E=
 github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ=
 github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
-github.com/PuerkitoBio/purell v1.1.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
 github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI=
 github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
 github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
@@ -32,23 +31,15 @@ github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdko
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
-github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
-github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
 github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
 github.com/chai2010/gettext-go v0.0.0-20160711120539-c6fed771bfd5 h1:7aWHqerlJ41y6FOsEUvknqgXnGmJyJSbjhAWq5pO4F8=
 github.com/chai2010/gettext-go v0.0.0-20160711120539-c6fed771bfd5/go.mod h1:/iP1qXHoty45bqomnu2LM+VVyAEdWN+vtSHGlQgyxbw=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
-github.com/coreos/bbolt v1.3.1-coreos.6/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
 github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
-github.com/coreos/etcd v3.3.15+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
 github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=
-github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc=
 github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
-github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
-github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
-github.com/coreos/pkg v0.0.0-20180108230652-97fdf19511ea/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
 github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE=
 github.com/davecgh/go-spew v0.0.0-20151105211317-5215b55f46b2/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -60,10 +51,8 @@ github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZm
 github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
 github.com/docker/docker v0.7.3-0.20190327010347-be7ac8be2ae0 h1:w3NnFcKR5241cfmQU5ZZAsf0xcpId6mWOupTvJlUX2U=
 github.com/docker/docker v0.7.3-0.20190327010347-be7ac8be2ae0/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
-github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
 github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96 h1:cenwrSVm+Z7QLSV/BsnenAOcDXdX4cMv4wP0B/5QbPg=
 github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM=
-github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
 github.com/elazarl/goproxy v0.0.0-20170405201442-c4fc26588b6e h1:p1yVGRW3nmb85p1Sh1ZJSDm4A4iKLS5QNbvUHMgGu/M=
 github.com/elazarl/goproxy v0.0.0-20170405201442-c4fc26588b6e/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
 github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
@@ -71,8 +60,6 @@ github.com/emicklei/go-restful v2.9.5+incompatible h1:spTtZBk5DYEvbxMVutUuTyh1Ao
 github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
 github.com/evanphx/json-patch v4.2.0+incompatible h1:fUDGZCv/7iAN7u0puUVhvKCcsR6vRfwrJatElLBEf0I=
 github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
-github.com/evanphx/json-patch v4.5.0+incompatible h1:ouOWdg56aJriqS0huScTkVXPC5IcNrDCXZ6OoTAWu7M=
-github.com/evanphx/json-patch v4.5.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
 github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d h1:105gxyaGwCFad8crR9dcMQWvV9Hvulu6hwUh4tWPJnM=
 github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d/go.mod h1:ZZMPRZwes7CROmyNKgQzC3XPs6L/G2EJLHddWejkmf4=
 github.com/fatih/camelcase v1.0.0/go.mod h1:yN2Sb0lFhZJUdVvtELVWefmrXpuZESvPmqwoZc+/fpc=
@@ -81,76 +68,39 @@ github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMo
 github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
-github.com/globalsign/mgo v0.0.0-20180905125535-1ca0a4f7cbcb/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q=
-github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q=
-github.com/go-errors/errors v1.0.1 h1:LUHzmkK3GUKUrL/1gfBUxAHzcev3apQlezX/+O7ma6w=
 github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q=
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
-github.com/go-logr/logr v0.1.0 h1:M1Tv3VzNlEHg6uyACnRdtrploV2P7wZqH8BoQMtz0cg=
 github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas=
-github.com/go-logr/zapr v0.1.0 h1:h+WVe9j6HAA01niTJPA/kKH0i7e0rLZBCwauQFcRE54=
-github.com/go-logr/zapr v0.1.0/go.mod h1:tabnROwaDl0UNxkVeFRbY8bwB37GwRv0P8lg6aAiEnk=
-github.com/go-openapi/analysis v0.0.0-20180825180245-b006789cd277/go.mod h1:k70tL6pCuVxPJOHXQ+wIac1FUrvNkHolPie/cLEU6hI=
-github.com/go-openapi/analysis v0.17.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik=
-github.com/go-openapi/analysis v0.18.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik=
-github.com/go-openapi/analysis v0.19.2/go.mod h1:3P1osvZa9jKjb8ed2TPng3f0i/UY9snX6gxi44djMjk=
-github.com/go-openapi/errors v0.17.0/go.mod h1:LcZQpmvG4wyF5j4IhA73wkLFQg+QJXOQHVjmcZxhka0=
-github.com/go-openapi/errors v0.18.0/go.mod h1:LcZQpmvG4wyF5j4IhA73wkLFQg+QJXOQHVjmcZxhka0=
-github.com/go-openapi/errors v0.19.2/go.mod h1:qX0BLWsyaKfvhluLejVpVNwNRdXZhEbTA4kxxpKBC94=
 github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+35s3my2LFTysnkMfxsJBAMHj/DoqoB9knIWoYG/Vk0=
-github.com/go-openapi/jsonpointer v0.17.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M=
-github.com/go-openapi/jsonpointer v0.18.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M=
 github.com/go-openapi/jsonpointer v0.19.2 h1:A9+F4Dc/MCNB5jibxf6rRvOvR/iFgQdyNx9eIhnGqq0=
 github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg=
 github.com/go-openapi/jsonpointer v0.19.3 h1:gihV7YNZK1iK6Tgwwsxo2rJbD1GTbdm72325Bq8FI3w=
 github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
 github.com/go-openapi/jsonreference v0.0.0-20160704190145-13c6e3589ad9/go.mod h1:W3Z9FmVs9qj+KR4zFKmDPGiLdk1D9Rlm7cyMvf57TTg=
-github.com/go-openapi/jsonreference v0.17.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I=
-github.com/go-openapi/jsonreference v0.18.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I=
 github.com/go-openapi/jsonreference v0.19.2 h1:o20suLFB4Ri0tuzpWtyHlh7E7HnkqTNLq6aR6WVNS1w=
 github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc=
 github.com/go-openapi/jsonreference v0.19.3 h1:5cxNfTy0UVC3X8JL5ymxzyoUZmo8iZb+jeTWn7tUa8o=
 github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8=
-github.com/go-openapi/loads v0.17.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU=
-github.com/go-openapi/loads v0.18.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU=
-github.com/go-openapi/loads v0.19.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU=
-github.com/go-openapi/loads v0.19.2/go.mod h1:QAskZPMX5V0C2gvfkGZzJlINuP7Hx/4+ix5jWFxsNPs=
-github.com/go-openapi/runtime v0.0.0-20180920151709-4f900dc2ade9/go.mod h1:6v9a6LTXWQCdL8k1AO3cvqx5OtZY/Y9wKTgaoP6YRfA=
-github.com/go-openapi/runtime v0.19.0/go.mod h1:OwNfisksmmaZse4+gpV3Ne9AyMOlP1lt4sK4FXt0O64=
 github.com/go-openapi/spec v0.0.0-20160808142527-6aced65f8501/go.mod h1:J8+jY1nAiCcj+friV/PDoE1/3eeccG9LYBs0tYvLOWc=
-github.com/go-openapi/spec v0.17.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI=
-github.com/go-openapi/spec v0.18.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI=
-github.com/go-openapi/spec v0.19.2/go.mod h1:sCxk3jxKgioEJikev4fgkNmwS+3kuYdJtcsZsD5zxMY=
 github.com/go-openapi/spec v0.19.3 h1:0XRyw8kguri6Yw4SxhsQA/atC88yqrk0+G4YhI2wabc=
 github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo=
-github.com/go-openapi/strfmt v0.17.0/go.mod h1:P82hnJI0CXkErkXi8IKjPbNBM6lV6+5pLP5l494TcyU=
-github.com/go-openapi/strfmt v0.18.0/go.mod h1:P82hnJI0CXkErkXi8IKjPbNBM6lV6+5pLP5l494TcyU=
-github.com/go-openapi/strfmt v0.19.0/go.mod h1:+uW+93UVvGGq2qGaZxdDeJqSAqBqBdl+ZPMF/cC8nDY=
+github.com/go-openapi/spec v0.19.5/go.mod h1:Hm2Jr4jv8G1ciIAo+frC/Ft+rR2kQDh8JHKHb3gWUSk=
 github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dpr1UfpPtxFw+EFuQ41HhCWZfha5jSVRG7C7I=
-github.com/go-openapi/swag v0.17.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg=
-github.com/go-openapi/swag v0.18.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg=
 github.com/go-openapi/swag v0.19.2 h1:jvO6bCMBEilGwMfHhrd61zIID4oIFdwb76V17SM88dE=
 github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
 github.com/go-openapi/swag v0.19.5 h1:lTz6Ys4CmqqCQmZPBlbQENR1/GucA2bzYTE12Pw4tFY=
 github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
-github.com/go-openapi/validate v0.18.0/go.mod h1:Uh4HdOzKt19xGIGm1qHf/ofbX1YQ4Y+MYsct2VUrAJ4=
-github.com/go-openapi/validate v0.19.2/go.mod h1:1tRCw7m3jtI8eNWEEliiAqUIcBztB2KDnRCRMUi7GTA=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d h1:3PaI8p3seN09VjbTYC/QWlUZdZ1qS1zGjy7LH2Wt07I=
 github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
-github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls=
-github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903 h1:LbsanbbD6LieFkXbj9YNNBupiGHJgFeLpO0j0Fza1h8=
 github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
-github.com/golang/groupcache v0.0.0-20180513044358-24b0969c4cb7 h1:u4bArs140e9+AfE52mFHOXVFnOSBJBRlzTHrOPLOIhE=
-github.com/golang/groupcache v0.0.0-20180513044358-24b0969c4cb7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/protobuf v0.0.0-20161109072736-4bd1920723d7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/golang/protobuf v1.0.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg=
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
@@ -166,30 +116,20 @@ github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
 github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
-github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg=
-github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/gofuzz v0.0.0-20161122191042-44d81051d367/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI=
 github.com/google/gofuzz v1.0.0 h1:A8PeW59pxE9IoFRqBp37U+mSNaQoZ46F1f0f863XSXw=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
 github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
-github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY=
 github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
 github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d h1:7XGaL1e6bYS1yIonGp9761ExpPPV1ui0SAC59Yube9k=
 github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY=
-github.com/googleapis/gnostic v0.3.1 h1:WeAefnSUHlBb0iJKwxFDZdbfGwkd7xRNuV+IpXMJhYk=
-github.com/googleapis/gnostic v0.3.1/go.mod h1:on+2t9HRStVgn95RSsFWFz+6Q0Snyqv1awfrALZdbtU=
 github.com/gophercloud/gophercloud v0.1.0 h1:P/nh25+rzXouhytV2pUHBb65fnds26Ghl8/391+sT5o=
 github.com/gophercloud/gophercloud v0.1.0/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEoIEcSTewFxm1c5g8=
-github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
-github.com/gregjones/httpcache v0.0.0-20170728041850-787624de3eb7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
 github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 h1:pdN6V1QBWetyv/0+wjACpqVH+eVULgEjkurDLq3goeM=
 github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
-github.com/grpc-ecosystem/go-grpc-middleware v0.0.0-20190222133341-cfaf5686ec79/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
-github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
-github.com/grpc-ecosystem/grpc-gateway v1.3.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU=
 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
@@ -198,15 +138,12 @@ github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
 github.com/imdario/mergo v0.3.5 h1:JboBksRwiiAJWvIYJVo46AfV+IAIKZpfrSzVKj42R4Q=
 github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
-github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28=
-github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
 github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/jonboulle/clockwork v0.1.0 h1:VKV+ZcuP6l3yW9doeqz6ziZGgcynBVQO+obU0+0hcPo=
 github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
 github.com/json-iterator/go v0.0.0-20180612202835-f2b4162afba3/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
-github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/json-iterator/go v1.1.8 h1:QiWkFLKq0T7mpzwOTu6BzNDbfTE8OLrYhVKYMLF46Ok=
 github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
@@ -227,8 +164,6 @@ github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de/go.mod h1:zAbeS9
 github.com/lithammer/dedent v1.1.0/go.mod h1:jrXYCQtgg0nJiN+StA2KgR7w6CiQNv9Fd/Z9BP0jIOc=
 github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
 github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
-github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
-github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63 h1:nTT4s92Dgz2HlrB2NaMgvlfqHH39OgMhA7z3PK7PGD4=
 github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
@@ -250,18 +185,13 @@ github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8m
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw=
 github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
-github.com/onsi/ginkgo v1.4.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
-github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.10.1 h1:q/mM8GF/n0shIN8SaAZ0V+jnLPzen6WIVZdiwrRlMlo=
 github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
-github.com/onsi/gomega v1.3.0/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
-github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME=
 github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
-github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
 github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI=
 github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
@@ -272,16 +202,12 @@ github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE
 github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA=
 github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
-github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM=
 github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
-github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
 github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
 github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
-github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
 github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446/go.mod h1:uYEyJGbgTkfkS4+E/PavXkNJcbFIpEtjt2B0KDQ5+9M=
 github.com/russross/blackfriday v1.5.2 h1:HyvC0ARfnZBqnXwABFeSZHpKvJHJJfPz81GNueLj0oo=
@@ -289,7 +215,6 @@ github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4=
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
-github.com/soheilhy/cmux v0.1.3/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
 github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
 github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
 github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
@@ -310,35 +235,19 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
-github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
-github.com/xiang90/probing v0.0.0-20160813154853-07dd2e8dfe18/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
 github.com/xlab/handysort v0.0.0-20150421192137-fb3537ed64a1/go.mod h1:QcJo0QPSfTONNIgpN5RA8prR7fF8nkF6cTWTcNerRO8=
 github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg=
 github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
-go.uber.org/atomic v0.0.0-20181018215023-8dc6146f7569/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
-go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
-go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU=
-go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
-go.uber.org/multierr v0.0.0-20180122172545-ddea229ff1df/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
-go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI=
-go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
-go.uber.org/zap v0.0.0-20180814183419-67bc79d13d15/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
-go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
-go.uber.org/zap v1.10.0 h1:ORx85nbTijNz8ljznvCMR1ZBIPKFn3jQrag10X2AsuM=
-go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20190320223903-b7391e95e576/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8 h1:1wopBVtVdWnn03fZelqdXTqk7U7zPQCb+T4rbU9ZEoU=
 golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586 h1:7KByu05hhLed2MO29w7p1XfZvZ13m8mub3shuVftRs0=
 golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7 h1:0hQKqeLdqlt5iIwVOBErRisrHJAN57yOiPRQItI20fU=
-golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190312203227-4b39c73a6495/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -346,24 +255,18 @@ golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMx
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
 golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
-golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
 golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20180112015858-5ccada7d0a7b/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20181005035420-146acd28ed58/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190320064053-1272bf9dcd53/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20190812203447-cdfb69ac37fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20191004110552-13f9640d40b9 h1:rjwSpXsdiK0dV8/Naq3kAw9ymfAeJIyd0upUIElB+lI=
 golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
@@ -376,9 +279,7 @@ golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20180117170059-2c42eef0765b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -387,18 +288,14 @@ golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5h
 golang.org/x/sys v0.0.0-20190209173611-3b5209105503/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190321052220-f7bb7a8bee54/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f h1:25KHgbfyiSm6vwQLbM3zZIe1v9p/3ea4Rz+nnM5K/i4=
 golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456 h1:ng0gs1AKnRRuEMZoTLLlbOd+C17zUDepwGQBb/n+JVg=
 golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190911201528-7ad0cfa0b7b5 h1:SW/0nsKCUaozCUtZTakri5laocGx/5bkDSSLrFUsa5s=
-golang.org/x/sys v0.0.0-20190911201528-7ad0cfa0b7b5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/text v0.3.1-0.20171227012246-e19ae1496984/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
@@ -406,24 +303,17 @@ golang.org/x/time v0.0.0-20181108054448-85acf8d2951c h1:fqgJT0MGcGpPgpWU7VRdRjuA
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.0.0-20191024005414-555d28b269f0 h1:/5xXl8Y5W96D+TtHSlonuFqGHIWVuyCkGJLwGh9JJFs=
-golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20181011042414-1f849cf54d09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190206041539-40960b6deb8e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
-golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
 golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
-golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190920225731-5eefd052ad72/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7 h1:9zdDQZ7Thm29KFXgAX/+yaf3eVbP7djjWp/dXAppNCc=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-gomodules.xyz/jsonpatch/v2 v2.0.1/go.mod h1:IhYNNY4jnS53ZnfE4PAmpKtDpTCj1JFXc+3mwe7XcUU=
 gonum.org/v1/gonum v0.0.0-20190331200053-3d26580ed485/go.mod h1:2ltnJ7xHfj0zHS40VVPYEAAMTa3ZGguvHGBSJeRWqE0=
 gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw=
 gonum.org/v1/netlib v0.0.0-20190331212654-76723241ea4e/go.mod h1:kS+toOQn6AQKjmKJ7gzohV1XkqsFehRA2FbsbkopSuQ=
@@ -435,25 +325,17 @@ google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
 google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
 google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
-google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
-google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
-gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
-gopkg.in/inf.v0 v0.9.0/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
 gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
 gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
-gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k=
-gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
-gopkg.in/yaml.v2 v2.0.0/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I=
@@ -463,30 +345,21 @@ gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo=
 gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-k8s.io/api v0.0.0-20190918155943-95b840bb6a1f/go.mod h1:uWuOHnjmNrtQomJrvEBg0c0HRNyQ+8KTEERVsK0PW48=
 k8s.io/api v0.0.0-20191214185829-ca1d04f8b0d3/go.mod h1:itOjKREfmUTvcjantxOsyYU5mbFsU7qUnyUuRfF5+5M=
 k8s.io/api v0.17.0 h1:H9d/lw+VkZKEVIUc8F3wgiQ+FUXTTr21M87jXLU7yqM=
 k8s.io/api v0.17.0/go.mod h1:npsyOePkeP0CPwyGfXDHxvypiYMJxBWAMpQxCaJ4ZxI=
-k8s.io/apiextensions-apiserver v0.0.0-20190918161926-8f644eb6e783 h1:V6ndwCPoao1yZ52agqOKaUAl7DYWVGiXjV7ePA2i610=
-k8s.io/apiextensions-apiserver v0.0.0-20190918161926-8f644eb6e783/go.mod h1:xvae1SZB3E17UpV59AWc271W/Ph25N+bjPyR63X6tPY=
-k8s.io/apimachinery v0.0.0-20190913080033-27d36303b655/go.mod h1:nL6pwRT8NgfF8TT68DBI8uEePRt89cSvoXUVqbkWHq4=
 k8s.io/apimachinery v0.0.0-20191214185652-442f8fb2f03a/go.mod h1:Ng1IY8TS7sC44KJxT/WUR6qFRfWwahYYYpNXyYRKOCY=
 k8s.io/apimachinery v0.0.0-20191216025728-0ee8b4573e3a/go.mod h1:Ng1IY8TS7sC44KJxT/WUR6qFRfWwahYYYpNXyYRKOCY=
 k8s.io/apimachinery v0.17.0 h1:xRBnuie9rXcPxUkDizUsGvPf1cnlZCFu210op7J7LJo=
 k8s.io/apimachinery v0.17.0/go.mod h1:b9qmWdKlLuU9EBh+06BtLcSf/Mu89rWL33naRxs1uZg=
-k8s.io/apiserver v0.0.0-20190918160949-bfa5e2e684ad/go.mod h1:XPCXEwhjaFN29a8NldXA901ElnKeKLrLtREO9ZhFyhg=
 k8s.io/cli-runtime v0.0.0-20191214191754-e6dc6d5c8724/go.mod h1:wzlq80lvjgHW9if6MlE4OIGC86MDKsy5jtl9nxz/IYY=
 k8s.io/cli-runtime v0.17.0 h1:XEuStbJBHCQlEKFyTQmceDKEWOSYHZkcYWKp3SsQ9Hk=
 k8s.io/cli-runtime v0.17.0/go.mod h1:1E5iQpMODZq2lMWLUJELwRu2MLWIzwvMgDBpn3Y81Qo=
-k8s.io/client-go v0.0.0-20190918160344-1fbdaa4c8d90/go.mod h1:J69/JveO6XESwVgG53q3Uz5OSfgsv4uxpScmmyYOOlk=
 k8s.io/client-go v0.0.0-20191214190045-a32a6f7a3052/go.mod h1:tAaoc/sYuIL0+njJefSAmE28CIcxyaFV4kbIujBlY2s=
 k8s.io/client-go v0.0.0-20191219150334-0b8da7416048/go.mod h1:ZEe8ZASDUAuqVGJ+UN0ka0PfaR+b6a6E1PGsSNZRui8=
 k8s.io/client-go v0.17.0 h1:8QOGvUGdqDMFrm9sD6IUFl256BcffynGoe80sxgTEDg=
 k8s.io/client-go v0.17.0/go.mod h1:TYgR6EUHs6k45hb6KWjVD6jFZvJV4gHDikv/It0xz+k=
-k8s.io/code-generator v0.0.0-20190912054826-cd179ad6a269/go.mod h1:V5BD6M4CyaN5m+VthcclXWsVcT1Hu+glwa1bi3MIsyE=
 k8s.io/code-generator v0.0.0-20191214185510-0b9b3c99f9f2/go.mod h1:BjGKcoq1MRUmcssvHiSxodCco1T6nVIt4YeCT5CMSao=
-k8s.io/component-base v0.0.0-20190918160511-547f6c5d7090/go.mod h1:933PBGtQFJky3TEwYx4aEPZ4IxqhWh3R6DCmzqIn1hA=
 k8s.io/component-base v0.0.0-20191214190519-d868452632e2/go.mod h1:wupxkh1T/oUDqyTtcIjiEfpbmIHGm8By/vqpSKC6z8c=
 k8s.io/component-base v0.17.0 h1:BnDFcmBDq+RPpxXjmuYnZXb59XNN9CaFrX8ba9+3xrA=
 k8s.io/component-base v0.17.0/go.mod h1:rKuRAokNMY2nn2A6LP/MiwpoaMRHpfRnrPaUJJj1Yoc=
@@ -494,16 +367,13 @@ k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8
 k8s.io/gengo v0.0.0-20190822140433-26a664648505/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
 k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk=
 k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk=
-k8s.io/klog v0.4.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I=
 k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8=
 k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I=
-k8s.io/kube-openapi v0.0.0-20190816220812-743ec37842bf/go.mod h1:1TqjTSzOxsLGIKfj0lK8EeCP7K1iUG65v09OM0/WG5E=
 k8s.io/kube-openapi v0.0.0-20191107075043-30be4d16710a h1:UcxjrRMyNx/i/y8G7kPvLyy7rfbeuf1PYyBf973pgyU=
 k8s.io/kube-openapi v0.0.0-20191107075043-30be4d16710a/go.mod h1:1TqjTSzOxsLGIKfj0lK8EeCP7K1iUG65v09OM0/WG5E=
 k8s.io/kubectl v0.0.0-20191219154910-1528d4eea6dd h1:nZX5+wEqTu/EBIYjrZlFOA63z4+Zcy96lDkCZPU9a9c=
 k8s.io/kubectl v0.0.0-20191219154910-1528d4eea6dd/go.mod h1:9ehGcuUGjXVZh0qbYSB0vvofQw2JQe6c6cO0k4wu/Oo=
 k8s.io/metrics v0.0.0-20191214191643-6b1944c9f765/go.mod h1:5V7rewilItwK0cz4nomU0b3XCcees2Ka5EBYWS1HBeM=
-k8s.io/utils v0.0.0-20190801114015-581e00157fb1/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew=
 k8s.io/utils v0.0.0-20191114184206-e782cd3c129f h1:GiPwtSzdP43eI1hpPCbROQCCIgCuiMMNF8YUVLF3vJo=
 k8s.io/utils v0.0.0-20191114184206-e782cd3c129f/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew=
 modernc.org/cc v1.0.0/go.mod h1:1Sk4//wdnYJiUIxnW8ddKpaOJCF37yAdqYnkxUpaYxw=
@@ -511,16 +381,9 @@ modernc.org/golex v1.0.0/go.mod h1:b/QX9oBD/LhixY6NDh+IdGv17hgB+51fET1i2kPSmvk=
 modernc.org/mathutil v1.0.0/go.mod h1:wU0vUrJsVWBZ4P6e7xtFJEhFSNsfRLJ8H458uRjg03k=
 modernc.org/strutil v1.0.0/go.mod h1:lstksw84oURvj9y3tn8lGvRxyRC1S2+g5uuIzNfIOBs=
 modernc.org/xc v1.0.0/go.mod h1:mRNCo0bvLjGhHO9WsyuKVU4q0ceiDDDoEeWDJHrNx8I=
-sigs.k8s.io/controller-runtime v0.4.0 h1:wATM6/m+3w8lj8FXNaO6Fs/rq/vqoOjO1Q116Z9NPsg=
-sigs.k8s.io/controller-runtime v0.4.0/go.mod h1:ApC79lpY3PHW9xj/w9pj+lYkLgwAAUZwfXkME1Lajns=
 sigs.k8s.io/kustomize v2.0.3+incompatible h1:JUufWFNlI44MdtnjUqVnvh29rR37PQFzPbLXqhyOyX0=
 sigs.k8s.io/kustomize v2.0.3+incompatible/go.mod h1:MkjgH3RdOWrievjo6c9T245dYlB5QeXV4WCbnt/PEpU=
-sigs.k8s.io/kustomize/kyaml v0.0.2 h1:Rl/wMrnpZzZjsVeFIIOAb92Kz/UfLrTUEXjiHW6oS0o=
-sigs.k8s.io/kustomize/kyaml v0.0.2/go.mod h1:rywm/rcR5LmCBghz9956tE45OdUPChFoXVVs+WmhMTI=
 sigs.k8s.io/structured-merge-diff v0.0.0-20190525122527-15d366b2352e/go.mod h1:wWxsB5ozmmv/SG7nM11ayaAW51xMvak/t1r0CSlcokI=
-sigs.k8s.io/structured-merge-diff v0.0.0-20190817042607-6149e4549fca/go.mod h1:IIgPezJWb76P0hotTxzDbWsMYB8APh18qZnxkomBpxA=
-sigs.k8s.io/testing_frameworks v0.1.2 h1:vK0+tvjF0BZ/RYFeZ1E6BYBwHJJXhjuZ3TdsEKH+UQM=
-sigs.k8s.io/testing_frameworks v0.1.2/go.mod h1:ToQrwSC3s8Xf/lADdZp3Mktcql9CG0UAmdJG9th5i0w=
 sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs=
 sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=
 vbom.ml/util v0.0.0-20160121211510-db5cfe13f5cc/go.mod h1:so/NYdZXCz+E3ZpW0uAoCj6uzU2+8OWDFv/HxUSs7kI=

cmd/kubectl/kubectlcobra/applier.go

@@ -1,205 +0,0 @@
-// Copyright 2019 The Kubernetes Authors.
-// SPDX-License-Identifier: Apache-2.0
-
-package kubectlcobra
-
-import (
-	"context"
-	"time"
-
-	"github.com/go-errors/errors"
-	"github.com/spf13/cobra"
-	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
-	"k8s.io/apimachinery/pkg/runtime"
-	"k8s.io/cli-runtime/pkg/genericclioptions"
-	"k8s.io/cli-runtime/pkg/resource"
-	"k8s.io/kubectl/pkg/cmd/apply"
-	"k8s.io/kubectl/pkg/cmd/util"
-	"k8s.io/kubectl/pkg/scheme"
-	"sigs.k8s.io/controller-runtime/pkg/client"
-	"sigs.k8s.io/kustomize/kstatus/wait"
-)
-
-// newApplier returns a new Applier. It will set up the applyOptions and
-// statusOptions which are responsible for capturing any command line flags.
-// It currently requires IOStreams, but this is a legacy from when
-// the ApplyOptions were responsible for printing progress. This is now
-// handled by a separate printer with the KubectlPrinterAdapter bridging
-// between the two.
-func newApplier(factory util.Factory, ioStreams genericclioptions.IOStreams) *Applier {
-	return &Applier{
-		applyOptions:  apply.NewApplyOptions(ioStreams),
-		statusOptions: NewStatusOptions(),
-		factory:       factory,
-		ioStreams:     ioStreams,
-	}
-}
-
-// resolver defines the interface the applier needs to observe status for resources.
-type resolver interface {
-	WaitForStatusOfObjects(ctx context.Context, objects []wait.KubernetesObject) <-chan wait.Event
-}
-
-// Applier performs the step of applying a set of resources into a cluster,
-// conditionally waits for all of them to be fully reconciled and finally
-// performs prune to clean up any resources that has been deleted.
-type Applier struct {
-	factory   util.Factory
-	ioStreams genericclioptions.IOStreams
-
-	applyOptions  *apply.ApplyOptions
-	statusOptions *StatusOptions
-	resolver      resolver
-}
-
-// Initialize sets up the Applier for actually doing an apply against
-// a cluster. This involves validating command line inputs and configuring
-// clients for communicating with the cluster.
-func (a *Applier) Initialize(cmd *cobra.Command) error {
-	a.applyOptions.PreProcessorFn = PrependGroupingObject(a.applyOptions)
-	err := a.applyOptions.Complete(a.factory, cmd)
-	if err != nil {
-		return errors.WrapPrefix(err, "error setting up ApplyOptions", 1)
-	}
-	// Default PostProcessor is configured in "Complete" function,
-	// so the prune must happen after "Complete".
-	a.applyOptions.PostProcessorFn = prune(a.factory, a.applyOptions)
-
-	resolver, err := a.newResolver(a.statusOptions.period)
-	if err != nil {
-		return errors.WrapPrefix(err, "error creating resolver", 1)
-	}
-	a.resolver = resolver
-	return nil
-}
-
-// SetFlags configures the command line flags needed for apply and
-// status. This is a temporary solution as we should separate the configuration
-// of cobra flags from the Applier.
-func (a *Applier) SetFlags(cmd *cobra.Command) {
-	a.applyOptions.DeleteFlags.AddFlags(cmd)
-	a.applyOptions.RecordFlags.AddFlags(cmd)
-	a.applyOptions.PrintFlags.AddFlags(cmd)
-	a.statusOptions.AddFlags(cmd)
-	a.applyOptions.Overwrite = true
-}
-
-// newResolver sets up a new Resolver for computing status. The configuration
-// needed for the resolver is taken from the Factory.
-func (a *Applier) newResolver(pollInterval time.Duration) (*wait.Resolver, error) {
-	config, err := a.factory.ToRESTConfig()
-	if err != nil {
-		return nil, errors.WrapPrefix(err, "error getting RESTConfig", 1)
-	}
-
-	mapper, err := a.factory.ToRESTMapper()
-	if err != nil {
-		return nil, errors.WrapPrefix(err, "error getting RESTMapper", 1)
-	}
-
-	c, err := client.New(config, client.Options{Scheme: scheme.Scheme, Mapper: mapper})
-	if err != nil {
-		return nil, errors.WrapPrefix(err, "error creating client", 1)
-	}
-
-	return wait.NewResolver(c, mapper, pollInterval), nil
-}
-
-// Run performs the Apply step. This happens asynchronously with updates
-// on progress and any errors are reported back on the event channel.
-// Cancelling the operation or setting timeout on how long to wait
-// for it complete can be done with the passed in context.
-// Note: There sn't currently any way to interrupt the operation
-// before all the given resources have been applied to the cluster. Any
-// cancellation or timeout will only affect how long we wait for the
-// resources to become current.
-func (a *Applier) Run(ctx context.Context) <-chan Event {
-	ch := make(chan Event)
-
-	go func() {
-		defer close(ch)
-		adapter := &KubectlPrinterAdapter{
-			ch: ch,
-		}
-		// The adapter is used to intercept what is meant to be printing
-		// in the ApplyOptions, and instead turn those into events.
-		a.applyOptions.ToPrinter = adapter.toPrinterFunc()
-		// This provides us with a slice of all the objects that will be
-		// applied to the cluster.
-		infos, _ := a.applyOptions.GetObjects()
-		err := a.applyOptions.Run()
-		if err != nil {
-			// If we see an error here we just report it on the channel and then
-			// give up. Eventually we might be able to determine which errors
-			// are fatal and which might allow us to continue.
-			ch <- Event{
-				EventType: ErrorEventType,
-				ErrorEvent: ErrorEvent{
-					Err: errors.WrapPrefix(err, "error applying resources", 1),
-				},
-			}
-			return
-		}
-
-		if a.statusOptions.wait {
-			statusChannel := a.resolver.WaitForStatusOfObjects(ctx, infosToObjects(infos))
-			// As long as the statusChannel remains open, we take every statusEvent,
-			// wrap it in an Event and send it on the channel.
-			for statusEvent := range statusChannel {
-				ch <- Event{
-					EventType:   StatusEventType,
-					StatusEvent: statusEvent,
-				}
-			}
-		}
-
-	}()
-	return ch
-}
-
-func infosToObjects(infos []*resource.Info) []wait.KubernetesObject {
-	var objects []wait.KubernetesObject
-	for _, info := range infos {
-		u := info.Object.(*unstructured.Unstructured)
-		objects = append(objects, u)
-	}
-	return objects
-}
-
-// EventType determines the type of events that are available.
-type EventType string
-
-const (
-	ErrorEventType  EventType = "error"
-	ApplyEventType  EventType = "apply"
-	StatusEventType EventType = "status"
-)
-
-// Event is the type of the objects that will be returned through
-// the channel that is returned from a call to Run. It contains
-// information about progress and errors encountered during
-// the process of doing apply, waiting for status and doing a prune.
-type Event struct {
-	// EventType is the type of event.
-	EventType EventType
-
-	// ErrorEvent contains information about any errors encountered.
-	ErrorEvent ErrorEvent
-
-	// ApplyEvent contains information about progress pertaining to
-	// applying a resource to the cluster.
-	ApplyEvent ApplyEvent
-
-	// StatusEvents contains information about the status of one of
-	// the applied resources.
-	StatusEvent wait.Event
-}
-
-type ErrorEvent struct {
-	Err error
-}
-
-type ApplyEvent struct {
-	Operation string
-	Object    runtime.Object
-}

cmd/kubectl/kubectlcobra/applier_test.go

@@ -1,48 +0,0 @@
-// Copyright 2019 The Kubernetes Authors.
-// SPDX-License-Identifier: Apache-2.0
-
-package kubectlcobra
-
-import (
-	"context"
-	"github.com/stretchr/testify/assert"
-	"k8s.io/cli-runtime/pkg/genericclioptions"
-	cmdtesting "k8s.io/kubectl/pkg/cmd/testing"
-	"testing"
-)
-
-// The applier is currently hard to test, as the dependencies on the ApplyOptions and
-// the resolver are hard to stub out. As we work to better separate the different
-// responsibilities of the apply functionality, we should also make it easier to test.
-// This provides some basic tests for now.
-
-func TestApplierWithUnknownFile(t *testing.T) {
-	tf := cmdtesting.NewTestFactory()
-	defer tf.Cleanup()
-	iostreams, _, _, _ := genericclioptions.NewTestIOStreams()
-	cmd := NewCmdApply("base", tf, iostreams)
-
-	applier := newApplier(tf, iostreams)
-	filenames := []string{"file.yaml"}
-	applier.applyOptions.DeleteFlags.FileNameFlags.Filenames = &filenames
-
-	err := applier.Initialize(cmd)
-	assert.NoError(t, err)
-
-	ch := applier.Run(context.TODO())
-
-	var events []Event
-	for msg := range ch {
-		events = append(events, msg)
-	}
-
-	if !assert.Equal(t, 1, len(events)) {
-		return
-	}
-
-	event := events[0]
-	if !assert.Equal(t, ErrorEventType, event.EventType) {
-		return
-	}
-	assert.Contains(t, event.ErrorEvent.Err.Error(), "does not exist")
-}

cmd/kubectl/kubectlcobra/basic_printer.go

@@ -1,62 +0,0 @@
-// Copyright 2019 The Kubernetes Authors.
-// SPDX-License-Identifier: Apache-2.0
-
-package kubectlcobra
-
-import (
-	"fmt"
-	"k8s.io/apimachinery/pkg/runtime/schema"
-	"strings"
-
-	"k8s.io/apimachinery/pkg/api/meta"
-	"k8s.io/cli-runtime/pkg/genericclioptions"
-	cmdutil "k8s.io/kubectl/pkg/cmd/util"
-	"sigs.k8s.io/kustomize/kstatus/wait"
-)
-
-// BasicPrinter is a simple implementation that just prints the events
-// from the channel in the default format for kubectl.
-// We need to support different printers for different output formats.
-type BasicPrinter struct {
-	ioStreams genericclioptions.IOStreams
-}
-
-// Print outputs the events from the provided channel in a simple
-// format on StdOut. As we support other printer implementations
-// this should probably be an interface.
-// This function will block until the channel is closed.
-func (b *BasicPrinter) Print(ch <-chan Event) {
-	for event := range ch {
-		switch event.EventType {
-		case ErrorEventType:
-			cmdutil.CheckErr(event.ErrorEvent.Err)
-		case ApplyEventType:
-			obj := event.ApplyEvent.Object
-			gvk := obj.GetObjectKind().GroupVersionKind()
-			name := "<unknown>"
-			if acc, err := meta.Accessor(obj); err == nil {
-				if n := acc.GetName(); len(n) > 0 {
-					name = n
-				}
-			}
-			fmt.Fprintf(b.ioStreams.Out, "%s %s\n", resourceIdToString(gvk.GroupKind(), name), event.ApplyEvent.Operation)
-		case StatusEventType:
-			statusEvent := event.StatusEvent
-			switch statusEvent.Type {
-			case wait.ResourceUpdate:
-				id := statusEvent.EventResource.ResourceIdentifier
-				gk := id.GroupKind
-				fmt.Fprintf(b.ioStreams.Out, "%s is %s: %s\n", resourceIdToString(gk, id.Name), statusEvent.EventResource.Status.String(), statusEvent.EventResource.Message)
-			case wait.Completed:
-				fmt.Fprint(b.ioStreams.Out, "all resources has reached the Current status\n")
-			case wait.Aborted:
-				fmt.Fprintf(b.ioStreams.Out, "resources failed to the reached Current status\n")
-			}
-		}
-	}
-}
-
-// resourceIdToString returns the string representation of a GroupKind and a resource name.
-func resourceIdToString(gk schema.GroupKind, name string) string {
-	return fmt.Sprintf("%s/%s", strings.ToLower(gk.String()), name)
-}

cmd/kubectl/kubectlcobra/commands.go

@@ -5,9 +5,7 @@
 package kubectlcobra
 
 import (
-	"context"
 	"flag"
-	"fmt"
 	"os"
 	"strings"
 
@@ -19,6 +17,9 @@ import (
 	cmdutil "k8s.io/kubectl/pkg/cmd/util"
 	"k8s.io/kubectl/pkg/util/i18n"
 	"sigs.k8s.io/kustomize/kyaml/commandutil"
+
+	// initialize auth
+	_ "k8s.io/client-go/plugin/pkg/client/auth"
 )
 
 // GetCommand returns a command from kubectl to install
@@ -80,10 +81,7 @@ func updateHelp(names []string, c *cobra.Command) {
 
 // NewCmdApply creates the `apply` command
 func NewCmdApply(baseName string, f util.Factory, ioStreams genericclioptions.IOStreams) *cobra.Command {
-	applier := newApplier(f, ioStreams)
-	printer := &BasicPrinter{
-		ioStreams: ioStreams,
-	}
+	o := apply.NewApplyOptions(ioStreams)
 
 	cmd := &cobra.Command{
 		Use:                   "apply (-f FILENAME | -k DIRECTORY)",
@@ -95,67 +93,25 @@ func NewCmdApply(baseName string, f util.Factory, ioStreams genericclioptions.IO
 		Run: func(cmd *cobra.Command, args []string) {
 			if len(args) > 0 {
 				// check is kustomize, if so update
-				applier.applyOptions.DeleteFlags.FileNameFlags.Kustomize = &args[0]
+				o.DeleteFlags.FileNameFlags.Kustomize = &args[0]
 			}
 
-			cmdutil.CheckErr(applier.Initialize(cmd))
-
-			// Create a context with the provided timout from the cobra parameter.
-			ctx, cancel := context.WithTimeout(context.Background(), applier.statusOptions.timeout)
-			defer cancel()
-			// Run the applier. It will return a channel where we can receive updates
-			// to keep track of progress and any issues.
-			ch := applier.Run(ctx)
-
-			// The printer will print updates from the channel. It will block
-			// until the channel is closed.
-			printer.Print(ch)
+			cmdutil.CheckErr(o.Complete(f, cmd))
+			cmdutil.CheckErr(o.Run())
 		},
 	}
 
-	applier.SetFlags(cmd)
+	// bind flag structs
+	o.DeleteFlags.AddFlags(cmd)
+	o.RecordFlags.AddFlags(cmd)
+	o.PrintFlags.AddFlags(cmd)
+
+	o.Overwrite = true
 
 	cmdutil.AddValidateFlags(cmd)
-	cmd.Flags().BoolVar(&applier.applyOptions.ServerDryRun, "server-dry-run", applier.applyOptions.ServerDryRun, "If true, request will be sent to server with dry-run flag, which means the modifications won't be persisted. This is an alpha feature and flag.")
+	cmd.Flags().BoolVar(&o.ServerDryRun, "server-dry-run", o.ServerDryRun, "If true, request will be sent to server with dry-run flag, which means the modifications won't be persisted. This is an alpha feature and flag.")
 	cmd.Flags().Bool("dry-run", false, "If true, only print the object that would be sent, without sending it. Warning: --dry-run cannot accurately output the result of merging the local manifest and the server-side data. Use --server-dry-run to get the merged result instead.")
 	cmdutil.AddServerSideApplyFlags(cmd)
 
 	return cmd
 }
-
-// PrependGroupingObject orders the objects to apply so the "grouping"
-// object stores the inventory, and it is first to be applied.
-func PrependGroupingObject(o *apply.ApplyOptions) func() error {
-	return func() error {
-		if o == nil {
-			return fmt.Errorf("ApplyOptions are nil")
-		}
-		infos, err := o.GetObjects()
-		if err != nil {
-			return err
-		}
-		_, exists := findGroupingObject(infos)
-		if exists {
-			if err := addInventoryToGroupingObj(infos); err != nil {
-				return err
-			}
-			if !sortGroupingObject(infos) {
-				return err
-			}
-		}
-		return nil
-	}
-}
-
-// Prune deletes previously applied objects that have been
-// omitted in the current apply. The previously applied objects
-// are reached through ConfigMap grouping objects.
-func prune(f util.Factory, o *apply.ApplyOptions) func() error {
-	return func() error {
-		po, err := NewPruneOptions(f, o)
-		if err != nil {
-			return err
-		}
-		return po.Prune()
-	}
-}

cmd/kubectl/kubectlcobra/commands_test.go

@@ -1,58 +0,0 @@
-// Copyright 2019 The Kubernetes Authors.
-// SPDX-License-Identifier: Apache-2.0
-
-// package kubectlcobra contains cobra commands from kubectl
-package kubectlcobra
-
-import (
-	"testing"
-
-	"k8s.io/cli-runtime/pkg/resource"
-	"k8s.io/kubectl/pkg/cmd/apply"
-)
-
-func TestPrependGroupingObject(t *testing.T) {
-	tests := []struct {
-		infos []*resource.Info
-	}{
-		{
-			infos: []*resource.Info{copyGroupingInfo()},
-		},
-		{
-			infos: []*resource.Info{pod1Info, pod3Info, copyGroupingInfo()},
-		},
-		{
-			infos: []*resource.Info{pod1Info, pod2Info, copyGroupingInfo(), pod3Info},
-		},
-	}
-
-	for _, test := range tests {
-		applyOptions := createApplyOptions(test.infos)
-		f := PrependGroupingObject(applyOptions)
-		err := f()
-		if err != nil {
-			t.Errorf("Error running pre-processor callback: %s", err)
-		}
-		infos, _ := applyOptions.GetObjects()
-		if len(test.infos) != len(infos) {
-			t.Fatalf("Wrong number of objects after prepending grouping object")
-		}
-		groupingInfo := infos[0]
-		if !isGroupingObject(groupingInfo.Object) {
-			t.Fatalf("First object is not the grouping object")
-		}
-		inventory, _ := retrieveInventoryFromGroupingObj(infos)
-		if len(inventory) != (len(infos) - 1) {
-			t.Errorf("Wrong number of inventory items stored in grouping object")
-		}
-	}
-
-}
-
-// createApplyOptions is a helper function to assemble the ApplyOptions
-// with the passed objects (infos).
-func createApplyOptions(infos []*resource.Info) *apply.ApplyOptions {
-	applyOptions := &apply.ApplyOptions{}
-	applyOptions.SetObjects(infos)
-	return applyOptions
-}

cmd/kubectl/kubectlcobra/grouping.go

@@ -1,268 +0,0 @@
-// Copyright 2020 The Kubernetes Authors.
-// SPDX-License-Identifier: Apache-2.0
-
-// package kubectlcobra contains cobra commands from kubectl
-package kubectlcobra
-
-import (
-	"fmt"
-	"hash/fnv"
-	"sort"
-	"strconv"
-	"strings"
-
-	"k8s.io/apimachinery/pkg/api/meta"
-	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
-	"k8s.io/apimachinery/pkg/runtime"
-	"k8s.io/cli-runtime/pkg/resource"
-)
-
-const (
-	GroupingLabel = "kustomize.config.k8s.io/inventory-id"
-	GroupingHash  = "kustomize.config.k8s.io/inventory-hash"
-)
-
-// retrieveGroupingLabel returns the string value of the GroupingLabel
-// for the passed object. Returns error if the passed object is nil or
-// is not a grouping object.
-func retrieveGroupingLabel(obj runtime.Object) (string, error) {
-	var groupingLabel string
-	if obj == nil {
-		return "", fmt.Errorf("Grouping object is nil.\n")
-	}
-	accessor, err := meta.Accessor(obj)
-	if err != nil {
-		return "", err
-	}
-	labels := accessor.GetLabels()
-	groupingLabel, exists := labels[GroupingLabel]
-	if !exists {
-		return "", fmt.Errorf("Grouping label does not exist for grouping object: %s\n", GroupingLabel)
-	}
-	return strings.TrimSpace(groupingLabel), nil
-}
-
-// isGroupingObject returns true if the passed object has the
-// grouping label.
-// TODO(seans3): Check type is ConfigMap.
-func isGroupingObject(obj runtime.Object) bool {
-	if obj == nil {
-		return false
-	}
-	groupingLabel, err := retrieveGroupingLabel(obj)
-	if err == nil && len(groupingLabel) > 0 {
-		return true
-	}
-	return false
-}
-
-// findGroupingObject returns the "Grouping" object (ConfigMap with
-// grouping label) if it exists, and a boolean describing if it was found.
-func findGroupingObject(infos []*resource.Info) (*resource.Info, bool) {
-	for _, info := range infos {
-		if info != nil && isGroupingObject(info.Object) {
-			return info, true
-		}
-	}
-	return nil, false
-}
-
-// sortGroupingObject reorders the infos slice to place the grouping
-// object in the first position. Returns true if grouping object found,
-// false otherwise.
-func sortGroupingObject(infos []*resource.Info) bool {
-	for i, info := range infos {
-		if info != nil && isGroupingObject(info.Object) {
-			// If the grouping object is not already in the first position,
-			// swap the grouping object with the first object.
-			if i > 0 {
-				infos[0], infos[i] = infos[i], infos[0]
-			}
-			return true
-		}
-	}
-	return false
-}
-
-// Adds the inventory of all objects (passed as infos) to the
-// grouping object. Returns an error if a grouping object does not
-// exist, or we are unable to successfully add the inventory to
-// the grouping object; nil otherwise. Each object is in
-// unstructured.Unstructured format.
-func addInventoryToGroupingObj(infos []*resource.Info) error {
-
-	// Iterate through the objects (infos), creating an Inventory struct
-	// as metadata for each object, or if it's the grouping object, store it.
-	var groupingInfo *resource.Info
-	var groupingObj *unstructured.Unstructured
-	inventoryMap := map[string]string{}
-	for _, info := range infos {
-		obj := info.Object
-		if isGroupingObject(obj) {
-			// If we have more than one grouping object--error.
-			if groupingObj != nil {
-				return fmt.Errorf("Error--applying more than one grouping object.")
-			}
-			var ok bool
-			groupingObj, ok = obj.(*unstructured.Unstructured)
-			if !ok {
-				return fmt.Errorf("Grouping object is not an Unstructured: %#v", groupingObj)
-			}
-			groupingInfo = info
-		} else {
-			if obj == nil {
-				return fmt.Errorf("Creating inventory; object is nil")
-			}
-			gk := obj.GetObjectKind().GroupVersionKind().GroupKind()
-			inventory, err := createInventory(info.Namespace, info.Name, gk)
-			if err != nil {
-				return err
-			}
-			inventoryMap[inventory.String()] = ""
-		}
-	}
-
-	// If we've found the grouping object, store the object metadata inventory
-	// in the grouping config map.
-	if groupingObj == nil {
-		return fmt.Errorf("Grouping object not found")
-	}
-
-	if len(inventoryMap) > 0 {
-		// Adds the inventory map to the ConfigMap "data" section.
-		err := unstructured.SetNestedStringMap(groupingObj.UnstructuredContent(),
-			inventoryMap, "data")
-		if err != nil {
-			return err
-		}
-		// Adds the hash of the inventory strings as an annotation to the
-		// grouping object. Inventory strings must be sorted to make hash
-		// deterministic.
-		inventoryList := mapKeysToSlice(inventoryMap)
-		sort.Strings(inventoryList)
-		invHash, err := calcInventoryHash(inventoryList)
-		if err != nil {
-			return err
-		}
-		// Add the hash as a suffix to the grouping object's name.
-		invHashStr := strconv.FormatUint(uint64(invHash), 16)
-		if err := addSuffixToName(groupingInfo, invHashStr); err != nil {
-			return err
-		}
-		annotations := groupingObj.GetAnnotations()
-		if annotations == nil {
-			annotations = map[string]string{}
-		}
-		annotations[GroupingHash] = invHashStr
-		groupingObj.SetAnnotations(annotations)
-	}
-	return nil
-}
-
-// retrieveInventoryFromGroupingObj returns a slice of pointers to the
-// inventory metadata. This function finds the grouping object, then
-// parses the stored resource metadata into Inventory structs. Returns
-// an error if there is a problem parsing the data into Inventory
-// structs, or if the grouping object is not in Unstructured format; nil
-// otherwise. If a grouping object does not exist, or it does not have a
-// "data" map, then returns an empty slice and no error.
-func retrieveInventoryFromGroupingObj(infos []*resource.Info) ([]*Inventory, error) {
-	inventory := []*Inventory{}
-	groupingInfo, exists := findGroupingObject(infos)
-	if exists {
-		groupingObj, ok := groupingInfo.Object.(*unstructured.Unstructured)
-		if !ok {
-			err := fmt.Errorf("Grouping object is not an Unstructured: %#v", groupingObj)
-			return inventory, err
-		}
-		invMap, exists, err := unstructured.NestedStringMap(groupingObj.Object, "data")
-		if err != nil {
-			err := fmt.Errorf("Error retrieving inventory from grouping object.")
-			return inventory, err
-		}
-		if exists {
-			for invStr := range invMap {
-				inv, err := parseInventory(invStr)
-				if err != nil {
-					return inventory, err
-				}
-				inventory = append(inventory, inv)
-			}
-		}
-	}
-	return inventory, nil
-}
-
-// calcInventoryHash returns an unsigned int32 representing the hash
-// of the inventory strings. If there is an error writing bytes to
-// the hash, then the error is returned; nil is returned otherwise.
-// Used to quickly identify the set of resources in the grouping object.
-func calcInventoryHash(inv []string) (uint32, error) {
-	h := fnv.New32a()
-	for _, is := range inv {
-		_, err := h.Write([]byte(is))
-		if err != nil {
-			return uint32(0), err
-		}
-	}
-	return h.Sum32(), nil
-}
-
-// retrieveInventoryHash takes a grouping object (encapsulated by
-// a resource.Info), and returns the string representing the hash
-// of the grouping inventory; returns empty string if the grouping
-// object is not in Unstructured format, or if the hash annotation
-// does not exist.
-func retrieveInventoryHash(groupingInfo *resource.Info) string {
-	var invHash = ""
-	groupingObj, ok := groupingInfo.Object.(*unstructured.Unstructured)
-	if ok {
-		annotations := groupingObj.GetAnnotations()
-		if annotations != nil {
-			invHash = annotations[GroupingHash]
-		}
-	}
-	return invHash
-}
-
-// mapKeysToSlice returns the map keys as a slice of strings.
-func mapKeysToSlice(m map[string]string) []string {
-	s := make([]string, len(m))
-	i := 0
-	for k := range m {
-		s[i] = k
-		i++
-	}
-	return s
-}
-
-// addSuffixToName adds the passed suffix (usually a hash) as a suffix
-// to the name of the passed object stored in the Info struct. Returns
-// an error if the object is not "*unstructured.Unstructured" or if the
-// name stored in the object differs from the name in the Info struct.
-func addSuffixToName(info *resource.Info, suffix string) error {
-
-	if info == nil {
-		return fmt.Errorf("Nil resource.Info")
-	}
-	suffix = strings.TrimSpace(suffix)
-	if len(suffix) == 0 {
-		return fmt.Errorf("Passed empty suffix")
-	}
-
-	accessor, _ := meta.Accessor(info.Object)
-	name := accessor.GetName()
-	if name != info.Name {
-		return fmt.Errorf("Grouping object (%s) and resource.Info (%s) have different names\n", name, info.Name)
-	}
-	// Error if name alread has suffix.
-	suffix = "-" + suffix
-	if strings.HasSuffix(name, suffix) {
-		return fmt.Errorf("Name already has suffix: %s\n", name)
-	}
-	name += suffix
-	accessor.SetName(name)
-	info.Name = name
-
-	return nil
-}

cmd/kubectl/kubectlcobra/grouping_test.go

@@ -1,600 +0,0 @@
-// Copyright 2020 The Kubernetes Authors.
-// SPDX-License-Identifier: Apache-2.0
-
-// package kubectlcobra contains cobra commands from kubectl
-package kubectlcobra
-
-import (
-	"fmt"
-	"testing"
-
-	corev1 "k8s.io/api/core/v1"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
-	"k8s.io/apimachinery/pkg/runtime"
-	"k8s.io/apimachinery/pkg/runtime/schema"
-	"k8s.io/cli-runtime/pkg/resource"
-)
-
-var testNamespace = "test-grouping-namespace"
-var groupingObjName = "test-grouping-obj"
-var pod1Name = "pod-1"
-var pod2Name = "pod-2"
-var pod3Name = "pod-3"
-
-var testGroupingLabel = "test-app-label"
-
-var groupingObj = unstructured.Unstructured{
-	Object: map[string]interface{}{
-		"apiVersion": "v1",
-		"kind":       "ConfigMap",
-		"metadata": map[string]interface{}{
-			"name":      groupingObjName,
-			"namespace": testNamespace,
-			"labels": map[string]interface{}{
-				GroupingLabel: testGroupingLabel,
-			},
-		},
-	},
-}
-
-var pod1 = unstructured.Unstructured{
-	Object: map[string]interface{}{
-		"apiVersion": "v1",
-		"kind":       "Pod",
-		"metadata": map[string]interface{}{
-			"name":      pod1Name,
-			"namespace": testNamespace,
-		},
-	},
-}
-
-var pod1Info = &resource.Info{
-	Namespace: testNamespace,
-	Name:      pod1Name,
-	Object:    &pod1,
-}
-
-var pod2 = unstructured.Unstructured{
-	Object: map[string]interface{}{
-		"apiVersion": "v1",
-		"kind":       "Pod",
-		"metadata": map[string]interface{}{
-			"name":      pod2Name,
-			"namespace": testNamespace,
-		},
-	},
-}
-
-var pod2Info = &resource.Info{
-	Namespace: testNamespace,
-	Name:      pod2Name,
-	Object:    &pod2,
-}
-
-var pod3 = unstructured.Unstructured{
-	Object: map[string]interface{}{
-		"apiVersion": "v1",
-		"kind":       "Pod",
-		"metadata": map[string]interface{}{
-			"name":      pod3Name,
-			"namespace": testNamespace,
-		},
-	},
-}
-
-var pod3Info = &resource.Info{
-	Namespace: testNamespace,
-	Name:      pod3Name,
-	Object:    &pod3,
-}
-
-var nonUnstructuredGroupingObj = &corev1.ConfigMap{
-	ObjectMeta: metav1.ObjectMeta{
-		Namespace: testNamespace,
-		Name:      groupingObjName,
-		Labels: map[string]string{
-			GroupingLabel: "true",
-		},
-	},
-}
-
-var nonUnstructuredGroupingInfo = &resource.Info{
-	Namespace: testNamespace,
-	Name:      groupingObjName,
-	Object:    nonUnstructuredGroupingObj,
-}
-
-var nilInfo = &resource.Info{
-	Namespace: testNamespace,
-	Name:      groupingObjName,
-	Object:    nil,
-}
-
-var groupingObjLabelWithSpace = unstructured.Unstructured{
-	Object: map[string]interface{}{
-		"apiVersion": "v1",
-		"kind":       "ConfigMap",
-		"metadata": map[string]interface{}{
-			"name":      groupingObjName,
-			"namespace": testNamespace,
-			"labels": map[string]interface{}{
-				GroupingLabel: "\tgrouping-label ",
-			},
-		},
-	},
-}
-
-func TestRetrieveGroupingLabel(t *testing.T) {
-	tests := []struct {
-		obj           runtime.Object
-		groupingLabel string
-		isError       bool
-	}{
-		// Nil grouping object throws error.
-		{
-			obj:           nil,
-			groupingLabel: "",
-			isError:       true,
-		},
-		// Pod is not a grouping object.
-		{
-			obj:           &pod2,
-			groupingLabel: "",
-			isError:       true,
-		},
-		// Retrieves label without preceding/trailing whitespace.
-		{
-			obj:           &groupingObjLabelWithSpace,
-			groupingLabel: "grouping-label",
-			isError:       false,
-		},
-		{
-			obj:           &groupingObj,
-			groupingLabel: testGroupingLabel,
-			isError:       false,
-		},
-	}
-
-	for _, test := range tests {
-		actual, err := retrieveGroupingLabel(test.obj)
-		if test.isError && err == nil {
-			t.Errorf("Did not receive expected error.\n")
-		}
-		if !test.isError {
-			if err != nil {
-				t.Fatalf("Received unexpected error: %s\n", err)
-			}
-			if test.groupingLabel != actual {
-				t.Errorf("Expected grouping label (%s), got (%s)\n", test.groupingLabel, actual)
-			}
-		}
-	}
-}
-
-func TestIsGroupingObject(t *testing.T) {
-	tests := []struct {
-		obj        runtime.Object
-		isGrouping bool
-	}{
-		{
-			obj:        nil,
-			isGrouping: false,
-		},
-		{
-			obj:        &groupingObj,
-			isGrouping: true,
-		},
-		{
-			obj:        &pod2,
-			isGrouping: false,
-		},
-	}
-
-	for _, test := range tests {
-		grouping := isGroupingObject(test.obj)
-		if test.isGrouping && !grouping {
-			t.Errorf("Grouping object not identified: %#v", test.obj)
-		}
-		if !test.isGrouping && grouping {
-			t.Errorf("Non-grouping object identifed as grouping obj: %#v", test.obj)
-		}
-	}
-}
-
-func TestFindGroupingObject(t *testing.T) {
-	tests := []struct {
-		infos  []*resource.Info
-		exists bool
-		name   string
-	}{
-		{
-			infos:  []*resource.Info{},
-			exists: false,
-			name:   "",
-		},
-		{
-			infos:  []*resource.Info{nil},
-			exists: false,
-			name:   "",
-		},
-		{
-			infos:  []*resource.Info{copyGroupingInfo()},
-			exists: true,
-			name:   groupingObjName,
-		},
-		{
-			infos:  []*resource.Info{pod1Info},
-			exists: false,
-			name:   "",
-		},
-		{
-			infos:  []*resource.Info{pod1Info, pod2Info, pod3Info},
-			exists: false,
-			name:   "",
-		},
-		{
-			infos:  []*resource.Info{pod1Info, pod2Info, copyGroupingInfo(), pod3Info},
-			exists: true,
-			name:   groupingObjName,
-		},
-	}
-
-	for _, test := range tests {
-		groupingObj, found := findGroupingObject(test.infos)
-		if test.exists && !found {
-			t.Errorf("Should have found grouping object")
-		}
-		if !test.exists && found {
-			t.Errorf("Grouping object found, but it does not exist: %#v", groupingObj)
-		}
-		if test.exists && found && test.name != groupingObj.Name {
-			t.Errorf("Grouping object name does not match: %s/%s", test.name, groupingObj.Name)
-		}
-	}
-}
-
-func TestSortGroupingObject(t *testing.T) {
-	tests := []struct {
-		infos  []*resource.Info
-		sorted bool
-	}{
-		{
-			infos:  []*resource.Info{},
-			sorted: false,
-		},
-		{
-			infos:  []*resource.Info{copyGroupingInfo()},
-			sorted: true,
-		},
-		{
-			infos:  []*resource.Info{pod1Info},
-			sorted: false,
-		},
-		{
-			infos:  []*resource.Info{pod1Info, pod2Info},
-			sorted: false,
-		},
-		{
-			infos:  []*resource.Info{copyGroupingInfo(), pod1Info},
-			sorted: true,
-		},
-		{
-			infos:  []*resource.Info{pod1Info, copyGroupingInfo()},
-			sorted: true,
-		},
-		{
-			infos:  []*resource.Info{pod1Info, pod2Info, copyGroupingInfo(), pod3Info},
-			sorted: true,
-		},
-		{
-			infos:  []*resource.Info{pod1Info, pod2Info, pod3Info, copyGroupingInfo()},
-			sorted: true,
-		},
-		{
-			infos:  []*resource.Info{copyGroupingInfo(), pod1Info, pod2Info, pod3Info},
-			sorted: true,
-		},
-	}
-
-	for _, test := range tests {
-		wasSorted := sortGroupingObject(test.infos)
-		if wasSorted && !test.sorted {
-			t.Errorf("Grouping object was sorted, but it shouldn't have been")
-		}
-		if !wasSorted && test.sorted {
-			t.Errorf("Grouping object was NOT sorted, but it should have been")
-		}
-		if wasSorted {
-			first := test.infos[0]
-			if !isGroupingObject(first.Object) {
-				t.Errorf("Grouping object was not sorted into first position")
-			}
-		}
-	}
-}
-
-func TestAddRetrieveInventoryToFromGroupingObject(t *testing.T) {
-	tests := []struct {
-		infos    []*resource.Info
-		expected []*Inventory
-		isError  bool
-	}{
-		// No grouping object is an error.
-		{
-			infos:   []*resource.Info{},
-			isError: true,
-		},
-		// No grouping object is an error.
-		{
-			infos:   []*resource.Info{pod1Info, pod2Info},
-			isError: true,
-		},
-		// Grouping object without other objects is OK.
-		{
-			infos:   []*resource.Info{copyGroupingInfo(), nilInfo},
-			isError: true,
-		},
-		{
-			infos:   []*resource.Info{nonUnstructuredGroupingInfo},
-			isError: true,
-		},
-		{
-			infos:    []*resource.Info{copyGroupingInfo()},
-			expected: []*Inventory{},
-			isError:  false,
-		},
-		// More than one grouping object is an error.
-		{
-			infos:    []*resource.Info{copyGroupingInfo(), copyGroupingInfo()},
-			expected: []*Inventory{},
-			isError:  true,
-		},
-		// More than one grouping object is an error.
-		{
-			infos:    []*resource.Info{copyGroupingInfo(), pod1Info, copyGroupingInfo()},
-			expected: []*Inventory{},
-			isError:  true,
-		},
-		// Basic test case: one grouping object, one pod.
-		{
-			infos: []*resource.Info{copyGroupingInfo(), pod1Info},
-			expected: []*Inventory{
-				&Inventory{
-					Namespace: testNamespace,
-					Name:      pod1Name,
-					GroupKind: schema.GroupKind{
-						Group: "",
-						Kind:  "Pod",
-					},
-				},
-			},
-			isError: false,
-		},
-		{
-			infos: []*resource.Info{pod1Info, copyGroupingInfo()},
-			expected: []*Inventory{
-				&Inventory{
-					Namespace: testNamespace,
-					Name:      pod1Name,
-					GroupKind: schema.GroupKind{
-						Group: "",
-						Kind:  "Pod",
-					},
-				},
-			},
-			isError: false,
-		},
-		{
-			infos: []*resource.Info{pod1Info, pod2Info, copyGroupingInfo(), pod3Info},
-			expected: []*Inventory{
-				&Inventory{
-					Namespace: testNamespace,
-					Name:      pod1Name,
-					GroupKind: schema.GroupKind{
-						Group: "",
-						Kind:  "Pod",
-					},
-				},
-				&Inventory{
-					Namespace: testNamespace,
-					Name:      pod2Name,
-					GroupKind: schema.GroupKind{
-						Group: "",
-						Kind:  "Pod",
-					},
-				},
-				&Inventory{
-					Namespace: testNamespace,
-					Name:      pod3Name,
-					GroupKind: schema.GroupKind{
-						Group: "",
-						Kind:  "Pod",
-					},
-				},
-			},
-			isError: false,
-		},
-		{
-			infos: []*resource.Info{pod1Info, pod2Info, pod3Info, copyGroupingInfo()},
-			expected: []*Inventory{
-				&Inventory{
-					Namespace: testNamespace,
-					Name:      pod1Name,
-					GroupKind: schema.GroupKind{
-						Group: "",
-						Kind:  "Pod",
-					},
-				},
-				&Inventory{
-					Namespace: testNamespace,
-					Name:      pod2Name,
-					GroupKind: schema.GroupKind{
-						Group: "",
-						Kind:  "Pod",
-					},
-				},
-				&Inventory{
-					Namespace: testNamespace,
-					Name:      pod3Name,
-					GroupKind: schema.GroupKind{
-						Group: "",
-						Kind:  "Pod",
-					},
-				},
-			},
-			isError: false,
-		},
-		{
-			infos: []*resource.Info{copyGroupingInfo(), pod1Info, pod2Info, pod3Info},
-			expected: []*Inventory{
-				&Inventory{
-					Namespace: testNamespace,
-					Name:      pod1Name,
-					GroupKind: schema.GroupKind{
-						Group: "",
-						Kind:  "Pod",
-					},
-				},
-				&Inventory{
-					Namespace: testNamespace,
-					Name:      pod2Name,
-					GroupKind: schema.GroupKind{
-						Group: "",
-						Kind:  "Pod",
-					},
-				},
-				&Inventory{
-					Namespace: testNamespace,
-					Name:      pod3Name,
-					GroupKind: schema.GroupKind{
-						Group: "",
-						Kind:  "Pod",
-					},
-				},
-			},
-			isError: false,
-		},
-	}
-
-	for _, test := range tests {
-		err := addInventoryToGroupingObj(test.infos)
-		if test.isError && err == nil {
-			t.Errorf("Should have produced an error, but returned none.")
-		}
-		if !test.isError {
-			if err != nil {
-				t.Fatalf("Received error when expecting none (%s)\n", err)
-			}
-			retrieved, err := retrieveInventoryFromGroupingObj(test.infos)
-			if err != nil {
-				t.Fatalf("Error retrieving inventory: %s\n", err)
-			}
-			if len(test.expected) != len(retrieved) {
-				t.Errorf("Expected inventory for %d resources, actual %d",
-					len(test.expected), len(retrieved))
-			}
-			for _, expected := range test.expected {
-				found := false
-				for _, actual := range retrieved {
-					if expected.Equals(actual) {
-						found = true
-						continue
-					}
-				}
-				if !found {
-					t.Errorf("Expected inventory (%s) not found", expected)
-				}
-			}
-			// If the grouping object has an inventory, check the
-			// grouping object has an inventory hash.
-			groupingInfo, exists := findGroupingObject(test.infos)
-			if exists && len(test.expected) > 0 {
-				invHash := retrieveInventoryHash(groupingInfo)
-				if len(invHash) == 0 {
-					t.Errorf("Grouping object missing inventory hash")
-				}
-			}
-		}
-	}
-}
-
-func TestAddSuffixToName(t *testing.T) {
-	tests := []struct {
-		info     *resource.Info
-		suffix   string
-		expected string
-		isError  bool
-	}{
-		// Nil info should return error.
-		{
-			info:     nil,
-			suffix:   "",
-			expected: "",
-			isError:  true,
-		},
-		// Empty suffix should return error.
-		{
-			info:     copyGroupingInfo(),
-			suffix:   "",
-			expected: "",
-			isError:  true,
-		},
-		// Empty suffix should return error.
-		{
-			info:     copyGroupingInfo(),
-			suffix:   " \t",
-			expected: "",
-			isError:  true,
-		},
-		{
-			info:     copyGroupingInfo(),
-			suffix:   "hashsuffix",
-			expected: groupingObjName + "-hashsuffix",
-			isError:  false,
-		},
-	}
-
-	for _, test := range tests {
-		//t.Errorf("%#v [%s]", test.info, test.suffix)
-		err := addSuffixToName(test.info, test.suffix)
-		if test.isError {
-			if err == nil {
-				t.Errorf("Should have produced an error, but returned none.")
-			}
-		}
-		if !test.isError {
-			if err != nil {
-				t.Fatalf("Received error when expecting none (%s)\n", err)
-			}
-			actualName, err := getObjectName(test.info.Object)
-			if err != nil {
-				t.Fatalf("Error getting object name: %s", err)
-			}
-			if actualName != test.info.Name {
-				t.Errorf("Object name (%s) does not match info name (%s)\n", actualName, test.info.Name)
-			}
-			if test.expected != actualName {
-				t.Errorf("Expected name (%s), got (%s)\n", test.expected, actualName)
-			}
-		}
-	}
-}
-
-func getObjectName(obj runtime.Object) (string, error) {
-	u, ok := obj.(*unstructured.Unstructured)
-	if !ok {
-		return "", fmt.Errorf("Grouping object is not Unstructured format")
-	}
-	return u.GetName(), nil
-}
-
-func copyGroupingInfo() *resource.Info {
-	groupingObjCopy := groupingObj.DeepCopy()
-	var groupingInfo = &resource.Info{
-		Namespace: testNamespace,
-		Name:      groupingObjName,
-		Object:    groupingObjCopy,
-	}
-	return groupingInfo
-}

cmd/kubectl/kubectlcobra/inventory.go

@@ -1,193 +0,0 @@
-// Copyright 2020 The Kubernetes Authors.
-// SPDX-License-Identifier: Apache-2.0
-
-// package kubectlcobra contains cobra commands from kubectl
-package kubectlcobra
-
-import (
-	"fmt"
-	"sort"
-	"strings"
-
-	"k8s.io/apimachinery/pkg/runtime/schema"
-)
-
-// Separates inventory fields. This string is allowable as a
-// ConfigMap key, but it is not allowed as a character in
-// resource name.
-const fieldSeparator = "_"
-
-// Inventory organizes and stores the indentifying information
-// for an object. This struct (as a string) is stored in a
-// grouping object to keep track of sets of applied objects.
-type Inventory struct {
-	Namespace string
-	Name      string
-	GroupKind schema.GroupKind
-}
-
-// createInventory returns a pointer to an Inventory struct filled
-// with the passed values. This function validates the passed fields
-// and returns an error for bad parameters.
-func createInventory(namespace string,
-	name string, gk schema.GroupKind) (*Inventory, error) {
-
-	// Namespace can be empty, but name cannot.
-	name = strings.TrimSpace(name)
-	if name == "" {
-		return nil, fmt.Errorf("Empty name for inventory object")
-	}
-	if gk.Empty() {
-		return nil, fmt.Errorf("Empty GroupKind for inventory object")
-	}
-
-	return &Inventory{
-		Namespace: strings.TrimSpace(namespace),
-		Name:      name,
-		GroupKind: gk,
-	}, nil
-}
-
-// parseInventory takes a string, splits it into its five fields,
-// and returns a pointer to an Inventory struct storing the
-// five fields. Example inventory string:
-//
-//   test-namespace/test-name/apps/v1/ReplicaSet
-//
-// Returns an error if unable to parse and create the Inventory
-// struct.
-func parseInventory(inv string) (*Inventory, error) {
-	parts := strings.Split(inv, fieldSeparator)
-	if len(parts) == 4 {
-		gk := schema.GroupKind{
-			Group: strings.TrimSpace(parts[2]),
-			Kind:  strings.TrimSpace(parts[3]),
-		}
-		return createInventory(parts[0], parts[1], gk)
-	}
-	return nil, fmt.Errorf("Unable to decode inventory: %s\n", inv)
-}
-
-// Equals returns true if the Inventory structs are identical;
-// false otherwise.
-func (i *Inventory) Equals(other *Inventory) bool {
-	if other == nil {
-		return false
-	}
-	return i.String() == other.String()
-}
-
-// String create a string version of the Inventory struct.
-func (i *Inventory) String() string {
-	return fmt.Sprintf("%s%s%s%s%s%s%s",
-		i.Namespace, fieldSeparator,
-		i.Name, fieldSeparator,
-		i.GroupKind.Group, fieldSeparator,
-		i.GroupKind.Kind)
-}
-
-// InventorySet encapsulates a grouping of unique Inventory
-// structs. Organizes the Inventory structs with a map,
-// which ensures there are no duplicates. Allows set
-// operations such as merging sets and subtracting sets.
-type InventorySet struct {
-	set map[string]*Inventory
-}
-
-// NewInventorySet returns a pointer to an InventorySet
-// struct grouping the passed Inventory items.
-func NewInventorySet(items []*Inventory) *InventorySet {
-	invSet := InventorySet{set: map[string]*Inventory{}}
-	invSet.AddItems(items)
-	return &invSet
-}
-
-// GetItems returns the set of pointers to Inventory
-// structs.
-func (is *InventorySet) GetItems() []*Inventory {
-	items := []*Inventory{}
-	for _, item := range is.set {
-		items = append(items, item)
-	}
-	return items
-}
-
-// AddItems adds Inventory structs to the set which
-// are not already in the set.
-func (is *InventorySet) AddItems(items []*Inventory) {
-	for _, item := range items {
-		if item != nil {
-			is.set[item.String()] = item
-		}
-	}
-}
-
-// DeleteItem removes an Inventory struct from the
-// set if it exists in the set. Returns true if the
-// Inventory item was deleted, false if it did not exist
-// in the set.
-func (is *InventorySet) DeleteItem(item *Inventory) bool {
-	if item == nil {
-		return false
-	}
-	if _, ok := is.set[item.String()]; ok {
-		delete(is.set, item.String())
-		return true
-	}
-	return false
-}
-
-// Merge combines the unique set of Inventory items from the
-// current set with the passed "other" set, returning a new
-// set or error. Returns an error if the passed set to merge
-// is nil.
-func (is *InventorySet) Merge(other *InventorySet) (*InventorySet, error) {
-	if other == nil {
-		return nil, fmt.Errorf("InventorySet to merge is nil.")
-	}
-	// Copy the current InventorySet into result
-	result := NewInventorySet(is.GetItems())
-	result.AddItems(other.GetItems())
-	return result, nil
-}
-
-// Subtract removes the Inventory items in the "other" set from the
-// current set, returning a new set. This does not modify the current
-// set. Returns an error if the passed set to subtract is nil.
-func (is *InventorySet) Subtract(other *InventorySet) (*InventorySet, error) {
-	if other == nil {
-		return nil, fmt.Errorf("InventorySet to subtract is nil.")
-	}
-	// Copy the current InventorySet into result
-	result := NewInventorySet(is.GetItems())
-	// Remove each item in "other" which exists in "result"
-	for _, item := range other.GetItems() {
-		result.DeleteItem(item)
-	}
-	return result, nil
-}
-
-// Equals returns true if the "other" inventory set is the same
-// as this current inventory set. Relies on the fact that the
-// inventory items are sorted for the String() function.
-func (is *InventorySet) Equals(other *InventorySet) bool {
-	if other == nil {
-		return false
-	}
-	return is.String() == other.String()
-}
-
-// String returns a string describing set of Inventory structs.
-func (is *InventorySet) String() string {
-	strs := []string{}
-	for _, item := range is.GetItems() {
-		strs = append(strs, item.String())
-	}
-	sort.Strings(strs)
-	return strings.Join(strs, ", ")
-}
-
-// Size returns the number of Inventory structs in the set.
-func (is *InventorySet) Size() int {
-	return len(is.set)
-}

cmd/kubectl/kubectlcobra/inventory_test.go

@@ -1,558 +0,0 @@
-// Copyright 2020 The Kubernetes Authors.
-// SPDX-License-Identifier: Apache-2.0
-
-// package kubectlcobra contains cobra commands from kubectl
-package kubectlcobra
-
-import (
-	"testing"
-
-	"k8s.io/apimachinery/pkg/runtime/schema"
-)
-
-func TestCreateInventory(t *testing.T) {
-	tests := []struct {
-		namespace string
-		name      string
-		gk        schema.GroupKind
-		expected  string
-		isError   bool
-	}{
-		{
-			namespace: "  \n",
-			name:      " test-name\t",
-			gk: schema.GroupKind{
-				Group: "apps",
-				Kind:  "ReplicaSet",
-			},
-			expected: "_test-name_apps_ReplicaSet",
-			isError:  false,
-		},
-		{
-			namespace: "test-namespace ",
-			name:      " test-name\t",
-			gk: schema.GroupKind{
-				Group: "apps",
-				Kind:  "ReplicaSet",
-			},
-			expected: "test-namespace_test-name_apps_ReplicaSet",
-			isError:  false,
-		},
-		// Error with empty name.
-		{
-			namespace: "test-namespace ",
-			name:      " \t",
-			gk: schema.GroupKind{
-				Group: "apps",
-				Kind:  "ReplicaSet",
-			},
-			expected: "",
-			isError:  true,
-		},
-		// Error with empty GroupKind.
-		{
-			namespace: "test-namespace",
-			name:      "test-name",
-			gk:        schema.GroupKind{},
-			expected:  "",
-			isError:   true,
-		},
-	}
-
-	for _, test := range tests {
-		inv, err := createInventory(test.namespace, test.name, test.gk)
-		if !test.isError {
-			if err != nil {
-				t.Errorf("Error creating inventory when it should have worked.")
-			} else if test.expected != inv.String() {
-				t.Errorf("Expected inventory (%s) != created inventory(%s)\n", test.expected, inv.String())
-			}
-		}
-		if test.isError && err == nil {
-			t.Errorf("Should have returned an error in createInventory()")
-		}
-	}
-}
-
-func TestInventoryEqual(t *testing.T) {
-	tests := []struct {
-		inventory1 *Inventory
-		inventory2 *Inventory
-		isEqual    bool
-	}{
-		// "Other" inventory is nil, then not equal.
-		{
-			inventory1: &Inventory{
-				Name: "test-inv",
-				GroupKind: schema.GroupKind{
-					Group: "apps",
-					Kind:  "Deployment",
-				},
-			},
-			inventory2: nil,
-			isEqual:    false,
-		},
-		// Two equal inventories without a namespace
-		{
-			inventory1: &Inventory{
-				Name: "test-inv",
-				GroupKind: schema.GroupKind{
-					Group: "apps",
-					Kind:  "Deployment",
-				},
-			},
-			inventory2: &Inventory{
-				Name: "test-inv",
-				GroupKind: schema.GroupKind{
-					Group: "apps",
-					Kind:  "Deployment",
-				},
-			},
-			isEqual: true,
-		},
-		// Two equal inventories with a namespace
-		{
-			inventory1: &Inventory{
-				Namespace: "test-namespace",
-				Name:      "test-inv",
-				GroupKind: schema.GroupKind{
-					Group: "apps",
-					Kind:  "Deployment",
-				},
-			},
-			inventory2: &Inventory{
-				Namespace: "test-namespace",
-				Name:      "test-inv",
-				GroupKind: schema.GroupKind{
-					Group: "apps",
-					Kind:  "Deployment",
-				},
-			},
-			isEqual: true,
-		},
-		// One inventory with a namespace, one without -- not equal.
-		{
-			inventory1: &Inventory{
-				Name: "test-inv",
-				GroupKind: schema.GroupKind{
-					Group: "apps",
-					Kind:  "Deployment",
-				},
-			},
-			inventory2: &Inventory{
-				Namespace: "test-namespace",
-				Name:      "test-inv",
-				GroupKind: schema.GroupKind{
-					Group: "apps",
-					Kind:  "Deployment",
-				},
-			},
-			isEqual: false,
-		},
-		// One inventory with a Deployment, one with a ReplicaSet -- not equal.
-		{
-			inventory1: &Inventory{
-				Name: "test-inv",
-				GroupKind: schema.GroupKind{
-					Group: "apps",
-					Kind:  "Deployment",
-				},
-			},
-			inventory2: &Inventory{
-				Name: "test-inv",
-				GroupKind: schema.GroupKind{
-					Group: "apps",
-					Kind:  "ReplicaSet",
-				},
-			},
-			isEqual: false,
-		},
-	}
-
-	for _, test := range tests {
-		actual := test.inventory1.Equals(test.inventory2)
-		if test.isEqual && !actual {
-			t.Errorf("Expected inventories equal, but actual is not: (%s)/(%s)\n", test.inventory1, test.inventory2)
-		}
-	}
-}
-
-func TestParseInventory(t *testing.T) {
-	tests := []struct {
-		invStr    string
-		inventory *Inventory
-		isError   bool
-	}{
-		{
-			invStr: "_test-name_apps_ReplicaSet\t",
-			inventory: &Inventory{
-				Name: "test-name",
-				GroupKind: schema.GroupKind{
-					Group: "apps",
-					Kind:  "ReplicaSet",
-				},
-			},
-			isError: false,
-		},
-		{
-			invStr: "test-namespace_test-name_apps_Deployment",
-			inventory: &Inventory{
-				Namespace: "test-namespace",
-				Name:      "test-name",
-				GroupKind: schema.GroupKind{
-					Group: "apps",
-					Kind:  "Deployment",
-				},
-			},
-			isError: false,
-		},
-		// Not enough fields -- error
-		{
-			invStr:    "_test-name_apps",
-			inventory: &Inventory{},
-			isError:   true,
-		},
-	}
-
-	for _, test := range tests {
-		actual, err := parseInventory(test.invStr)
-		if !test.isError {
-			if err != nil {
-				t.Errorf("Error parsing inventory when it should have worked.")
-			} else if !test.inventory.Equals(actual) {
-				t.Errorf("Expected inventory (%s) != parsed inventory (%s)\n", test.inventory, actual)
-			}
-		}
-		if test.isError && err == nil {
-			t.Errorf("Should have returned an error in parseInventory()")
-		}
-	}
-}
-
-var inventory1 = Inventory{
-	Namespace: "test-namespace",
-	Name:      "test-inv-1",
-	GroupKind: schema.GroupKind{
-		Group: "apps",
-		Kind:  "Deployment",
-	},
-}
-
-var inventory2 = Inventory{
-	Namespace: "test-namespace",
-	Name:      "test-inv-2",
-	GroupKind: schema.GroupKind{
-		Group: "",
-		Kind:  "Pod",
-	},
-}
-
-var inventory3 = Inventory{
-	Namespace: "test-namespace",
-	Name:      "test-inv-3",
-	GroupKind: schema.GroupKind{
-		Group: "",
-		Kind:  "Service",
-	},
-}
-
-var inventory4 = Inventory{
-	Namespace: "test-namespace",
-	Name:      "test-inv-4",
-	GroupKind: schema.GroupKind{
-		Group: "apps",
-		Kind:  "DaemonSet",
-	},
-}
-
-func TestNewInventorySet(t *testing.T) {
-	tests := []struct {
-		items        []*Inventory
-		expectedStr  string
-		expectedSize int
-	}{
-		{
-			items:        []*Inventory{},
-			expectedStr:  "",
-			expectedSize: 0,
-		},
-		{
-			items:        []*Inventory{&inventory1},
-			expectedStr:  "test-namespace_test-inv-1_apps_Deployment",
-			expectedSize: 1,
-		},
-		{
-			items:        []*Inventory{&inventory1, &inventory2},
-			expectedStr:  "test-namespace_test-inv-1_apps_Deployment, test-namespace_test-inv-2__Pod",
-			expectedSize: 2,
-		},
-	}
-
-	for _, test := range tests {
-		invSet := NewInventorySet(test.items)
-		actualStr := invSet.String()
-		actualSize := invSet.Size()
-		if test.expectedStr != actualStr {
-			t.Errorf("Expected InventorySet (%s), got (%s)\n", test.expectedStr, actualStr)
-		}
-		if test.expectedSize != actualSize {
-			t.Errorf("Expected InventorySet size (%d), got (%d)\n", test.expectedSize, actualSize)
-		}
-		actualItems := invSet.GetItems()
-		if len(test.items) != len(actualItems) {
-			t.Errorf("Expected num inventory items (%d), got (%d)\n", len(test.items), len(actualItems))
-		}
-	}
-}
-
-func TestInventorySetAddItems(t *testing.T) {
-	tests := []struct {
-		initialItems  []*Inventory
-		addItems      []*Inventory
-		expectedItems []*Inventory
-	}{
-		// Adding no items to empty inventory set.
-		{
-			initialItems:  []*Inventory{},
-			addItems:      []*Inventory{},
-			expectedItems: []*Inventory{},
-		},
-		// Adding item to empty inventory set.
-		{
-			initialItems:  []*Inventory{},
-			addItems:      []*Inventory{&inventory1},
-			expectedItems: []*Inventory{&inventory1},
-		},
-		// Adding no items does not change the inventory set
-		{
-			initialItems:  []*Inventory{&inventory1},
-			addItems:      []*Inventory{},
-			expectedItems: []*Inventory{&inventory1},
-		},
-		// Adding an item which alread exists does not increase size.
-		{
-			initialItems:  []*Inventory{&inventory1, &inventory2},
-			addItems:      []*Inventory{&inventory1},
-			expectedItems: []*Inventory{&inventory1, &inventory2},
-		},
-		{
-			initialItems:  []*Inventory{&inventory1, &inventory2},
-			addItems:      []*Inventory{&inventory3, &inventory4},
-			expectedItems: []*Inventory{&inventory1, &inventory2, &inventory3, &inventory4},
-		},
-	}
-
-	for _, test := range tests {
-		invSet := NewInventorySet(test.initialItems)
-		invSet.AddItems(test.addItems)
-		if len(test.expectedItems) != invSet.Size() {
-			t.Errorf("Expected num inventory items (%d), got (%d)\n", len(test.expectedItems), invSet.Size())
-		}
-	}
-}
-
-func TestInventorySetDeleteItem(t *testing.T) {
-	tests := []struct {
-		initialItems  []*Inventory
-		deleteItem    *Inventory
-		expected      bool
-		expectedItems []*Inventory
-	}{
-		{
-			initialItems:  []*Inventory{},
-			deleteItem:    nil,
-			expected:      false,
-			expectedItems: []*Inventory{},
-		},
-		{
-			initialItems:  []*Inventory{},
-			deleteItem:    &inventory1,
-			expected:      false,
-			expectedItems: []*Inventory{},
-		},
-		{
-			initialItems:  []*Inventory{&inventory2},
-			deleteItem:    &inventory1,
-			expected:      false,
-			expectedItems: []*Inventory{&inventory2},
-		},
-		{
-			initialItems:  []*Inventory{&inventory1},
-			deleteItem:    &inventory1,
-			expected:      true,
-			expectedItems: []*Inventory{},
-		},
-		{
-			initialItems:  []*Inventory{&inventory1, &inventory2},
-			deleteItem:    &inventory1,
-			expected:      true,
-			expectedItems: []*Inventory{&inventory2},
-		},
-	}
-
-	for _, test := range tests {
-		invSet := NewInventorySet(test.initialItems)
-		actual := invSet.DeleteItem(test.deleteItem)
-		if test.expected != actual {
-			t.Errorf("Expected return value (%t), got (%t)\n", test.expected, actual)
-		}
-		if len(test.expectedItems) != invSet.Size() {
-			t.Errorf("Expected num inventory items (%d), got (%d)\n", len(test.expectedItems), invSet.Size())
-		}
-	}
-}
-
-func TestInventorySetMerge(t *testing.T) {
-	tests := []struct {
-		set1   []*Inventory
-		set2   []*Inventory
-		merged []*Inventory
-	}{
-		{
-			set1:   []*Inventory{},
-			set2:   []*Inventory{},
-			merged: []*Inventory{},
-		},
-		{
-			set1:   []*Inventory{},
-			set2:   []*Inventory{&inventory1},
-			merged: []*Inventory{&inventory1},
-		},
-		{
-			set1:   []*Inventory{&inventory1},
-			set2:   []*Inventory{},
-			merged: []*Inventory{&inventory1},
-		},
-		{
-			set1:   []*Inventory{&inventory1, &inventory2},
-			set2:   []*Inventory{&inventory1},
-			merged: []*Inventory{&inventory1, &inventory2},
-		},
-		{
-			set1:   []*Inventory{&inventory1, &inventory2},
-			set2:   []*Inventory{&inventory1, &inventory2},
-			merged: []*Inventory{&inventory1, &inventory2},
-		},
-		{
-			set1:   []*Inventory{&inventory1, &inventory2},
-			set2:   []*Inventory{&inventory3, &inventory4},
-			merged: []*Inventory{&inventory1, &inventory2, &inventory3, &inventory4},
-		},
-	}
-
-	for _, test := range tests {
-		invSet1 := NewInventorySet(test.set1)
-		invSet2 := NewInventorySet(test.set2)
-		expected := NewInventorySet(test.merged)
-		merged, _ := invSet1.Merge(invSet2)
-		if expected.Size() != merged.Size() {
-			t.Errorf("Expected merged inventory set size (%d), got (%d)\n", expected.Size(), merged.Size())
-		}
-	}
-}
-
-func TestInventorySetSubtract(t *testing.T) {
-	tests := []struct {
-		initialItems  []*Inventory
-		subtractItems []*Inventory
-		expected      []*Inventory
-	}{
-		{
-			initialItems:  []*Inventory{},
-			subtractItems: []*Inventory{},
-			expected:      []*Inventory{},
-		},
-		{
-			initialItems:  []*Inventory{},
-			subtractItems: []*Inventory{&inventory1},
-			expected:      []*Inventory{},
-		},
-		{
-			initialItems:  []*Inventory{&inventory1},
-			subtractItems: []*Inventory{},
-			expected:      []*Inventory{&inventory1},
-		},
-		{
-			initialItems:  []*Inventory{&inventory1, &inventory2},
-			subtractItems: []*Inventory{&inventory1},
-			expected:      []*Inventory{&inventory2},
-		},
-		{
-			initialItems:  []*Inventory{&inventory1, &inventory2},
-			subtractItems: []*Inventory{&inventory1, &inventory2},
-			expected:      []*Inventory{},
-		},
-		{
-			initialItems:  []*Inventory{&inventory1, &inventory2},
-			subtractItems: []*Inventory{&inventory3, &inventory4},
-			expected:      []*Inventory{&inventory1, &inventory2},
-		},
-	}
-
-	for _, test := range tests {
-		invInitialItems := NewInventorySet(test.initialItems)
-		invSubtractItems := NewInventorySet(test.subtractItems)
-		expected := NewInventorySet(test.expected)
-		actual, _ := invInitialItems.Subtract(invSubtractItems)
-		if expected.Size() != actual.Size() {
-			t.Errorf("Expected subtracted inventory set size (%d), got (%d)\n", expected.Size(), actual.Size())
-		}
-	}
-}
-
-func TestInventorySetEquals(t *testing.T) {
-	tests := []struct {
-		set1    []*Inventory
-		set2    []*Inventory
-		isEqual bool
-	}{
-		{
-			set1:    []*Inventory{},
-			set2:    []*Inventory{&inventory1},
-			isEqual: false,
-		},
-		{
-			set1:    []*Inventory{&inventory1},
-			set2:    []*Inventory{},
-			isEqual: false,
-		},
-		{
-			set1:    []*Inventory{&inventory1, &inventory2},
-			set2:    []*Inventory{&inventory1},
-			isEqual: false,
-		},
-		{
-			set1:    []*Inventory{&inventory1, &inventory2},
-			set2:    []*Inventory{&inventory3, &inventory4},
-			isEqual: false,
-		},
-		// Empty sets are equal.
-		{
-			set1:    []*Inventory{},
-			set2:    []*Inventory{},
-			isEqual: true,
-		},
-		{
-			set1:    []*Inventory{&inventory1},
-			set2:    []*Inventory{&inventory1},
-			isEqual: true,
-		},
-		// Ordering of the inventory items does not matter for equality.
-		{
-			set1:    []*Inventory{&inventory1, &inventory2},
-			set2:    []*Inventory{&inventory2, &inventory1},
-			isEqual: true,
-		},
-	}
-
-	for _, test := range tests {
-		invSet1 := NewInventorySet(test.set1)
-		invSet2 := NewInventorySet(test.set2)
-		if !invSet1.Equals(invSet2) && test.isEqual {
-			t.Errorf("Expected equal inventory sets; got unequal (%s)/(%s)\n", invSet1, invSet2)
-		}
-		if invSet1.Equals(invSet2) && !test.isEqual {
-			t.Errorf("Expected inequal inventory sets; got equal (%s)/(%s)\n", invSet1, invSet2)
-		}
-	}
-}