update go.mod for release

Skip writing to unedited files

.golangci-kustomize.yml

@@ -7,8 +7,13 @@ linters:
     - bodyclose
     - deadcode
     - depguard
+    # - dogsled
     - dupl
+    # - errcheck
+    # - funlen
+    # - gochecknoinits
     - goconst
+    # - gocritic
     - gocyclo
     - gofmt
     - goimports
@@ -21,6 +26,7 @@ linters:
     - lll
     - misspell
     - nakedret
+    # - scopelint
     - staticcheck
     - structcheck
     # stylecheck demands that acronyms not be treated as words
@@ -28,9 +34,10 @@ linters:
     # - stylecheck
     - typecheck
     - unconvert
-    - unused
     - unparam
+    - unused
     - varcheck
+    # - whitespace
 
 linters-settings:
   dupl:

.travis.yml

@@ -1,43 +0,0 @@
-os:
-  - linux
-  - osx
-# TODO: Speed up the slowness of the osx travis runs
-# Maybe cache brew installs?
-#
-# TODO: Uncomment when some gets the tests running on Windows.
-#  - windows
-
-addons:
-  apt:
-    packages:
-    - tree
-  homebrew:
-    packages:
-    - tree
-    update: true
-
-# Only clone the most recent commit.
-git:
-  depth: 1
-
-language: go
-
-go:
-  - "1.13"
-
-go_import_path: sigs.k8s.io/kustomize
-
-before_install:
-  - source ./travis/consider-early-travis-exit.sh
-
-# Skip the install process; let pre-commit.sh do it.
-install: true
-
-script:
-  - make verify-kustomize
-  - ./travis/kyaml-pre-commit.sh
-  - ./travis/check-go-mod.sh
-
-# TBD. Suppressing for now.
-notifications:
-  email: false

Makefile

@@ -17,18 +17,30 @@ verify-kustomize: \
 	test-examples-kustomize-against-HEAD \
 	test-examples-kustomize-against-latest
 
+# The following target referenced by a file in
+# https://github.com/kubernetes/test-infra/tree/master/config/jobs/kubernetes-sigs/kustomize
+.PHONY: prow-presubmit-check
+prow-presubmit-check: \
+	lint-kustomize \
+	test-unit-kustomize-all \
+	test-unit-cmd-all \
+	test-go-mod
+
+.PHONY: verify-kustomize-e2e
+verify-kustomize-e2e: test-examples-e2e-kustomize
+
 # Other builds in this repo might want a different linter version.
 # Without one Makefile to rule them all, the different makes
 # cannot assume that golanci-lint is at the version they want
 # since everything uses the same implicit GOPATH.
 # This installs in a temp dir to avoid overwriting someone else's
 # linter, then installs in MYGOBIN with a new name.
-# Version pinned by api/go.mod
+# Version pinned by hack/go.mod
 $(MYGOBIN)/golangci-lint-kustomize:
 	( \
 		set -e; \
-		export GOBIN=$$(mktemp -d) \
-		cd api; \
+		export GOBIN=$$(mktemp -d); \
+		cd hack; \
 		GO111MODULE=on go install github.com/golangci/golangci-lint/cmd/golangci-lint; \
 		mv $$GOBIN/golangci-lint $(MYGOBIN)/golangci-lint-kustomize \
 	)
@@ -48,6 +60,11 @@ $(MYGOBIN)/goimports:
 	cd api; \
 	go install golang.org/x/tools/cmd/goimports
 
+# Install resource from whatever is checked out.
+$(MYGOBIN)/resource:
+	cd cmd/resource; \
+	go install .
+
 # To pin pluginator, use this recipe instead:
 #   cd api;
 #   go install sigs.k8s.io/kustomize/pluginator/v2
@@ -189,10 +206,26 @@ test-unit-kustomize-all: \
 	test-unit-kustomize-cli \
 	test-unit-kustomize-plugins
 
+test-unit-cmd-all:
+	./travis/kyaml-pre-commit.sh
+
+test-go-mod:
+	./travis/check-go-mod.sh
+
 .PHONY:
 test-examples-kustomize-against-HEAD: $(MYGOBIN)/kustomize $(MYGOBIN)/mdrip
 	./hack/testExamplesAgainstKustomize.sh HEAD
 
+.PHONY:
+test-examples-e2e-kustomize: $(MYGOBIN)/mdrip $(MYGOBIN)/kind
+	( \
+		set -e; \
+		/bin/rm -f $(MYGOBIN)/kustomize; \
+		echo "Installing kustomize from ."; \
+		cd kustomize; go install .; cd ..; \
+		./hack/testExamplesE2EAgainstKustomize.sh .; \
+	)
+
 .PHONY:
 test-examples-kustomize-against-latest: $(MYGOBIN)/mdrip
 	( \
@@ -237,6 +270,16 @@ $(MYGOBIN)/helm:
 		rm -rf $$d \
 	)
 
+$(MYGOBIN)/kind:
+	( \
+        set -e; \
+        d=$(shell mktemp -d); cd $$d; \
+        wget -O ./kind https://github.com/kubernetes-sigs/kind/releases/download/v0.7.0/kind-$(shell uname)-amd64; \
+        chmod +x ./kind; \
+        mv ./kind $(MYGOBIN); \
+        rm -rf $$d; \
+	)
+
 .PHONY: clean
 clean: kustomize-external-go-plugin-clean
 	go clean --cache

README.md

@@ -107,7 +107,7 @@ Take the work from step (1) above, move it into a
 `someApp` subdirectory called `base`, then
 place overlays in a sibling directory.
 
-An overlay is just another kustomization, refering to
+An overlay is just another kustomization, referring to
 the base, and referring to patches to apply to that
 base.
 

api/builtins/ImageTagTransformer.go

@@ -144,8 +144,10 @@ func (p *ImageTagTransformerPlugin) findContainers(obj map[string]interface{}) e
 }
 
 func isImageMatched(s, t string) bool {
-	// Tag values are limited to [a-zA-Z0-9_.-].
-	pattern, _ := regexp.Compile("^" + t + "(@sha256)?(:[a-zA-Z0-9_.-]*)?$")
+	// Tag values are limited to [a-zA-Z0-9_.{}-].
+	// Some tools like Bazel rules_k8s allow tag patterns with {} characters.
+	// More info: https://github.com/bazelbuild/rules_k8s/pull/423
+	pattern, _ := regexp.Compile("^" + t + "(@sha256)?(:[a-zA-Z0-9_.{}-]*)?$")
 	return pattern.MatchString(s)
 }
 

api/builtins/NamespaceTransformer.go

@@ -52,7 +52,7 @@ func (p *NamespaceTransformerPlugin) Transform(m resmap.ResMap) error {
 
 		matches := m.GetMatchingResourcesByCurrentId(r.CurId().Equals)
 		if len(matches) != 1 {
-			return fmt.Errorf("namespace tranformation produces ID conflict: %+v", matches)
+			return fmt.Errorf("namespace transformation produces ID conflict: %+v", matches)
 		}
 	}
 	return nil

api/internal/crawl/README.md

@@ -209,7 +209,7 @@ but they are functional.
 
 ## Technical details
 
-### Overall design and imlpementation
+### Overall design and implementations
 
 There are a few components that are all running together in order to get
 the overall application to work smoothly. This section will provide a brief

api/internal/crawl/backend/search_backend.go

@@ -44,7 +44,7 @@ type kustomizeSearch struct {
 // /register: not implemented, but meant as an endpoint for adding new
 // kustomization files to the corpus.
 func NewKustomizeSearch(ctx context.Context) (*kustomizeSearch, error) {
-	idx, err := index.NewKustomizeIndex(ctx)
+	idx, err := index.NewKustomizeIndex(ctx, "kustomize")
 	if err != nil {
 		return nil, err
 	}

api/internal/crawl/cmd/crawler/crawler.go

@@ -2,12 +2,15 @@ package main
 
 import (
 	"context"
+	"flag"
 	"fmt"
 	"log"
 	"net/http"
 	"os"
 	"time"
 
+	"sigs.k8s.io/kustomize/api/internal/crawl/utils"
+
 	"sigs.k8s.io/kustomize/api/internal/crawl/crawler"
 	"sigs.k8s.io/kustomize/api/internal/crawl/crawler/github"
 	"sigs.k8s.io/kustomize/api/internal/crawl/doc"
@@ -25,6 +28,7 @@ const (
 )
 
 type CrawlMode int
+
 const (
 	CrawlUnknown CrawlMode = iota
 	// Crawl all the kustomization files in all the repositories of a Github user
@@ -45,7 +49,7 @@ func NewCrawlMode(s string) CrawlMode {
 		return CrawlUser
 	case "github-repo":
 		return CrawlRepo
-	case "":
+	case "index+github":
 		return CrawlIndexAndGithub
 	case "index":
 		return CrawlIndex
@@ -56,30 +60,33 @@ func NewCrawlMode(s string) CrawlMode {
 	}
 }
 
-func Usage() {
-	fmt.Printf("Usage: %s [mode] [githubUser|githubRepo]\n", os.Args[0])
-	fmt.Printf("\tmode can be one of [github-user, github-repo, index, github]\n")
-	fmt.Printf("%s: crawl all the documents in the index and crawling all the kustomization files on Github\n", os.Args[0])
-	fmt.Printf("%s index: crawl all the documents in the index\n", os.Args[0])
-	fmt.Printf("%s gihub: crawl all the kustomization files on Github\n", os.Args[0])
-	fmt.Printf("%s github-user <github-user>: Crawl all the kustomization files in all the repositories of a Github user\n", os.Args[0])
-	fmt.Printf("\tFor example, %s github-user kubernetes-sigs\n", os.Args[0])
-	fmt.Printf("%s github-repo <github-repo>: Crawl all the kustomization files in a Github repo\n", os.Args[0])
-	fmt.Printf("\tFor example, %s github-repo kubernetes-sigs/kustomize\n", os.Args[0])
-}
-
 func main() {
+	indexNamePtr := flag.String(
+		"index", "kustomize", "The name of the ElasticSearch index.")
+	modePtr := flag.String("mode", "index+github",
+		`The crawling mode, which can be one of [github-user, github-repo, index, github, index+github].
+  * github-user: crawl all the kustomization files in all the repositories of a Github user (--github-user must be specified for this mode).
+  * github-repo: crawl all the kustomization files in a Github repository (--github-repo must be specified for this mode).
+  * index: crawl all the documents in the index.
+  * gihub: crawl all the kustomization files on Github.
+  * index+github: crawl all the documents in the index and crawling all the kustomization files on Github.`)
+	githubUserPtr := flag.String("github-user", "",
+		"A github user name (e.g., kubernetes-sigs). This flag is required for the `github-user` mode.")
+	githubRepoPtr := flag.String("github-repo", "",
+		"A github repository name (e.g., kubernetes-sigs/kustomize). This flag is required for the `github-repo` mode.")
+	flag.Parse()
+
 	githubToken := os.Getenv(githubAccessTokenVar)
 	if githubToken == "" {
-		fmt.Printf("Must set the variable '%s' to make github requests.\n",
+		log.Printf("Must set the variable '%s' to make github requests.\n",
 			githubAccessTokenVar)
 		return
 	}
 
 	ctx := context.Background()
-	idx, err := index.NewKustomizeIndex(ctx)
+	idx, err := index.NewKustomizeIndex(ctx, *indexNamePtr)
 	if err != nil {
-		fmt.Printf("Could not create an index: %v\n", err)
+		log.Printf("Could not create an index: %v\n", err)
 		return
 	}
 
@@ -87,7 +94,7 @@ func main() {
 	cache, err := redis.DialURL(cacheURL)
 	clientCache := &http.Client{}
 	if err != nil {
-		fmt.Printf("Error: redis could not make a connection: %v\n", err)
+		log.Printf("Error: redis could not make a connection: %v\n", err)
 	} else {
 		clientCache = httpclient.NewClient(cache)
 	}
@@ -108,10 +115,10 @@ func main() {
 		case *doc.KustomizationDocument:
 			switch mode {
 			case index.Delete:
-				fmt.Println("Deleting: ", d)
+				log.Printf("Deleting: %v", d)
 				return idx.Delete(d.ID())
 			default:
-				fmt.Println("Inserting: ", d)
+				log.Printf("Inserting: %v", d)
 				return idx.Put(d.ID(), d)
 			}
 		default:
@@ -119,20 +126,15 @@ func main() {
 		}
 	}
 
-	// seen tracks the IDs of all the documents in the index.
+	// seen tracks the IDs of all the documents in the index and their corresponding file types.
 	// This helps avoid indexing a given document multiple times.
-	seen := make(map[string]struct{})
+	seen := utils.NewSeenMap()
 
-	var mode CrawlMode
-	if len(os.Args) == 1 {
-		mode = CrawlIndexAndGithub
-	} else {
-		mode = NewCrawlMode(os.Args[1])
-	}
+	mode := NewCrawlMode(*modePtr)
 
 	ghCrawlerConstructor := func(user, repo string) crawler.Crawler {
 		if user != "" {
-			return 	github.NewCrawler(githubToken, retryCount, clientCache,
+			return github.NewCrawler(githubToken, retryCount, clientCache,
 				github.QueryWith(
 					github.Filename("kustomization.yaml"),
 					github.Filename("kustomization.yml"),
@@ -165,11 +167,11 @@ func main() {
 		it := idx.IterateQuery(query, 10000, 60*time.Second)
 		for it.Next() {
 			for _, hit := range it.Value().Hits.Hits {
-				seedDocs = append(seedDocs, hit.Document.Copy())
+				seedDocs = append(seedDocs, hit.Document.Document.Copy())
 			}
 		}
 		if err := it.Err(); err != nil {
-			fmt.Printf("Error iterating: %v\n", err)
+			log.Fatalf("getSeedDocsFunc Error iterating: %v\n", err)
 		}
 	}
 
@@ -185,23 +187,29 @@ func main() {
 		crawler.CrawlFromSeed(ctx, seedDocs, crawlers, docConverter, indexFunc, seen)
 	case CrawlGithub:
 		crawlers := []crawler.Crawler{ghCrawlerConstructor("", "")}
+		// add all the documents in the index into seen.
+		// this greatly reduces the time overhead of CrawlGithub.
+		getSeedDocsFunc()
+		for _, d := range seedDocs {
+			seen[d.ID()] = d.FileType
+		}
 		crawler.CrawlGithub(ctx, crawlers, docConverter, indexFunc, seen)
 	case CrawlUser:
-		if len(os.Args) < 3 {
-			Usage()
-			log.Fatalf("Please specify a github user!")
+		if *githubUserPtr == "" {
+			flag.Usage()
+			log.Fatalf("Please specify a github user with the github-user flag!")
 		}
-		crawlers := []crawler.Crawler{ghCrawlerConstructor(os.Args[2], "")}
+		crawlers := []crawler.Crawler{ghCrawlerConstructor(*githubUserPtr, "")}
 		crawler.CrawlGithub(ctx, crawlers, docConverter, indexFunc, seen)
 	case CrawlRepo:
-		if len(os.Args) < 3 {
-			Usage()
-			log.Fatalf("Please specify a github repo!")
+		if *githubRepoPtr == "" {
+			flag.Usage()
+			log.Fatalf("Please specify a github repository with the github-repo flag!")
 		}
-		crawlers := []crawler.Crawler{ghCrawlerConstructor("", os.Args[2])}
+		crawlers := []crawler.Crawler{ghCrawlerConstructor("", *githubRepoPtr)}
 		crawler.CrawlGithub(ctx, crawlers, docConverter, indexFunc, seen)
 	case CrawlUnknown:
-		Usage()
-		log.Fatalf("The crawler mode must be one of [github-user, github-repo, index, github]")
+		flag.Usage()
+		log.Fatalf("The --mode flag must be one of [github-user, github-repo, index, github, index+github].")
 	}
 }

api/internal/crawl/cmd/kustomize_stats/main.go

@@ -2,15 +2,16 @@ package main
 
 import (
 	"context"
+	"crypto/sha256"
 	"flag"
 	"fmt"
 	"log"
-	"path/filepath"
 	"sort"
 	"time"
 
+	"sigs.k8s.io/kustomize/api/internal/crawl/doc"
+
 	"sigs.k8s.io/kustomize/api/internal/crawl/index"
-	"sigs.k8s.io/kustomize/api/konfig"
 )
 
 // iterateArr adds each item in arr into countMap.
@@ -25,29 +26,90 @@ func iterateArr(arr []string, countMap map[string]int) {
 
 }
 
-// isKustomizationFile determines whether a file path is a kustomization file
-func isKustomizationFile(path string) bool {
-	basename := filepath.Base(path)
-	for _, name := range konfig.RecognizedKustomizationFileNames() {
-		if basename == name {
-			return true
-		}
-	}
-	return false
-}
-
-// SortMapKeyByValue takes a map as its input, sorts its keys according to their values
+// SortMapKeyByValueInt takes a map as its input, sorts its keys according to their values
 // in the map, and outputs the sorted keys as a slice.
-func SortMapKeyByValue(m map[string]int) []string {
+func SortMapKeyByValueInt(m map[string]int) []string {
 	keys := make([]string, 0, len(m))
 	for key := range m {
 		keys = append(keys, key)
 	}
 	// sort keys according to their values in the map m
-	sort.Slice(keys, func(i, j int) bool {return m[keys[i]] > m[keys[j]]})
+	sort.Slice(keys, func(i, j int) bool { return m[keys[i]] > m[keys[j]] })
 	return keys
 }
 
+// SortMapKeyByValue takes a map as its input, sorts its keys according to their values
+// in the map, and outputs the sorted keys as a slice.
+func SortMapKeyByValueLen(m map[string][]string) []string {
+	keys := make([]string, 0, len(m))
+	for key := range m {
+		keys = append(keys, key)
+	}
+	// sort keys according to their values in the map m
+	sort.Slice(keys, func(i, j int) bool { return len(m[keys[i]]) > len(m[keys[j]]) })
+	return keys
+}
+
+func GeneratorOrTransformerStats(docs []*doc.KustomizationDocument) {
+	n := len(docs)
+	if n == 0 {
+		return
+	}
+
+	fileType := docs[0].FileType
+	fmt.Printf("There are totally %d %s files.\n", n, fileType)
+
+	GitRepositorySummary(docs, fileType)
+
+	// key of kindToUrls: a string in the KustomizationDocument.Kinds field
+	// value of kindToUrls: a slice of string urls defining a given kind.
+	kindToUrls := make(map[string][]string)
+
+	for _, d := range docs {
+		url := fmt.Sprintf("%s/blob/%s/%s", d.RepositoryURL, d.DefaultBranch, d.FilePath)
+		for _, kind := range d.Kinds {
+			if _, ok := kindToUrls[kind]; !ok {
+				kindToUrls[kind] = []string{url}
+			} else {
+				kindToUrls[kind] = append(kindToUrls[kind], url)
+			}
+		}
+	}
+	fmt.Printf("There are totally %d kinds of %s\n", len(kindToUrls), fileType)
+	sortedKeys := SortMapKeyByValueLen(kindToUrls)
+	for _, k := range sortedKeys {
+		sort.Strings(kindToUrls[k])
+		fmt.Printf("%s kind %s appears %d times\n", fileType, k, len(kindToUrls[k]))
+		for _, url := range kindToUrls[k] {
+			fmt.Printf("%s\n", url)
+		}
+	}
+}
+
+// GitRepositorySummary counts the distribution of docs:
+// 1) how many git repositories are these docs from?
+// 2) how many docs are from each git repository?
+func GitRepositorySummary(docs []*doc.KustomizationDocument, fileType string) {
+	m := make(map[string]int)
+	for _, d := range docs {
+		if _, ok := m[d.RepositoryURL]; ok {
+			m[d.RepositoryURL]++
+		} else {
+			m[d.RepositoryURL] = 1
+		}
+	}
+	sortedKeys := SortMapKeyByValueInt(m)
+	topN := 10
+	i := 0
+	for _, k := range sortedKeys {
+		if i >= topN {
+			break
+		}
+		fmt.Printf("%d %s are from %s\n", m[k], fileType, k)
+		i++
+	}
+}
+
 func main() {
 	topKindsPtr := flag.Int(
 		"kinds", -1,
@@ -64,10 +126,12 @@ If you only want to list the 10 most popular identifiers, set the flag to 10.`)
 		`the number of kustomize features to be listed according to their popularities.
 By default, all the features will be listed.
 If you only want to list the 10 most popular features, set the flag to 10.`)
+	indexNamePtr := flag.String(
+		"index", "kustomize", "The name of the ElasticSearch index.")
 	flag.Parse()
 
 	ctx := context.Background()
-	idx, err := index.NewKustomizeIndex(ctx)
+	idx, err := index.NewKustomizeIndex(ctx, *indexNamePtr)
 	if err != nil {
 		log.Fatalf("Could not create an index: %v\n", err)
 	}
@@ -85,35 +149,59 @@ If you only want to list the 10 most popular features, set the flag to 10.`)
 	// ids tracks the unique IDs of the documents in the index
 	ids := make(map[string]struct{})
 
+	// generatorFiles include all the non-kustomization files whose FileType is generator
+	generatorFiles := make([]*doc.KustomizationDocument, 0)
+
+	// transformersFiles include all the non-kustomization files whose FileType is transformer
+	transformersFiles := make([]*doc.KustomizationDocument, 0)
+
+	checksums := make(map[string]int)
+
 	// get all the documents in the index
 	query := []byte(`{ "query":{ "match_all":{} } }`)
 	it := idx.IterateQuery(query, 10000, 60*time.Second)
 	for it.Next() {
 		for _, hit := range it.Value().Hits.Hits {
+			sum := fmt.Sprintf("%x", sha256.Sum256([]byte(hit.Document.DocumentData)))
+			if _, ok := checksums[sum]; ok {
+				checksums[sum]++
+			} else {
+				checksums[sum] = 1
+			}
+
 			// check whether there is any duplicate IDs in the index
 			if _, ok := ids[hit.ID]; !ok {
 				ids[hit.ID] = struct{}{}
 			} else {
-				fmt.Printf("Found duplicate ID (%s)\n", hit.ID)
+				log.Printf("Found duplicate ID (%s)\n", hit.ID)
 			}
 
 			count++
 			iterateArr(hit.Document.Kinds, kindsMap)
 			iterateArr(hit.Document.Identifiers, identifiersMap)
 
-			if isKustomizationFile(hit.Document.FilePath) {
+			if doc.IsKustomizationFile(hit.Document.FilePath) {
 				kustomizationFilecount++
 				iterateArr(hit.Document.Identifiers, kustomizeIdentifiersMap)
+
+			} else {
+				switch hit.Document.FileType {
+				case "generator":
+					generatorFiles = append(generatorFiles, hit.Document.Copy())
+				case "transformer":
+					transformersFiles = append(transformersFiles, hit.Document.Copy())
+				}
 			}
 		}
 	}
+
 	if err := it.Err(); err != nil {
-		fmt.Printf("Error iterating: %v\n", err)
+		log.Fatalf("Error iterating: %v\n", err)
 	}
 
-	sortedKindsMapKeys := SortMapKeyByValue(kindsMap)
-	sortedIdentifiersMapKeys := SortMapKeyByValue(identifiersMap)
-	sortedKustomizeIdentifiersMapKeys := SortMapKeyByValue(kustomizeIdentifiersMap)
+	sortedKindsMapKeys := SortMapKeyByValueInt(kindsMap)
+	sortedIdentifiersMapKeys := SortMapKeyByValueInt(identifiersMap)
+	sortedKustomizeIdentifiersMapKeys := SortMapKeyByValueInt(kustomizeIdentifiersMap)
 
 	fmt.Printf(`The count of unique document IDs in the kustomize index: %d
 There are %d documents in the kustomize index.
@@ -147,4 +235,15 @@ There are %d documents in the kustomize index.
 			kustomizeFeatureCount++
 		}
 	}
+
+	GeneratorOrTransformerStats(generatorFiles)
+	GeneratorOrTransformerStats(transformersFiles)
+
+	fmt.Printf("There are total %d checksums of document contents\n", len(checksums))
+	sortedChecksums := SortMapKeyByValueInt(checksums)
+	sortedChecksums = sortedChecksums[:20]
+	fmt.Printf("The top 20 checksums are:\n")
+	for _, key := range sortedChecksums {
+		fmt.Printf("checksum %s apprears %d\n", key, checksums[key])
+	}
 }

api/internal/crawl/cmd/log-parser/kustomize-stats-cmd

@@ -0,0 +1,7 @@
+wget <log-file-url> -O log
+go build .
+./log-parser log >out
+cat out | grep "kind \`" | cut -d\` -f2 | tail -n 50
+cat out | grep "kind \`" | awk '{print $6}' | tail -n 50
+cat out | grep "feature \`" | grep -v "\`resources\`" | grep -v -e "\`apiVersion\`" | grep -v -e "\`apiversion\`" | cut -d\` -f2
+cat out | grep "feature \`" | grep -v "\`resources\`" | grep -v -e "\`apiVersion\`" | grep -v -e "\`apiversion\`" | awk '{print $6}'

api/internal/crawl/cmd/log-parser/main.go

@@ -36,6 +36,7 @@ func main() {
 
 		m := entry.(map[string]interface{})
 		if payload, ok := m["textPayload"]; ok {
+			// use fmt.Printf here instead of log.Printf to avoid the time and code location info the log package provides
 			fmt.Printf("%s", payload)
 		} else {
 			log.Printf("the log entry does not have the `textPayload` field: %s\n", line)

api/internal/crawl/config/base/kustomization.yaml

@@ -2,5 +2,4 @@ configmapGenerator:
 - name: elasticsearch-config
   literals:
   - es-url="http://esbasic-master:9200"
-  - kustomize-index-name="kustomize"
   - plugin-index-name="plugin"

api/internal/crawl/config/crawler/base/.gitignore

@@ -0,0 +1 @@
+github_api_secret.txt

api/internal/crawl/config/crawler/cronjob/cronjob.yaml

@@ -1,9 +1,10 @@
 apiVersion: batch/v1beta1
 kind: CronJob
 metadata:
-  name: crawler
+  name: crawler-cronjob
 spec:
-  schedule: "5 0 * * */1"
+  # run the cronjob at 00:00 every 7 days
+  schedule: "0 0 */7 * *"
   jobTemplate:
     spec:
       template:
@@ -11,7 +12,9 @@ spec:
           restartPolicy: OnFailure
           containers:
           - name: crawler
-            image: gcr.io/kustomize-search/crawler:latest
+            image: gcr.io/haiyanmeng-gke-dev/crawler:v1
+            command: ["/crawler"]
+            args: ["--mode=index+github", "--github-repo=kubernetes-sigs/kustomize", "--index=kustomize"]
             imagePullPolicy: Always
             env:
             - name: GITHUB_ACCESS_TOKEN

api/internal/crawl/config/crawler/job/README.md

@@ -1,4 +1,4 @@
-There are three ways of running the crawler job.
+The crawler job can run in one of the following mode:
 
 # Crawling all the documents in the index and crawling all the kustomization files on Github
 
@@ -7,14 +7,13 @@ of the container should be:
 
 ```
         command: ["/crawler"]
-        args: []
 ```
 
 Or
 
 ```
         command: ["/crawler"]
-        args: [""]
+        args: ["--mode=index+github"]
 ```
 
 # Crawling all the documents in the index
@@ -23,7 +22,7 @@ The `command` and `args` field of the container should be:
 
 ```
         command: ["/crawler"]
-        args: ["index"]
+        args: ["--mode=index"]
 ```
 
 # Crawling all the kustomization files on Github
@@ -32,7 +31,7 @@ The `command` and `args` field of the container should be:
 
 ```
         command: ["/crawler"]
-        args: ["github"]
+        args: ["--mode=github"]
 ```
 
 # Crawling all the kustomization files in a Github repo
@@ -41,7 +40,7 @@ The `command` and `args` field of the container should be like:
 
 ```
         command: ["/crawler"]
-        args: ["github-repo", "kubernetes-sigs/kustomize"]
+        args: ["--mode=github-repo", "--github-repo=kubernetes-sigs/kustomize"]
 ```
 
 # Crawling all the kustomization files in all the repositories of a Github user
@@ -50,5 +49,5 @@ The `command` and `args` field of the container should be like:
 
 ```
         command: ["/crawler"]
-        args: ["github-user", "kubernetes-sigs"]
+        args: ["--github-user", "--github-user=kubernetes-sigs"]
 ```

api/internal/crawl/config/crawler/job/job.yaml

@@ -11,7 +11,7 @@ spec:
         image: gcr.io/haiyanmeng-gke-dev/crawler:v1
         imagePullPolicy: Always
         command: ["/crawler"]
-        args: ["github-repo", "kubernetes-sigs/kustomize"]
+        args: ["--mode=github-repo", "--github-repo=kubernetes-sigs/kustomize", "--index=kustomize"]
         env:
         - name: GITHUB_ACCESS_TOKEN
           valueFrom:

api/internal/crawl/config/crawler/kustomize_stats/job.yaml

@@ -11,7 +11,7 @@ spec:
         image: gcr.io/haiyanmeng-gke-dev/kustomize_stats:v1
         imagePullPolicy: Always
         command: ["/kustomize_stats"]
-        args: ["--kinds=50", "--identifiers=50", "--kustomize-features=50"]
+        args: ["--index=kustomize", "--kinds=51", "--identifiers=50", "--kustomize-features=50"]
         env:
         - name: ELASTICSEARCH_URL
           valueFrom:

api/internal/crawl/config/elastic/esbackup.yaml

@@ -0,0 +1,23 @@
+# ESBackup depends on ESCluster, and is depended by ESSnapshot.
+# Creating `esbackup/kustomize-backbup` will create the `kustomize-backup` snapshot repository.
+# Deleting `esbackup/kustomize-backbup` will delete the `kustomize-backup` snapshot repository.
+# Deleting `esbackup/kustomize-backbup` will NOT delete the snapshots in the `kustomize-backup` snapshot repository, instead makes all the snapshots in the repository inaccessible.
+# Deleting `esbackup/kustomize-backbup` will NOT delete the essnapshot objects depending on it, but will cause those essnapshot objects to be reconciled, which update the status of the essnapshot objects to reflect the fact that the esbackup object is missing.
+# If you delete the `kustomize-backup` snapshot repository directly without deleting `esbackup/kustomize-backbup`, the ESBackup object will not recreate the snapshot repository.
+apiVersion: elasticsearch.cloud.google.com/v1alpha1
+kind: ESBackup
+metadata:
+  name: kustomize-backup
+spec:
+  storage:
+    gcs:
+      # the bucket must exist for the snapshot repository to be created successfully.
+      bucket: kustomize-backup
+      # the path does not need to exist.
+      # If the path does not exist, the controller will create the folder in the GCS bucket.
+      # If the path already exists and includes snapshots, these snapshots can be used.
+      path: kustomize
+      secret:
+        name: kustomizesa
+  escluster:
+    name: esbasic

api/internal/crawl/config/elastic/escluster.yaml

@@ -1,3 +1,4 @@
+# ESCluster is depended by ESBackup and ESRestore.
 apiVersion: elasticsearch.cloud.google.com/v1alpha1
 kind: ESCluster
 metadata:
@@ -8,6 +9,13 @@ spec:
     - repository-gcs
     - ingest-user-agent
     - ingest-geoip
+  # To set `gcpserviceaccount`,
+  # First, create and download a GCP service account into a json file, named `sakey.json` following the instruction:
+  #     https://www.elastic.co/guide/en/elasticsearch/plugins/6.5/repository-gcs-usage.html#repository-gcs-using-service-account
+  # Second, create a secret for the service account using the following command:
+  #     $ kubectl create secret generic kustomizesa --from-file=./sakey.json
+  gcpserviceaccount:
+    name: kustomizesa
   config:
     env:
       example: test

api/internal/crawl/config/elastic/esrestore.yaml

@@ -0,0 +1,19 @@
+# ESRestore depends on both ESCluster and ESSnapshot.
+# Creating `esrestore/kustomize-restore` will restore the `kuostmize` index in the `kustomize-snapshot` snapshot to a new index named `kusotmize-restore`.
+# Deleting `esrestore/kustomize-restore` will not delete the restored index.
+# Deleting `esrestore/kustomize-restore` should happen before deleting `essnapshot/kustomize-snapshot`.
+# After the restore is complete, if the `kusotmize-restore` index is deleted manually, the ESRestore object will NOT restore the `kustomize` index to it again.
+# The correct way of using ESRestore is: create a ESRestore object to restore the index; delete the ESRestore object after the restore is complete.
+apiVersion: elasticsearch.cloud.google.com/v1alpha1
+kind: ESRestore
+metadata:
+  name: kustomize-restore
+spec:
+  include_global_state: true
+  ignore_unavailable: true
+  rename_pattern: kustomize
+  rename_replacement: kustomize-restore
+  essnapshot:
+    name: kustomize-snapshot
+  escluster:
+    name: esbasic

api/internal/crawl/config/elastic/essnapshot.yaml

@@ -0,0 +1,23 @@
+# ESSnapshot depends on ESBackup, and is depended by ESRestore.
+# Creating `essnapshot/kustomize-snapshot` will create a snapshot named `kustomize-snapshot` in the `kustomize-backup` snapshot repository.
+# After being created, the `kustomize-snapshot` snapshot will not be automatically updated when the `kuostomize` index is updated.
+# If you delete `essnapshot/kustomize-snapshot` and recreate it, the new snapshot will capture the current status of the `kustomize` index.
+# Deleting `essnapshot/kustomize-snapshot` will delete the snapshot.
+# Deleting `essnapshot/kustomize-snapshot` should happen before deleting `esbackup/kustomize-backup`.
+# If the `kustomize-snapshot` snapshot is deleted directly without deleting `essnapshot/kustomize-snapshot`, the ESSnapshot object will recreate the snapshot.
+# The correct way of using ESSnapshot is: create an ESSnapshot object to create a snapshot, keep the ESSnapshot object until the snapshot is no longer needed.
+# To update the snapshot to capture the latest version of the index, you can either:
+#   1) delete the snapshot, and wait for the ESSnapshot object to recreate the snapshot;
+#   2) delete the ESSnapshot object, and recreate the ESSnapshot object.
+apiVersion: elasticsearch.cloud.google.com/v1alpha1
+kind: ESSnapshot
+metadata:
+  name: kustomize-snapshot
+spec:
+  # indices are optional. If not specified all indices are selected.
+  indices:
+  - kustomize
+  include_global_state: true
+  ignore_unavailable: true
+  esbackup:
+    name: kustomize-backup

api/internal/crawl/crawler/crawler.go

@@ -10,6 +10,8 @@ import (
 	"os"
 	"sync"
 
+	"sigs.k8s.io/kustomize/api/internal/crawl/utils"
+
 	"sigs.k8s.io/kustomize/api/internal/crawl/index"
 
 	_ "github.com/gomodule/redigo/redis"
@@ -29,13 +31,15 @@ type Crawler interface {
 	// Crawl returns when it is done processing. This method does not take
 	// ownership of the channel. The channel is write only, and it
 	// designates where the crawler should forward the documents.
-	Crawl(ctx context.Context, output chan<- CrawledDocument) error
+	Crawl(ctx context.Context, output chan<- CrawledDocument, seen utils.SeenMap) error
 
 	// Get the document data given the FilePath, Repo, and Ref/Tag/Branch.
 	FetchDocument(context.Context, *doc.Document) error
 	// Write to the document what the created time is.
 	SetCreated(context.Context, *doc.Document) error
 
+	SetDefaultBranch(*doc.Document)
+
 	Match(*doc.Document) bool
 }
 
@@ -43,7 +47,12 @@ type CrawledDocument interface {
 	ID() string
 	GetDocument() *doc.Document
 	// Get all the Documents directly referred in a Document.
-	GetResources() ([]*doc.Document, error)
+	// For a Document representing a non-kustomization file, an empty slice will be returned.
+	// For a Document representing a kustomization file:
+	// the `includeResources` parameter determines whether the documents referred in the `resources` field are returned or not;
+	// the `includeTransformers` parameter determines whether the documents referred in the `transformers` field are returned or not;
+	// the `includeGenerators` parameter determines whether the documents referred in the `generators` field are returned or not.
+	GetResources(includeResources, includeTransformers, includeGenerators bool) ([]*doc.Document, error)
 	WasCached() bool
 }
 
@@ -69,25 +78,27 @@ func findMatch(d *doc.Document, crawlers []Crawler) Crawler {
 }
 
 func addBranches(cdoc CrawledDocument, match Crawler, indx IndexFunc,
-	seen map[string]struct{}, stack *CrawlSeed) {
+	seen utils.SeenMap, stack *CrawlSeed) {
 
-	seen[cdoc.ID()] = struct{}{}
+	seen.Set(cdoc.ID(), cdoc.GetDocument().FileType)
+
+	match.SetDefaultBranch(cdoc.GetDocument())
 
 	// Insert into index
 	if err := indx(cdoc, index.InsertOrUpdate); err != nil {
-		logger.Printf("Failed to insert or update %s %s: %v",
-			cdoc.GetDocument().RepositoryURL, cdoc.GetDocument().FilePath, err)
+		logger.Printf("Failed to insert or update doc(%s): %v",
+			cdoc.GetDocument().Path(), err)
 		return
 	}
 
-	deps, err := cdoc.GetResources()
+	deps, err := cdoc.GetResources(true, true, true)
 	if err != nil {
 		logger.Println(err)
 		return
 	}
 
 	for _, dep := range deps {
-		if _, ok := seen[dep.ID()]; ok {
+		if seen.Seen(dep.ID()) && seen.Value(dep.ID()) == dep.FileType {
 			continue
 		}
 		*stack = append(*stack, dep)
@@ -95,7 +106,7 @@ func addBranches(cdoc CrawledDocument, match Crawler, indx IndexFunc,
 }
 
 func doCrawl(ctx context.Context, docsPtr *CrawlSeed, crawlers []Crawler, conv Converter, indx IndexFunc,
-	seen map[string]struct{}, stack *CrawlSeed) {
+	seen utils.SeenMap, stack *CrawlSeed, refreshDoc bool, updateFileType bool) {
 
 	UpdatedDocCount := 0
 	seenDocCount := 0
@@ -105,6 +116,7 @@ func doCrawl(ctx context.Context, docsPtr *CrawlSeed, crawlers []Crawler, conv C
 	SetCreatedErrCount := 0
 	convErrCount := 0
 	deleteDocCount := 0
+	crawledDocCount := 0
 
 	// During the execution of the for loop, more Documents may be added into (*docsPtr).
 	for len(*docsPtr) > 0 {
@@ -114,13 +126,19 @@ func doCrawl(ctx context.Context, docsPtr *CrawlSeed, crawlers []Crawler, conv C
 		// remove the last Document in (*docPtr)
 		*docsPtr = (*docsPtr)[:(len(*docsPtr) - 1)]
 
-		if _, ok := seen[tail.ID()]; ok {
-			seenDocCount++
-			continue
+		crawledDocCount++
+		logger.Printf("Crawling doc %d: %s", crawledDocCount, tail.Path())
+
+		if seen.Seen(tail.ID()) {
+			if !updateFileType || seen.Value(tail.ID()) == tail.FileType {
+				logger.Printf("this doc has been seen before")
+				seenDocCount++
+				continue
+			}
 		}
 
 		if tail.WasCached() {
-			logger.Printf("%s %s is cached already", tail.RepositoryURL, tail.FilePath)
+			logger.Printf("doc(%s) is cached already", tail.Path())
 			cachedDocCount++
 			continue
 		}
@@ -132,36 +150,52 @@ func doCrawl(ctx context.Context, docsPtr *CrawlSeed, crawlers []Crawler, conv C
 			continue
 		}
 
-		logger.Println("Crawling ", tail.RepositoryURL, tail.FilePath)
-		if err := match.FetchDocument(ctx, tail); err != nil {
-			logger.Printf("FetchDocument failed on %s %s: %v",
-				tail.RepositoryURL, tail.FilePath, err)
-			FetchDocumentErrCount++
-			// delete the document from the index
-			cdoc := &doc.KustomizationDocument{
-				Document: *tail,
-			}
-			seen[cdoc.ID()] = struct{}{}
-			if err := indx(cdoc, index.Delete); err != nil {
-				logger.Printf("Failed to delete %s %s: %v",
-					cdoc.RepositoryURL, cdoc.FilePath, err)
-			}
-			deleteDocCount++
-			continue
+		if tail.User == "" {
+			tail.User = doc.UserName(tail.RepositoryURL)
 		}
 
-		if err := match.SetCreated(ctx, tail); err != nil {
-			logger.Printf("SetCreated failed on %s %s: %v",
-				tail.RepositoryURL, tail.FilePath, err)
-			SetCreatedErrCount++
+		// If the Document represents a kustomization root, FetchDcoument will change
+		// the `filePath` field of the Document by adding `kustomization.yaml` or
+		// `kustomization.yml` or `kustomization` into the the field.
+		// Therefore, it is necessary to add the ID of the Document into seen before
+		// calling FetchDocument. Otherwise, the binary may enter into an infinite loop
+		// if a kustomization file points to its kustmozation root in its `resources` or
+		// `bases` field.
+		seen.Set(tail.ID(), tail.FileType)
+
+		if refreshDoc || tail.DefaultBranch == "" {
+			match.SetDefaultBranch(tail)
+		}
+
+		if refreshDoc || tail.DocumentData == "" {
+			if err := match.FetchDocument(ctx, tail); err != nil {
+				logger.Printf("FetchDocument failed on doc(%s): %v", tail.Path(), err)
+				FetchDocumentErrCount++
+				// delete the document from the index
+				cdoc := &doc.KustomizationDocument{
+					Document: *tail,
+				}
+				seen.Set(cdoc.ID(), tail.FileType)
+				if err := indx(cdoc, index.Delete); err != nil {
+					logger.Printf("Failed to delete doc(%s): %v", cdoc.Path(), err)
+				}
+				deleteDocCount++
+				continue
+			}
+		}
+
+		if refreshDoc || tail.CreationTime == nil {
+			if err := match.SetCreated(ctx, tail); err != nil {
+				logger.Printf("SetCreated failed on doc(%s): %v", tail.Path(), err)
+				SetCreatedErrCount++
+			}
 		}
 
 		cdoc, err := conv(tail)
 		// If conv returns an error, cdoc can still be added into the index so that
 		// cdoc.Document can be searched.
 		if err != nil {
-			logger.Printf("conv failed on %s %s: %v",
-				tail.RepositoryURL, tail.FilePath, err)
+			logger.Printf("conv failed on doc(%s): %v", tail.Path(), err)
 			convErrCount++
 		}
 
@@ -182,7 +216,7 @@ func doCrawl(ctx context.Context, docsPtr *CrawlSeed, crawlers []Crawler, conv C
 // CrawlFromSeed updates all the documents in seed, and crawls all the new
 // documents referred in the seed.
 func CrawlFromSeed(ctx context.Context, seed CrawlSeed, crawlers []Crawler,
-	conv Converter, indx IndexFunc, seen map[string]struct{}) {
+	conv Converter, indx IndexFunc, seen utils.SeenMap) {
 
 	// stack tracks the documents directly referred in other documents.
 	stack := make(CrawlSeed, 0)
@@ -190,14 +224,14 @@ func CrawlFromSeed(ctx context.Context, seed CrawlSeed, crawlers []Crawler,
 	// Exploit seed to update bulk of corpus.
 	logger.Printf("updating %d documents from seed\n", len(seed))
 	// each unique document in seed will be crawled once.
-	doCrawl(ctx, &seed, crawlers, conv, indx, seen, &stack)
+	doCrawl(ctx, &seed, crawlers, conv, indx, seen, &stack, true, false)
 
 	// Traverse any new documents added while updating corpus.
 	logger.Printf("crawling %d new documents found in the seed\n", len(stack))
 	// While crawling each document in stack, the documents directly referred in the document
 	// will be added into stack.
 	// After this statement is done, stack will become empty.
-	doCrawl(ctx, &stack, crawlers, conv, indx, seen, &stack)
+	doCrawl(ctx, &stack, crawlers, conv, indx, seen, &stack, false, true)
 }
 
 // CrawlGithubRunner is a blocking function and only returns once all of the
@@ -218,7 +252,7 @@ func CrawlFromSeed(ctx context.Context, seed CrawlSeed, crawlers []Crawler,
 // from the seed will be processed before any other documents from the
 // crawlers.
 func CrawlGithubRunner(ctx context.Context, output chan<- CrawledDocument,
-	crawlers []Crawler) []error {
+	crawlers []Crawler, seen utils.SeenMap) []error {
 
 	errs := make([]error, len(crawlers))
 	wg := sync.WaitGroup{}
@@ -252,7 +286,7 @@ func CrawlGithubRunner(ctx context.Context, output chan<- CrawledDocument,
 				}
 			}()
 			defer close(docs)
-			errs[idx] = crawler.Crawl(ctx, docs)
+			errs[idx] = crawler.Crawl(ctx, docs, seen)
 		}(i, crawler, docs) // Copies the index and the crawler
 	}
 
@@ -262,7 +296,7 @@ func CrawlGithubRunner(ctx context.Context, output chan<- CrawledDocument,
 
 // CrawlGithub crawls all the kustomization files on Github.
 func CrawlGithub(ctx context.Context, crawlers []Crawler, conv Converter,
-	indx IndexFunc, seen map[string]struct{}) {
+	indx IndexFunc, seen utils.SeenMap) {
 	// stack tracks the documents directly referred in other documents.
 	stack := make(CrawlSeed, 0)
 
@@ -274,8 +308,14 @@ func CrawlGithub(ctx context.Context, crawlers []Crawler, conv Converter,
 	wg.Add(1)
 	go func() {
 		defer wg.Done()
+		docCount := 0
 		for cdoc := range ch {
-			if _, ok := seen[cdoc.ID()]; ok {
+			docCount++
+			logger.Printf("Processing doc %d found on Github", docCount)
+			// all the docs here are kustomization files found by querying Github, and
+			// their `FileType` fields all should be empty.
+			if seen.Seen(cdoc.ID()) {
+				logger.Printf("the doc has been seen before")
 				continue
 			}
 			match := findMatch(cdoc.GetDocument(), crawlers)
@@ -289,7 +329,7 @@ func CrawlGithub(ctx context.Context, crawlers []Crawler, conv Converter,
 	}()
 
 	logger.Println("processing the documents found from crawling github")
-	if errs := CrawlGithubRunner(ctx, ch, crawlers); errs != nil {
+	if errs := CrawlGithubRunner(ctx, ch, crawlers, seen); errs != nil {
 		for _, err := range errs {
 			logIfErr(err)
 		}
@@ -300,5 +340,5 @@ func CrawlGithub(ctx context.Context, crawlers []Crawler, conv Converter,
 	// Handle deps of newly discovered documents.
 	logger.Printf("crawling the %d new documents referred by other documents",
 		len(stack))
-	doCrawl(ctx, &stack, crawlers, conv, indx, seen, &stack)
+	doCrawl(ctx, &stack, crawlers, conv, indx, seen, &stack, false, true)
 }

api/internal/crawl/crawler/crawler_test.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"log"
 	"reflect"
 	"sort"
 	"strings"
@@ -11,6 +12,8 @@ import (
 	"testing"
 	"time"
 
+	"sigs.k8s.io/kustomize/api/internal/crawl/utils"
+
 	"sigs.k8s.io/kustomize/api/internal/crawl/index"
 
 	"sigs.k8s.io/kustomize/api/internal/crawl/doc"
@@ -34,6 +37,8 @@ func (c testCrawler) Match(d *doc.Document) bool {
 	return d != nil
 }
 
+func (c testCrawler) SetDefaultBranch(d *doc.Document) {}
+
 func (c testCrawler) FetchDocument(_ context.Context, d *doc.Document) error {
 	if i, ok := c.lukp[d.ID()]; ok {
 		d.DocumentData = c.docs[i].DocumentData
@@ -75,7 +80,7 @@ func newCrawler(matchPrefix string, err error,
 
 // Crawl implements the Crawler interface for testing.
 func (c testCrawler) Crawl(_ context.Context,
-	output chan<- CrawledDocument) error {
+	output chan<- CrawledDocument, _ utils.SeenMap) error {
 
 	for i, d := range c.docs {
 		isResource := true
@@ -110,7 +115,7 @@ func (s sortableDocs) Len() int {
 }
 
 func TestCrawlGithubRunner(t *testing.T) {
-	fmt.Println("testing CrawlGithubRunner")
+	log.Println("testing CrawlGithubRunner")
 	tests := []struct {
 		tc   []Crawler
 		errs []error
@@ -181,8 +186,9 @@ func TestCrawlGithubRunner(t *testing.T) {
 			defer close(output)
 			defer wg.Done()
 
+			seen := utils.NewSeenMap()
 			errs := CrawlGithubRunner(context.Background(),
-				output, test.tc)
+				output, test.tc, seen)
 
 			// Check that errors are returned as they should be.
 			if !reflect.DeepEqual(errs, test.errs) {
@@ -215,7 +221,7 @@ func TestCrawlGithubRunner(t *testing.T) {
 }
 
 func TestCrawlFromSeed(t *testing.T) {
-	fmt.Println("testing CrawlFromSeed")
+	log.Println("testing CrawlFromSeed")
 
 	tests := []struct {
 		seed    CrawlSeed
@@ -322,7 +328,7 @@ resources:
 				visited[d.ID()]++
 				return nil
 			},
-			make(map[string]struct{}),
+			utils.NewSeenMap(),
 		)
 		if lv, lc := len(visited), len(tc.corpus); lv != lc {
 			t.Errorf("error: %d of %d documents visited.", lv, lc)

api/internal/crawl/crawler/github/crawler.go

@@ -16,6 +16,8 @@ import (
 	"strings"
 	"time"
 
+	"sigs.k8s.io/kustomize/api/internal/crawl/utils"
+
 	"sigs.k8s.io/kustomize/api/internal/crawl/crawler"
 	"sigs.k8s.io/kustomize/api/internal/crawl/doc"
 	"sigs.k8s.io/kustomize/api/internal/crawl/httpclient"
@@ -30,6 +32,8 @@ var logger = log.New(os.Stdout, "Github Crawler: ",
 type githubCrawler struct {
 	client GhClient
 	query  Query
+	// branchMap maps github repositories to their default branches
+	branchMap map[string]string
 }
 
 type GhClient struct {
@@ -51,13 +55,60 @@ func NewCrawler(accessToken string, retryCount uint64, client *http.Client,
 			},
 			accessToken: accessToken,
 		},
-		query: query,
+		query:     query,
+		branchMap: map[string]string{},
 	}
 }
 
+func (gc githubCrawler) SetDefaultBranch(d *doc.Document) {
+	url := gc.client.ReposRequest(d.RepositoryFullName())
+	defaultBranch, err := gc.client.GetDefaultBranch(url, d.RepositoryURL, gc.branchMap)
+	if err != nil {
+		logger.Printf(
+			"(error: %v) setting default_branch to master\n", err)
+		defaultBranch = "master"
+	}
+	d.DefaultBranch = defaultBranch
+	gc.branchMap[d.RepositoryURL] = d.DefaultBranch
+}
+
+func (gc githubCrawler) DefaultBranch(repo string) string {
+	return gc.branchMap[repo]
+}
+
 // Implements crawler.Crawler.
-func (gc githubCrawler) Crawl(
-	ctx context.Context, output chan<- crawler.CrawledDocument) error {
+func (gc githubCrawler) Crawl(ctx context.Context,
+	output chan<- crawler.CrawledDocument, seen utils.SeenMap) error {
+
+	ranges := []RangeWithin{
+		RangeWithin{
+		start: uint64(0),
+		end:   githubMaxFileSize,
+	},
+	}
+
+	errs := make(multiError, 0)
+	for len(ranges) > 0 {
+		tailRange := ranges[len(ranges) - 1]
+		ranges = ranges[:(len(ranges) - 1)]
+		reProcessQueryRanges, err := gc.CrawlSingleRange(ctx, output, seen, tailRange.start, tailRange.end)
+		if err != nil {
+			errs = append(errs, err)
+		}
+		ranges = append(ranges, reProcessQueryRanges...)
+	}
+
+	if len(errs) > 0 {
+		return errs
+	}
+	return nil
+}
+
+func (gc githubCrawler) CrawlSingleRange(ctx context.Context,
+	output chan<- crawler.CrawledDocument, seen utils.SeenMap,
+	lowerBound, upperBound uint64) ([]RangeWithin, error) {
+
+	log.Printf("CrawlSingleRange [%d, %d]", lowerBound, upperBound)
 
 	noETagClient := GhClient{
 		RequestConfig: gc.client.RequestConfig,
@@ -66,12 +117,25 @@ func (gc githubCrawler) Crawl(
 		accessToken:   gc.client.accessToken,
 	}
 
+	var reProcessQueryRanges []RangeWithin
+
+	var ranges []string
+	var err error
 	// Since Github returns a max of 1000 results per query, we can use
 	// multiple queries that split the search space into chunks of at most
 	// 1000 files to get all of the data.
-	ranges, err := FindRangesForRepoSearch(newCache(noETagClient, gc.query))
+	for i := 0; i < 5; i++ {
+		ranges, err = FindRangesForRepoSearch(newCache(noETagClient, gc.query),
+			lowerBound, upperBound)
+		if err == nil {
+			logger.Printf("FindRangesForRepoSearch succeeded after %d retries", i)
+			break
+		} else {
+			time.Sleep(time.Minute)
+		}
+	}
 	if err != nil {
-		return fmt.Errorf("could not split %v into ranges, %v\n",
+		return reProcessQueryRanges, fmt.Errorf("could not split %v into ranges, %v\n",
 			gc.query, err)
 	}
 
@@ -79,35 +143,31 @@ func (gc githubCrawler) Crawl(
 
 	// Query each range for files.
 	errs := make(multiError, 0)
+	queryResult := RangeQueryResult{}
 	for _, query := range ranges {
-		err := processQuery(ctx, gc.client, query, output)
+		reProcessQuery, rangeResult, err := processQuery(ctx, gc.client, query, output, seen, gc.branchMap)
 		if err != nil {
 			errs = append(errs, err)
 		}
+		queryResult.Add(rangeResult)
+		if reProcessQuery {
+			reProcessQueryRanges = append(reProcessQueryRanges, RangeSizes(query))
+		}
 	}
 
+	logger.Printf("Summary of Crawl: %s", queryResult.String())
+
 	if len(errs) > 0 {
-		return errs
+		return reProcessQueryRanges, errs
 	}
 
-	return nil
+	return reProcessQueryRanges, nil
 }
 
 // FetchDocument first tries to fetch the document with d.FilePath. If it fails,
 // it will try to add each string in konfig.RecognizedKustomizationFileNames() to
 // d.FilePath, and try to fetch the document again.
 func (gc githubCrawler) FetchDocument(_ context.Context, d *doc.Document) error {
-	// set the default branch if it is empty
-	if d.DefaultBranch == "" {
-		url := gc.client.ReposRequest(d.RepositoryFullName())
-		defaultBranch, err := gc.client.GetDefaultBranch(url)
-		if err != nil {
-			logger.Printf(
-				"(error: %v) setting default_branch to master\n", err)
-			defaultBranch = "master"
-		}
-		d.DefaultBranch = defaultBranch
-	}
 	repoURL := d.RepositoryURL + "/" + d.FilePath + "?ref=" + d.DefaultBranch
 	repoSpec, err := git.NewRepoSpecFromUrl(repoURL)
 	if err != nil {
@@ -176,10 +236,32 @@ func (gc githubCrawler) Match(d *doc.Document) bool {
 	return strings.Contains(repoSpec.Host, "github.com")
 }
 
+type RangeQueryResult struct {
+	totalDocCnt uint64
+	seenDocCnt  uint64
+	newDocCnt   uint64
+	errorCnt    uint64
+}
+
+func (r *RangeQueryResult) Add(other RangeQueryResult) {
+	r.totalDocCnt += other.totalDocCnt
+	r.newDocCnt += other.newDocCnt
+	r.seenDocCnt += other.seenDocCnt
+	r.errorCnt += other.errorCnt
+}
+
+func (r *RangeQueryResult) String() string {
+	return fmt.Sprintf("got %d files from API. "+
+		"%d have been seen before. %d are new and sent to the output channel."+
+		" %d have kustomizationResultAdapter errors.",
+		r.totalDocCnt, r.seenDocCnt, r.newDocCnt, r.errorCnt)
+}
+
 // processQuery follows all of the pages in a query, and updates/adds the
 // documents from the crawl to the datastore/index.
 func processQuery(ctx context.Context, gcl GhClient, query string,
-	output chan<- crawler.CrawledDocument) error {
+	output chan<- crawler.CrawledDocument, seen utils.SeenMap,
+	branchMap map[string]string) (bool, RangeQueryResult, error) {
 
 	queryPages := make(chan GhResponseInfo)
 
@@ -195,57 +277,84 @@ func processQuery(ctx context.Context, gcl GhClient, query string,
 		close(queryPages)
 	}()
 
+	reProcessQuery := false
+
 	errs := make(multiError, 0)
-	errorCnt := 0
-	totalCnt := 0
+	result := RangeQueryResult{}
+	pageID := 1
 	for page := range queryPages {
 		if page.Error != nil {
 			errs = append(errs, page.Error)
 			continue
 		}
-
+		pageResult := RangeQueryResult{}
 		for _, file := range page.Parsed.Items {
-			k, err := kustomizationResultAdapter(gcl, file)
+			k, err := kustomizationResultAdapter(gcl, file, seen, branchMap)
 			if err != nil {
 				logger.Printf("kustomizationResultAdapter failed: %v", err)
 				errs = append(errs, err)
-				errorCnt++
+				pageResult.errorCnt++
 			}
 			if k != nil {
+				pageResult.newDocCnt++
 				output <- k
+			} else {
+				pageResult.seenDocCnt++
 			}
-			totalCnt++
+			pageResult.totalDocCnt++
 		}
 
-		logger.Printf("got %d files out of %d from API. %d of %d had errors\n",
-			totalCnt, page.Parsed.TotalCount, errorCnt, totalCnt)
+		logger.Printf("processQuery [TotalCount %d - page %d]: %s",
+			page.Parsed.TotalCount, pageID, pageResult.String())
+		result.Add(pageResult)
+
+		pageID++
+
+		if page.Parsed.TotalCount > githubMaxResultsPerQuery {
+			reProcessQuery = true
+		}
 	}
 
-	return errs
+	logger.Printf("Summary of processQuery: %s", result.String())
+
+	return reProcessQuery, result, errs
 }
 
-func kustomizationResultAdapter(gcl GhClient, k GhFileSpec) (
-	crawler.CrawledDocument, error) {
-
-	data, err := gcl.GetFileData(k)
-	if err != nil {
-		return nil, err
-	}
-
+func kustomizationResultAdapter(gcl GhClient, k GhFileSpec, seen utils.SeenMap,
+	branchMap map[string]string) (crawler.CrawledDocument, error) {
 	url := gcl.ReposRequest(k.Repository.FullName)
-	defaultBranch, err := gcl.GetDefaultBranch(url)
+	defaultBranch, err := gcl.GetDefaultBranch(url, k.Repository.URL, branchMap)
 	if err != nil {
 		logger.Printf(
 			"(error: %v) setting default_branch to master\n", err)
 		defaultBranch = "master"
 	}
 
+	// document here is a kustomization file found by querying Github, whose
+	// `FileType` field should be empty.
+	document := doc.Document{
+		FilePath:      k.Path,
+		DefaultBranch: defaultBranch,
+		RepositoryURL: k.Repository.URL,
+		User:          doc.UserName(k.Repository.URL),
+	}
+
+	if seen.Seen(document.ID()) {
+		return nil, nil
+	}
+
+	data, err := gcl.GetFileData(k)
+	if err != nil {
+		return nil, err
+	}
+
 	d := doc.KustomizationDocument{
 		Document: doc.Document{
 			DocumentData:  string(data),
 			FilePath:      k.Path,
 			DefaultBranch: defaultBranch,
 			RepositoryURL: k.Repository.URL,
+			User:          doc.UserName(k.Repository.URL),
 		},
 	}
 	creationTime, err := gcl.GetFileCreationTime(k)
@@ -270,7 +379,7 @@ func (gcl GhClient) ForwardPaginatedQuery(ctx context.Context, query string,
 	output chan<- GhResponseInfo) error {
 
 	logger.Println("querying: ", query)
-	response := gcl.parseGithubResponse(query)
+	response := gcl.parseGithubResponseWithRetry(query)
 
 	if response.Error != nil {
 		return response.Error
@@ -283,7 +392,7 @@ func (gcl GhClient) ForwardPaginatedQuery(ctx context.Context, query string,
 		case <-ctx.Done():
 			return nil
 		default:
-			response = gcl.parseGithubResponse(response.NextURL)
+			response = gcl.parseGithubResponseWithRetry(response.NextURL)
 			if response.Error != nil {
 				return response.Error
 			}
@@ -344,7 +453,15 @@ func CloseResponseBody(resp *http.Response) {
 	}
 }
 
-func (gcl GhClient) GetDefaultBranch(url string) (string, error) {
+// GetDefaultBranch gets the default branch of a github repository.
+// m is a map which maps a github repository to its default branch.
+// If repo is already in m, the default branch for url will be obtained from m;
+// otherwise, a query will be made to github to obtain the default branch.
+func (gcl GhClient) GetDefaultBranch(url, repo string, m map[string]string) (string, error) {
+	if v, ok := m[repo]; ok {
+		return v, nil
+	}
+
 	resp, err := gcl.GetReposData(url)
 	if err != nil {
 		return "", fmt.Errorf(
@@ -470,6 +587,8 @@ type githubResponse struct {
 	// This is the number of files that match the query.
 	TotalCount uint64 `json:"total_count,omitempty"`
 
+	IncompleteResults bool `json:"incomplete_results,omitempty"`
+
 	// Github representation of a file.
 	Items []GhFileSpec `json:"items,omitempty"`
 }
@@ -512,6 +631,17 @@ func parseGithubLinkFormat(links string) (string, string) {
 	return next, last
 }
 
+func (gcl GhClient) parseGithubResponseWithRetry(getRequest string) GhResponseInfo {
+	resp := gcl.parseGithubResponse(getRequest)
+	retries := 0
+	for resp.Parsed.IncompleteResults {
+		resp = gcl.parseGithubResponse(getRequest)
+		retries++
+	}
+	log.Printf("The result of query(%s) is complete after %d retries", getRequest, retries)
+	return resp
+}
+
 func (gcl GhClient) parseGithubResponse(getRequest string) GhResponseInfo {
 	resp, err := gcl.SearchGithubAPI(getRequest)
 	requestInfo := GhResponseInfo{
@@ -589,7 +719,7 @@ func (gcl GhClient) Do(query string) (*http.Response, error) {
 
 	// gcl.client.Do: a non-2xx status code doesn't cause an error.
 	// See https://golang.org/pkg/net/http/#Client.Do for more info.
-	resp, err :=  gcl.client.Do(req)
+	resp, err := gcl.client.Do(req)
 	if resp != nil && resp.StatusCode != http.StatusOK {
 		err = fmt.Errorf("GhClient.Do(%s) failed with response code: %d",
 			query, resp.StatusCode)

api/internal/crawl/crawler/github/queries.go

@@ -117,7 +117,7 @@ type RequestConfig struct {
 // understand why the request object is useful.
 func (rc RequestConfig) CodeSearchRequestWith(query Query) request {
 	vals := url.Values{
-		"sort": []string{"indexed"},
+		"sort":  []string{"indexed"},
 		"order": []string{"desc"},
 	}
 	req := rc.makeRequest("search/code", query, vals)
@@ -166,7 +166,7 @@ type request struct {
 	query Query
 }
 
-// CopyWith copies the requests and adds the extra query parameters. Usefull
+// CopyWith copies the requests and adds the extra query parameters. It is useful
 // for dynamically adding sizes to a filename only query without modifying it.
 func (r request) CopyWith(queryParams ...queryField) request {
 	cpy := r

api/internal/crawl/crawler/github/split_search_ranges.go

@@ -93,13 +93,15 @@ package github
 //   apiCallsPerResult * 10(pages) * 100(resultsPerPage) * totalResults / 1000
 //   = apiCallsPerResult * totalResults.
 //
-// So it could very well take apiCallsPerResult * 50 times longer to acutally
+// So it could very well take apiCallsPerResult * 50 times longer to actually
 // fetch the results (assuming the quotas for the API calls are the same as the
 // search API), than it does to perform these range searches.
 
 import (
 	"fmt"
 	"math/bits"
+	"strconv"
+	"strings"
 )
 
 // Files cannot be more than 2^19 bytes, according to
@@ -112,7 +114,7 @@ const (
 // Interface instead of struct for testing purposes.
 // Not expecting to have multiple implementations.
 type cachedSearch interface {
-	CountResults(uint64) (uint64, error)
+	CountResults(uint64, uint64) (uint64, error)
 	RequestString(filesize rangeFormatter) string
 }
 
@@ -139,7 +141,7 @@ type cachedSearch interface {
 //    problematic). The current cache implementation looks at the
 //    predecessor entry to find out if the current value is monotonic.
 //    This is where the bit trick is used, since each step in the binary
-//    search is adding or ommiting to add a decreasing power of 2 to the query
+//    search is adding or omitting to add a decreasing power of 2 to the query
 //    value, we can remove the least significant set bit to find the
 //    predecessor in constant time. Ultimately since the search is rate
 //    limited, we could also easily afford to compute this in linear time
@@ -161,16 +163,16 @@ func newCache(client GhClient, query Query) githubCachedSearch {
 	}
 }
 
-func (c githubCachedSearch) CountResults(upperBound uint64) (uint64, error) {
+func (c githubCachedSearch) CountResults(lowerBound, upperBound uint64) (uint64, error) {
 	count, cached := c.cache[upperBound]
 	if cached {
 		return count, nil
 	}
 
-	sizeRange := RangeWithin{0, upperBound}
+	sizeRange := RangeWithin{lowerBound, upperBound}
 	rangeRequest := c.RequestString(sizeRange)
 
-	result := c.gcl.parseGithubResponse(rangeRequest)
+	result := c.gcl.parseGithubResponseWithRetry(rangeRequest)
 	if result.Error != nil {
 		return count, result.Error
 	}
@@ -204,7 +206,7 @@ func (c githubCachedSearch) CountResults(upperBound uint64) (uint64, error) {
 			"Retrying query... current lower bound: %d, got: %d\n",
 			c.cache[prev], result.Parsed.TotalCount)
 
-		result = c.gcl.parseGithubResponse(rangeRequest)
+		result = c.gcl.parseGithubResponseWithRetry(rangeRequest)
 		if result.Error != nil {
 			return count, result.Error
 		}
@@ -219,8 +221,8 @@ func (c githubCachedSearch) CountResults(upperBound uint64) (uint64, error) {
 	}
 
 	count = result.Parsed.TotalCount
-	logger.Printf("Caching new query %s, with count %d\n",
-		sizeRange.RangeString(), count)
+	logger.Printf("Caching new query %s, with count %d (incomplete_results: %v)\n",
+		sizeRange.RangeString(), count, result.Parsed.IncompleteResults)
 	c.cache[upperBound] = count
 	return count, nil
 }
@@ -238,8 +240,8 @@ func (c githubCachedSearch) RequestString(filesize rangeFormatter) string {
 // This would mean that the search as it is could not find all files. If queries
 // are sorted by last indexed, and retrieved on regular intervals, it should be
 // sufficient to get most if not all documents.
-func FindRangesForRepoSearch(cache cachedSearch) ([]string, error) {
-	totalFiles, err := cache.CountResults(githubMaxFileSize)
+func FindRangesForRepoSearch(cache cachedSearch, lowerBound, upperBound uint64) ([]string, error) {
+	totalFiles, err := cache.CountResults(lowerBound, upperBound)
 	if err != nil {
 		return nil, err
 	}
@@ -247,7 +249,7 @@ func FindRangesForRepoSearch(cache cachedSearch) ([]string, error) {
 
 	if githubMaxResultsPerQuery >= totalFiles {
 		return []string{
-			cache.RequestString(RangeWithin{0, githubMaxFileSize}),
+			cache.RequestString(RangeWithin{lowerBound, upperBound}),
 		}, nil
 	}
 
@@ -275,6 +277,7 @@ func FindRangesForRepoSearch(cache cachedSearch) ([]string, error) {
 	// range.
 	filesAccessible := uint64(0)
 	sizes := make([]uint64, 0)
+	sizes = append(sizes, lowerBound)
 	for filesAccessible < totalFiles {
 		target := filesAccessible + githubMaxResultsPerQuery
 		if target >= totalFiles {
@@ -284,22 +287,22 @@ func FindRangesForRepoSearch(cache cachedSearch) ([]string, error) {
 		logger.Printf("%d accessible files, next target = %d\n",
 			filesAccessible, target)
 
-		cur, err := lowerBoundFileCount(cache, target)
+		size, err := FindFileSize(cache, target, lowerBound, upperBound)
 		if err != nil {
 			return nil, err
 		}
 
 		// If there are more than 1000 files in the next bucket, we must
 		// advance anyway and lose out on some files :(.
-		if l := len(sizes); l > 0 && sizes[l-1] == cur {
-			cur++
+		if l := len(sizes); l > 0 && sizes[l-1] == size {
+			size++
 		}
 
-		nextAccessible, err := cache.CountResults(cur)
+		nextAccessible, err := cache.CountResults(lowerBound, size)
 		if err != nil {
 			return nil, fmt.Errorf(
 				"cache should be populated at %d already, got %v",
-				cur, err)
+				size, err)
 		}
 		if nextAccessible < filesAccessible {
 			return nil, fmt.Errorf(
@@ -309,31 +312,31 @@ func FindRangesForRepoSearch(cache cachedSearch) ([]string, error) {
 
 		filesAccessible = nextAccessible
 		if nextAccessible < totalFiles {
-			sizes = append(sizes, cur)
+			sizes = append(sizes, size)
 		}
 	}
-
+	sizes = append(sizes, upperBound)
 	return formatFilesizeRanges(cache, sizes), nil
 }
 
-// lowerBoundFileCount finds the filesize range from [0, return value] that has
+// FindFileSize finds the filesize range from [lowerBound, return value] that has
 // the largest file count that is smaller than or equal to
 // githubMaxResultsPerQuery. It is important to note that this returned value
 // could already be in a previous range if the next file size has more than 1000
 // results. It is left to the caller to handle this bit of logic and guarantee
 // forward progession in this case.
-func lowerBoundFileCount(
-	cache cachedSearch, targetFileCount uint64) (uint64, error) {
+func FindFileSize(
+	cache cachedSearch, targetFileCount, lowerBound, upperBound uint64) (uint64, error) {
 
 	// Binary search for file sizes that make up the next <=1000 element
 	// chunk.
-	cur := uint64(0)
-	increase := githubMaxFileSize / 2
+	cur := lowerBound
+	increase := (upperBound - lowerBound) / 2
 
 	for increase > 0 {
 		mid := cur + increase
 
-		count, err := cache.CountResults(mid)
+		count, err := cache.CountResults(lowerBound, mid)
 		if err != nil {
 			return count, err
 		}
@@ -353,26 +356,24 @@ func lowerBoundFileCount(
 }
 
 func formatFilesizeRanges(cache cachedSearch, sizes []uint64) []string {
-	ranges := make([]string, 0, len(sizes)+1)
-
-	if len(sizes) > 0 {
-		ranges = append(ranges, cache.RequestString(
-			RangeLessThan{sizes[0] + 1},
-		))
+	n := len(sizes)
+	if n < 2 {
+		return []string{}
 	}
 
-	for i := 0; i < len(sizes)-1; i += 1 {
-		ranges = append(ranges, cache.RequestString(
-			RangeWithin{sizes[i] + 1, sizes[i+1]},
-		))
-
-		if i != len(sizes)-2 {
-			continue
-		}
-		ranges = append(ranges, cache.RequestString(
-			RangeGreaterThan{sizes[i+1]},
-		))
+	ranges := make([]string, 0, n-1)
+	ranges = append(ranges, cache.RequestString(RangeWithin{sizes[0], sizes[1]}))
+	for i := 1; i < n-1; i++ {
+		ranges = append(ranges, cache.RequestString(RangeWithin{sizes[i] + 1, sizes[i+1]}))
 	}
-
 	return ranges
 }
+
+func RangeSizes(s string) RangeWithin {
+	start := strings.Index(s, "+size:") + len("+size:")
+	end := strings.Index(s, "&")
+	ranges := strings.Split(s[start:end], "..")
+	lowerBound, _ := strconv.ParseUint(ranges[0], 10, 64)
+	upperBound, _ := strconv.ParseUint(ranges[1], 10, 64)
+	return RangeWithin{lowerBound, upperBound}
+}

api/internal/crawl/crawler/github/split_search_ranges_test.go

@@ -2,6 +2,7 @@ package github
 
 import (
 	"fmt"
+	"log"
 	"reflect"
 	"testing"
 )
@@ -10,8 +11,8 @@ type testCachedSearch struct {
 	cache map[uint64]uint64
 }
 
-func (c testCachedSearch) CountResults(upperBound uint64) (uint64, error) {
-	fmt.Printf("CountResults(%05x)\n", upperBound)
+func (c testCachedSearch) CountResults(lowerBound, upperBound uint64) (uint64, error) {
+	log.Printf("CountResults(%05x)\n", upperBound)
 	count, ok := c.cache[upperBound]
 	if !ok {
 		return count, fmt.Errorf("cache not set at %x", upperBound)
@@ -72,19 +73,29 @@ func TestRangeSplitting(t *testing.T) {
 		},
 	}
 
-	requests, err := FindRangesForRepoSearch(cache)
+	requests, err := FindRangesForRepoSearch(cache, 0, 524288)
 	if err != nil {
 		t.Errorf("Error while finding ranges: %v", err)
 	}
 	expected := []string{
-		"<107",      // cache.RequestString(RangeLessThan{0x6b}),
-		"107..128",  // cache.RequestString(RangeWithin{0x6b, 0x80}),
-		"129..256",  // cache.RequestString(RangeWithin{0x81, 0x100}),
-		"257..4095", // cache.RequestString(RangeWithin{0x101, 0xfff}),
-		">4095",     // cache.RequestString(RangeGreaterThan{0xfff}),
+		"0..106",       // cache.RequestString(RangeWithin{0x00, 0x6a}),
+		"107..128",     // cache.RequestString(RangeWithin{0x6b, 0x80}),
+		"129..256",     // cache.RequestString(RangeWithin{0x81, 0x100}),
+		"257..4095",    // cache.RequestString(RangeWithin{0x101, 0xfff}),
+		"4096..524288", // cache.RequestString(RangeWithin{0x1000, 0x80000}),
 	}
 
 	if !reflect.DeepEqual(requests, expected) {
 		t.Errorf("Expected requests (%v) to equal (%v)", requests, expected)
 	}
 }
+
+func TestRangeSizes(t *testing.T) {
+	s := "https://api.github.com/search/code?q=filename:kustomization.yaml+filename:kustomization.yml" +
+		"+filename:kustomization+size:2365..10000&order=desc&per_page=100&sort=indexed"
+	returnedResult := RangeSizes(s)
+	expectedResult := RangeWithin{uint64(2365), uint64(10000)}
+	if !reflect.DeepEqual(returnedResult, expectedResult) {
+		t.Errorf("RangeSizes expected (%v), got (%v)",expectedResult, returnedResult)
+	}
+}

api/internal/crawl/doc/doc.go

@@ -2,6 +2,8 @@ package doc
 
 import (
 	"fmt"
+	"log"
+	"path/filepath"
 	"sort"
 	"strings"
 
@@ -44,21 +46,36 @@ type KustomizationDocument struct {
 
 type set map[string]struct{}
 
+func (doc *KustomizationDocument) Copy() *KustomizationDocument {
+	return &KustomizationDocument{
+		Document:    *(doc.Document.Copy()),
+		Kinds:       doc.Kinds,
+		Identifiers: doc.Identifiers,
+		Values:      doc.Values,
+	}
+}
+
 func (doc *KustomizationDocument) String() string {
 	return fmt.Sprintf("%s %s %s %v %v %v len(identifiers):%v  len(values):%v",
 		doc.RepositoryURL, doc.FilePath, doc.DefaultBranch, doc.CreationTime,
 		doc.IsSame, doc.Kinds, len(doc.Identifiers), len(doc.Values))
 }
 
-// Implements the CrawlerDocument interface.
-func (doc *KustomizationDocument) GetResources() ([]*Document, error) {
-	isResource := true
-	for _, suffix := range konfig.RecognizedKustomizationFileNames() {
-		if strings.HasSuffix(doc.FilePath, "/"+suffix) {
-			isResource = false
+// IsKustomizationFile determines whether a file path is a kustomization file
+func IsKustomizationFile(path string) bool {
+	basename := filepath.Base(path)
+	for _, name := range konfig.RecognizedKustomizationFileNames() {
+		if basename == name {
+			return true
 		}
 	}
-	if isResource {
+	return false
+}
+
+// Implements the CrawlerDocument interface.
+func (doc *KustomizationDocument) GetResources(
+	includeResources, includeTransformers, includeGenerators bool) ([]*Document, error) {
+	if !IsKustomizationFile(doc.FilePath) {
 		return []*Document{}, nil
 	}
 
@@ -76,20 +93,44 @@ func (doc *KustomizationDocument) GetResources() ([]*Document, error) {
 	}
 	k.FixKustomizationPostUnmarshalling()
 
-	res := make([]*Document, 0, len(k.Resources))
-	for _, r := range k.Resources {
+	res := make([]*Document, 0)
+
+	if includeResources {
+		resourceDocs := doc.CollectDocuments(k.Resources, "resource")
+		res = append(res, resourceDocs...)
+	}
+
+	if includeGenerators {
+		generatorDocs := doc.CollectDocuments(k.Generators, "generator")
+		res = append(res, generatorDocs...)
+	}
+
+	if includeTransformers {
+		transformerDocs := doc.CollectDocuments(k.Transformers, "transformer")
+		res = append(res, transformerDocs...)
+	}
+
+	return res, nil
+}
+
+// CollectDocuments construct a Document for each path in paths, and return
+// a slice of Document pointers.
+func (doc *KustomizationDocument) CollectDocuments(
+	paths []string, fileType string) []*Document {
+	docs := make([]*Document, 0, len(paths))
+	for _, r := range paths {
 		if strings.TrimSpace(r) == "" {
 			continue
 		}
 		next, err := doc.Document.FromRelativePath(r)
 		if err != nil {
-			fmt.Printf("GetResources error: %v\n", err)
+			log.Printf("CollectDocuments error: %v\n", err)
 			continue
 		}
-		res = append(res, &next)
+		next.FileType = fileType
+		docs = append(docs, &next)
 	}
-
-	return res, nil
+	return docs
 }
 
 func (doc *KustomizationDocument) readBytes() ([]map[string]interface{}, error) {

api/internal/crawl/doc/doc_test.go

@@ -189,11 +189,13 @@ metadata:
 	}
 }
 
+type TestStructForGetResources struct {
+	doc       KustomizationDocument
+	resources []*Document
+}
+
 func TestGetResources(t *testing.T) {
-	tests := []struct {
-		doc       KustomizationDocument
-		resources []*Document
-	}{
+	tests := []TestStructForGetResources{
 		{
 			doc: KustomizationDocument{
 				Document: Document{
@@ -213,19 +215,27 @@ resources:
 				{
 					RepositoryURL: "sigs.k8s.io/kustomize",
 					FilePath:      "some/path/to/base",
+					FileType:      "resource",
+					User:          "sigs.k8s.io",
 				},
 				{
 					RepositoryURL: "sigs.k8s.io/kustomize",
 					FilePath:      "some/path/to/otherbase",
+					FileType:      "resource",
+					User:          "sigs.k8s.io",
 				},
 				{
 					RepositoryURL: "sigs.k8s.io/kustomize",
 					FilePath:      "some/path/to/kdir/file.yaml",
+					FileType:      "resource",
+					User:          "sigs.k8s.io",
 				},
 				{
 					RepositoryURL: "https://github.com/kubernetes-sigs/kustomize",
 					FilePath:      "examples/helloWorld",
 					DefaultBranch: "v3.1.0",
+					FileType:      "resource",
+					User:          "kubernetes-sigs",
 				},
 			},
 		},
@@ -248,9 +258,12 @@ resources:
 			resources: []*Document{},
 		},
 	}
+	runTest(t, tests, true, false, false)
+}
 
+func runTest(t *testing.T, tests []TestStructForGetResources, includeResources, includeTransformers, includeGenerators bool) {
 	for _, test := range tests {
-		res, err := test.doc.GetResources()
+		res, err := test.doc.GetResources(includeResources, includeTransformers, includeGenerators)
 		if err != nil {
 			t.Errorf("Unexpected error: %v\n", err)
 			continue
@@ -284,3 +297,83 @@ resources:
 		}
 	}
 }
+
+func TestGetResourcesAndGenerators(t *testing.T) {
+	tests := []TestStructForGetResources{
+		{
+			doc: KustomizationDocument{
+				Document: Document{
+					RepositoryURL: "sigs.k8s.io/kustomize",
+					FilePath:      "some/path/to/kdir/kustomization.yaml",
+					DocumentData: `
+resources:
+- file.yaml
+
+generators:
+- gen.yaml
+
+transformers:
+- tr.yaml
+`},
+			},
+			resources: []*Document{
+				{
+					RepositoryURL: "sigs.k8s.io/kustomize",
+					FilePath:      "some/path/to/kdir/gen.yaml",
+					FileType:      "generator",
+					User:          "sigs.k8s.io",
+				},
+				{
+					RepositoryURL: "sigs.k8s.io/kustomize",
+					FilePath:      "some/path/to/kdir/file.yaml",
+					FileType:      "resource",
+					User:          "sigs.k8s.io",
+				},
+			},
+		},
+	}
+	runTest(t, tests, true, false, true)
+}
+
+func TestGetResourcesAndGeneratorsAndTransformers(t *testing.T) {
+	tests := []TestStructForGetResources{
+		{
+			doc: KustomizationDocument{
+				Document: Document{
+					RepositoryURL: "sigs.k8s.io/kustomize",
+					FilePath:      "some/path/to/kdir/kustomization.yaml",
+					DocumentData: `
+resources:
+- file.yaml
+
+generators:
+- gen.yaml
+
+transformers:
+- tr.yaml
+`},
+			},
+			resources: []*Document{
+				{
+					RepositoryURL: "sigs.k8s.io/kustomize",
+					FilePath:      "some/path/to/kdir/tr.yaml",
+					FileType:      "transformer",
+					User:          "sigs.k8s.io",
+				},
+				{
+					RepositoryURL: "sigs.k8s.io/kustomize",
+					FilePath:      "some/path/to/kdir/gen.yaml",
+					FileType:      "generator",
+					User:          "sigs.k8s.io",
+				},
+				{
+					RepositoryURL: "sigs.k8s.io/kustomize",
+					FilePath:      "some/path/to/kdir/file.yaml",
+					FileType:      "resource",
+					User:          "sigs.k8s.io",
+				},
+			},
+		},
+	}
+	runTest(t, tests, true, true, true)
+}

api/internal/crawl/doc/docname.go

@@ -11,12 +11,18 @@ import (
 )
 
 type Document struct {
-	RepositoryURL string     `json:"repositoryUrl,omitempty"`
+	RepositoryURL string `json:"repositoryUrl,omitempty"`
+	// User makes it easy to aggregate data in the user level instead
+	// of the repository level
+	User          string     `json:"user,omitempty"`
 	FilePath      string     `json:"filePath,omitempty"`
 	DefaultBranch string     `json:"defaultBranch,omitempty"`
 	DocumentData  string     `json:"document,omitempty"`
 	CreationTime  *time.Time `json:"creationTime,omitempty"`
 	IsSame        bool       `json:"-"`
+	// FileType can be one of the following:
+	// "generator", "transformer", "resource", "".
+	FileType string `json:"fileType,omitempty"`
 }
 
 // Implements the CrawlerDocument interface.
@@ -27,14 +33,21 @@ func (doc *Document) GetDocument() *Document {
 func (doc *Document) Copy() *Document {
 	return &Document{
 		RepositoryURL: doc.RepositoryURL,
+		User:          doc.User,
 		FilePath:      doc.FilePath,
 		DefaultBranch: doc.DefaultBranch,
 		DocumentData:  doc.DocumentData,
 		CreationTime:  doc.CreationTime,
 		IsSame:        doc.IsSame,
+		FileType:      doc.FileType,
 	}
 }
 
+func (doc *Document) Path() string {
+	return fmt.Sprintf("repoURL: %s filePath: %s branch: %s",
+		doc.RepositoryURL, doc.FilePath, doc.DefaultBranch)
+}
+
 // Implements the CrawlerDocument interface.
 func (doc *Document) WasCached() bool {
 	return doc.IsSame
@@ -47,6 +60,7 @@ func (doc *Document) FromRelativePath(newFile string) (Document, error) {
 			RepositoryURL: repoSpec.Host + path.Clean(repoSpec.OrgRepo),
 			FilePath:      path.Clean(repoSpec.Path),
 			DefaultBranch: repoSpec.Ref,
+			User:          UserName(repoSpec.Host + path.Clean(repoSpec.OrgRepo)),
 		}, nil
 	}
 	// else document is probably relative path.
@@ -54,6 +68,7 @@ func (doc *Document) FromRelativePath(newFile string) (Document, error) {
 	ret := Document{
 		RepositoryURL: doc.RepositoryURL,
 		DefaultBranch: doc.DefaultBranch,
+		User:          UserName(doc.RepositoryURL),
 	}
 	ogDir, _ := path.Split(doc.FilePath)
 
@@ -78,13 +93,7 @@ func (doc *Document) ID() string {
 }
 
 func (doc *Document) RepositoryFullName() string {
-	url := strings.TrimRight(doc.RepositoryURL, "/")
-
-	gitPrefix := "git@github.com:"
-	if strings.HasPrefix(url, gitPrefix) {
-		url = url[len(gitPrefix):]
-	}
-
+	url := TrimUrl(doc.RepositoryURL)
 	sections := strings.Split(url, "/")
 	l := len(sections)
 	if l < 2 {
@@ -92,3 +101,24 @@ func (doc *Document) RepositoryFullName() string {
 	}
 	return path.Join(sections[l-2], sections[l-1])
 }
+
+// TrimUrl removes all the trailing slashes and the "git@github.com:" prefix (if exists).
+func TrimUrl(s string) string {
+	url := strings.TrimRight(s, "/")
+
+	gitPrefix := "git@github.com:"
+	if strings.HasPrefix(url, gitPrefix) {
+		url = url[len(gitPrefix):]
+	}
+	return url
+}
+
+func UserName(repositoryURL string) string {
+	url := TrimUrl(repositoryURL)
+	sections := strings.Split(url, "/")
+	l := len(sections)
+	if l < 2 {
+		return url
+	}
+	return sections[l-2]
+}

api/internal/crawl/doc/docname_test.go

@@ -28,6 +28,7 @@ func TestFromRelativePath(t *testing.T) {
 						RepositoryURL: "example.com/repo",
 						FilePath:      "path/to/other/file/resource.yaml",
 						DefaultBranch: "master",
+						User:          "example.com",
 					},
 				},
 				{
@@ -36,6 +37,7 @@ func TestFromRelativePath(t *testing.T) {
 						RepositoryURL: "example.com/repo",
 						FilePath:      "path/to/other/file/patch.yaml",
 						DefaultBranch: "master",
+						User:          "example.com",
 					},
 				},
 				{
@@ -44,6 +46,7 @@ func TestFromRelativePath(t *testing.T) {
 						RepositoryURL: "example.com/repo",
 						FilePath:      "path/to/file/service.yaml",
 						DefaultBranch: "master",
+						User:          "example.com",
 					},
 				},
 			},
@@ -65,7 +68,7 @@ func TestFromRelativePath(t *testing.T) {
 
 func TestDocument_RepositoryFullName(t *testing.T) {
 	testCases := []struct {
-		doc Document
+		doc                        Document
 		expectedRepositoryFullName string
 	}{
 		{
@@ -108,4 +111,40 @@ func TestDocument_RepositoryFullName(t *testing.T) {
 				returnedRepositoryFullName)
 		}
 	}
-}
+}
+
+func TestDocument_UserName(t *testing.T) {
+	testCases := []struct {
+		repositoryURL    string
+		expectedUserName string
+	}{
+		{
+			repositoryURL:    "https://github.com/user/repo",
+			expectedUserName: "user",
+		},
+		{
+			repositoryURL:    "https://github.com//user/repo////",
+			expectedUserName: "user",
+		},
+		{
+			repositoryURL:    "repo/",
+			expectedUserName: "repo",
+		},
+		{
+			repositoryURL:    "",
+			expectedUserName: "",
+		},
+		{
+			repositoryURL:    "git@github.com:user/repo",
+			expectedUserName: "user",
+		},
+	}
+
+	for _, tc := range testCases {
+		returnedUserName := UserName(tc.repositoryURL)
+		if returnedUserName != tc.expectedUserName {
+			t.Errorf("UserName expected %s, got %s",
+				tc.expectedUserName, returnedUserName)
+		}
+	}
+}

api/internal/crawl/doc/unique_doc.go

@@ -0,0 +1,36 @@
+package doc
+
+import (
+	"sigs.k8s.io/kustomize/api/internal/crawl/utils"
+)
+
+// UniqueDocuments make sure a Document with a given ID appears only once
+type UniqueDocuments struct {
+	docs   []*Document
+	docIDs utils.SeenMap
+}
+
+func NewUniqueDocuments() UniqueDocuments {
+	return UniqueDocuments{
+		docs:   []*Document{},
+		docIDs: utils.NewSeenMap(),
+	}
+}
+
+func (uds *UniqueDocuments) Add(d *Document) {
+	if uds.docIDs.Seen(d.ID()) {
+		return
+	}
+	uds.docs = append(uds.docs, d)
+	uds.docIDs.Set(d.ID(), "")
+}
+
+func (uds *UniqueDocuments) AddDocuments(docs []*Document) {
+	for _, d := range docs {
+		uds.Add(d)
+	}
+}
+
+func (uds *UniqueDocuments) Documents() []*Document {
+	return uds.docs
+}

api/internal/crawl/index/elasticsearch.go

@@ -20,12 +20,18 @@ const IndexConfig = `
 				"repositoryUrl": {
 					"type": "keyword"
 				},
+				"user": {
+					"type": "keyword"
+				},
 				"filePath": {
 					"type": "keyword"
 				},
 				"defaultBranch": {
 					"type": "keyword"
 				},
+				"fileType": {
+					"type": "keyword"
+				},
 				"document": {
 					"type": "text"
 				},
@@ -87,7 +93,7 @@ func (idx *index) responseErrorOrNil(info string, res *esapi.Response,
 
 	defer res.Body.Close()
 	if res.IsError() {
-		return fmt.Errorf("%s: %s", messageStart, res.String())
+		return fmt.Errorf("%s: %s [%d]", messageStart, res.String(), res.StatusCode)
 	}
 
 	if reader != nil {

api/internal/crawl/index/kustomize.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
+	"log"
 	"strings"
 	"time"
 
@@ -17,6 +18,7 @@ const (
 )
 
 type Mode int
+
 const (
 	InsertOrUpdate = iota
 	Delete
@@ -97,14 +99,14 @@ type KustomizeIndex struct {
 }
 
 // Create index reference to the index containing the kustomize documents.
-func NewKustomizeIndex(ctx context.Context) (*KustomizeIndex, error) {
-	idx, err := newIndex(ctx, "kustomize")
+func NewKustomizeIndex(ctx context.Context, indexName string) (*KustomizeIndex, error) {
+	idx, err := newIndex(ctx, indexName)
 	if err != nil {
 		return nil, err
 	}
 
 	indicesExistsOp := idx.client.Indices.Exists
-	resp, err := indicesExistsOp([]string{"kustomize"},
+	resp, err := indicesExistsOp([]string{indexName},
 		indicesExistsOp.WithContext(idx.ctx),
 		indicesExistsOp.WithPretty())
 	if err != nil {
@@ -112,9 +114,9 @@ func NewKustomizeIndex(ctx context.Context) (*KustomizeIndex, error) {
 	}
 
 	if resp.StatusCode == 200 {
-		fmt.Printf("The kustomize index already exists\n")
+		log.Printf("The %s index already exists", indexName)
 	} else {
-		fmt.Printf("Creating the kustomize index\n")
+		log.Printf("Creating the %s index\n", indexName)
 		if err := idx.CreateIndex([]byte(IndexConfig)); err != nil {
 			return nil, err
 		}
@@ -252,7 +254,7 @@ func (it *KustomizeIterator) Next() bool {
 	}
 
 	if it.err == nil {
-		fmt.Printf("updating scroll: %s\n", *it.scrollImpl.ScrollID)
+		log.Printf("updating scroll: %s\n", *it.scrollImpl.ScrollID)
 		it.err = it.update(*it.scrollImpl.ScrollID, reader)
 	}
 
@@ -268,7 +270,7 @@ func (it *KustomizeIterator) Value() KustomizeResult {
 	return it.scrollImpl
 }
 
-// Check if any errors have occured.
+// Check if any errors have occurred.
 func (it *KustomizeIterator) Err() error {
 	return it.err
 }
@@ -341,7 +343,7 @@ func (ki *KustomizeIndex) Search(query string,
 	if err != nil {
 		return nil, fmt.Errorf("failed to format query %s", query)
 	}
-	fmt.Printf("formated query: %s\n", data)
+	log.Printf("formated query: %s\n", data)
 
 	var kr ElasticKustomizeResult
 	err = ki.index.Search(data, opts.SearchOptions, func(results io.Reader) error {

api/internal/crawl/search_cmds/creationTime.md

@@ -1,6 +1,6 @@
 Find out the largest value of the `creationTime` field:
 ```
-curl -X POST "${ElasticSearchURL}:9200/kustomize/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+curl -s -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
 {
     "aggs" : {
         "max_creationTime" : { "max" : { "field" : "creationTime" } }
@@ -11,7 +11,7 @@ curl -X POST "${ElasticSearchURL}:9200/kustomize/_search?size=0&pretty" -H 'Cont
 
 Find out the smallest value of the `creationTime` field:
 ```
-curl -X POST "${ElasticSearchURL}:9200/kustomize/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+curl -s -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
 {
     "aggs" : {
         "min_creationTime" : { "min" : { "field" : "creationTime" } }
@@ -22,12 +22,12 @@ curl -X POST "${ElasticSearchURL}:9200/kustomize/_search?size=0&pretty" -H 'Cont
 
 Find out the smallest value of the `creationTime` field of all the kustomization files:
 ```
-curl -X POST "${ElasticSearchURL}:9200/kustomize/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+curl -s -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
 {
   "query": {
     "bool": {
       "filter": [
-        { "regexp": { "filePath": ".*/kustomization((.yaml)?|(.yml)?)/*" }}
+        { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }}
       ]
     }
   },
@@ -40,13 +40,58 @@ curl -X POST "${ElasticSearchURL}:9200/kustomize/_search?size=0&pretty" -H 'Cont
 
 Find out the smallest value of the `creationTime` field of all kustomize resource files:
 ```
-curl -X POST "${ElasticSearchURL}:9200/kustomize/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+curl -s -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
 {
   "query": {
     "bool": {
       "must_not": {
-        "regexp": { "filePath": ".*/kustomization((.yaml)?|(.yml)?)/*" }
-      }
+        "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }
+      }, 
+      "filter": [
+        { "regexp": { "fileType": "resource"  }}
+      ]
+    }
+  },
+  "aggs" : {
+    "min_creationTime" : { "min" : { "field" : "creationTime" } }
+  }
+}
+'
+```
+
+Find out the smallest value of the `creationTime` field of all kustomize generator files:
+```
+curl -s -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+  "query": {
+    "bool": {
+      "must_not": {
+        "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }
+      }, 
+      "filter": [
+        { "regexp": { "fileType": "generator"  }}
+      ]
+    }
+  },
+  "aggs" : {
+    "min_creationTime" : { "min" : { "field" : "creationTime" } }
+  }
+}
+'
+```
+
+Find out the smallest value of the `creationTime` field of all kustomize transformer files:
+```
+curl -s -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+  "query": {
+    "bool": {
+      "must_not": {
+        "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }
+      }, 
+      "filter": [
+        { "regexp": { "fileType": "transformer"  }}
+      ]
     }
   },
   "aggs" : {
@@ -58,7 +103,7 @@ curl -X POST "${ElasticSearchURL}:9200/kustomize/_search?size=0&pretty" -H 'Cont
 
 Query all the documents whose `creationTime` <= `2016-07-29T17:38:26.000Z`:
 ```
-curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type: application/json' -d'
+curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
 {
   "query": {
     "range": {
@@ -73,7 +118,7 @@ curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type
 
 Query all the documents whose `creationTime` falls within the specific range:
 ```
-curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type: application/json' -d'
+curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
 {
   "query": {
     "range": {
@@ -87,14 +132,38 @@ curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type
 '
 ```
 
+Query all the kustomization files whose `creationTime` falls within the specific range:
+```
+curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+  "size": 20,
+  "query": {
+    "bool": {
+      "filter": {
+        "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" }
+      },
+      "must": {
+        "range": {
+          "creationTime": {
+            "gte": "2017-09-24T15:49:57.000Z",
+            "lte": "2017-09-24T15:49:57.000Z"
+          }
+        }
+      }
+    }
+  }
+}
+'
+```
+
 Aggregate how many new kustomization files were added into Github each month:
 ```
-curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
 {
     "query": {
         "bool": {
             "filter": [
-                { "regexp": { "filePath": ".*/kustomization((.yaml)?|(.yml)?)/*" }}
+                { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" }}
             ]
         }
     },
@@ -112,12 +181,67 @@ curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?size=0&pretty" -H 'Conte
 
 Aggregate how many new kustomize resource files were added into Github each month:
 ```
-curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
 {
     "query": {
         "bool": {
             "must_not": [
-                { "regexp": { "filePath": ".*/kustomization((.yaml)?|(.yml)?)/*" }}
+                { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" }}
+            ],
+            "filter": [
+                { "regexp": { "fileType": "resource" }}
+            ]
+        }
+    },
+    "aggs" : {
+        "newFiles_over_time" : {
+            "date_histogram" : {
+                "field" : "creationTime",
+                "interval" : "month"
+            }
+        }
+    }
+}
+'
+```
+
+Aggregate how many new kustomize generator files were added into Github each month:
+```
+curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+    "query": {
+        "bool": {
+            "must_not": [
+                { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" }}
+            ],
+            "filter": [
+                { "regexp": { "fileType": "generator" }}
+            ]
+        }
+    },
+    "aggs" : {
+        "newFiles_over_time" : {
+            "date_histogram" : {
+                "field" : "creationTime",
+                "interval" : "month"
+            }
+        }
+    }
+}
+'
+```
+
+Aggregate how many new kustomize transformer files were added into Github each month:
+```
+curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+    "query": {
+        "bool": {
+            "must_not": [
+                { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" }}
+            ],
+            "filter": [
+                { "regexp": { "fileType": "transformer" }}
             ]
         }
     },
@@ -135,12 +259,12 @@ curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?size=0&pretty" -H 'Conte
 
 Aggregate how many new kustomization files were added into Github each year:
 ```
-curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
 {
     "query": {
         "bool": {
             "filter": [
-                { "regexp": { "filePath": ".*/kustomization((.yaml)?|(.yml)?)/*" }}
+                { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" }}
             ]
         }
     },
@@ -158,13 +282,16 @@ curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?size=0&pretty" -H 'Conte
 
 Aggregate how many new kustomize resource files were added into Github each year:
 ```
-curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
 {
     "query": {
         "bool": {
             "must_not": [
-                { "regexp": { "filePath": ".*/kustomization((.yaml)?|(.yml)?)/*" }}
-            ]
+                { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" }}
+            ],
+            "filter": [
+                { "regexp": { "fileType": "resource" }}
+            ]            
         }
     },
     "aggs" : {
@@ -177,4 +304,108 @@ curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?size=0&pretty" -H 'Conte
     }
 }
 '
+```
+
+Aggregate how many new kustomize generator files were added into Github each year:
+```
+curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+    "query": {
+        "bool": {
+            "must_not": [
+                { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" }}
+            ],
+            "filter": [
+                { "regexp": { "fileType": "generator" }}
+            ]            
+        }
+    },
+    "aggs" : {
+        "newFiles_over_time" : {
+            "date_histogram" : {
+                "field" : "creationTime",
+                "interval" : "year"
+            }
+        }
+    }
+}
+'
+```
+
+Aggregate how many new kustomize transformer files were added into Github each year:
+```
+curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+    "query": {
+        "bool": {
+            "must_not": [
+                { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" }}
+            ],
+            "filter": [
+                { "regexp": { "fileType": "transformer" }}
+            ]            
+        }
+    },
+    "aggs" : {
+        "newFiles_over_time" : {
+            "date_histogram" : {
+                "field" : "creationTime",
+                "interval" : "year"
+            }
+        }
+    }
+}
+'
+```
+
+Find the generator files created within the given time range:
+```
+curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+  "query": {
+    "bool": {
+      "filter": [
+       { "regexp": { "fileType": "generator" }}
+      ],
+      "must_not": {
+        "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }
+      },
+      "must": {
+        "range": {
+          "creationTime": {
+           "gte": "2019-04-26T16:40:02.000Z",
+           "lte": "2019-04-26T16:40:02.000Z"
+          }
+        }
+      }
+    }
+  }
+}
+'
+```
+
+Find the transformer files created within the given time range:
+```
+curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+  "query": {
+    "bool": {
+      "filter": [
+       { "regexp": { "fileType": "transformer" }}
+      ],
+      "must_not": {
+        "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }
+      },
+      "must": {
+        "range": {
+          "creationTime": {
+           "gte": "2019-04-26T16:40:02.000Z",
+           "lte": "2019-04-26T16:40:02.000Z"
+          }
+        }
+      }
+    }
+  }
+}
+'
 ```

api/internal/crawl/search_cmds/defaultBranch.md

@@ -1,6 +1,6 @@
 Count distinct values of the `defaultBranch` field:
 ```
-curl -X POST "${ElasticSearchURL}:9200/kustomize/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+curl -s -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
 {
     "aggs" : {
         "defaultBranch_count" : {
@@ -17,7 +17,7 @@ curl -X POST "${ElasticSearchURL}:9200/kustomize/_search?size=0&pretty" -H 'Cont
 List all the github branches where kustomization files and kustomize resource files live, 
 and how many kustomization files and kustomize resource files live in each branch:
 ```
-curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type: application/json' -d'
+curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
 {
     "aggs" : {
         "defaultBranch" : {

api/internal/crawl/search_cmds/fieldExistence.md

@@ -1,7 +1,7 @@
 Count the documents whose `document` field is empty (The reason why the `document` field
 of a document is empty is because of empty documents):
 ```
-curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type: application/json' -d'
+curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
 {
    "size": 10000,
     "query": {
@@ -19,7 +19,7 @@ curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type
 
 Find all the documents having the `creationTime` field set:
 ```
-curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type: application/json' -d'
+curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
 {
     "query": {
         "exists": {
@@ -32,7 +32,7 @@ curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type
 
 Find all the documents whose `creationTime` field is not set:
 ```
-curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type: application/json' -d'
+curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
 {
    "size": 10000,
     "query": {

api/internal/crawl/search_cmds/fileType.md

@@ -0,0 +1,301 @@
+Find all the documents having the `fileType` field set:
+```
+curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+    "query": {
+        "exists": {
+            "field": "fileType"
+        }
+    }
+}
+'
+```
+
+Find all the documents whose `fileType` field is not set:
+```
+curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+   "size": 10000,
+    "query": {
+        "bool": {
+            "must_not": {
+                "exists": {
+                    "field": "fileType"
+                }
+            }
+        }
+    }
+}
+'
+```
+
+Search for all the documents whose `fileType` field is `resource`:
+```
+curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+  "query": {
+    "bool": {
+      "filter": [
+       { "regexp": { "fileType": "resource" }}
+      ]
+    }
+  }
+}
+'
+```
+
+Search for all the kustomization files whose `fileType` field is `resource`:
+```
+curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+  "query": {
+    "bool": {
+      "filter": [
+       { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }}, 
+       { "regexp": { "fileType": "resource" }}
+      ]
+    }
+  }
+}
+'
+```
+
+Search for all the kustomize resource files:
+```
+curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+  "query": {
+    "bool": {
+      "filter": [
+       { "regexp": { "fileType": "resource" }}
+      ],
+      "must_not": {
+        "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }
+      }
+    }
+  }
+}
+'
+```
+
+Search all the kustomization files including a `generators` field:
+```
+curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+  "size": 10000,
+  "query": {
+    "bool": {
+      "must": {
+        "match" : {
+          "identifiers" : {
+            "query" : "generators"
+          }
+        }
+      },
+      "filter": {
+        "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }
+      }
+    }
+  }
+}
+'
+```
+
+Search for all the documents whose `fileType` field is `generator`:
+```
+curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+  "query": {
+    "bool": {
+      "filter": [
+       { "regexp": { "fileType": "generator" }}
+      ]
+    }
+  }
+}
+'
+```
+
+Search for all the kustomization files whose `fileType` field is `generator`:
+```
+curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+  "query": {
+    "bool": {
+      "filter": [
+       { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }}, 
+       { "regexp": { "fileType": "generator" }}
+      ]
+    }
+  }
+}
+'
+```
+
+Search for all the kustomize generator files:
+```
+curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+  "query": {
+    "bool": {
+      "filter": [
+       { "regexp": { "fileType": "generator" }}
+      ],
+      "must_not": {
+        "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }
+      }
+    }
+  }
+}
+'
+```
+
+Search all the kustomization files including a `transformers` field:
+```
+curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+  "size": 10000,
+  "query": {
+    "bool": {
+      "must": {
+        "match" : {
+          "identifiers" : {
+            "query" : "transformers"
+          }
+        }
+      },
+      "filter": {
+        "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }
+      }
+    }
+  }
+}
+'
+```
+
+Search for all the documents whose `fileType` field is `transformer`:
+```
+curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+  "query": {
+    "bool": {
+      "filter": [
+       { "regexp": { "fileType": "transformer" }}
+      ]
+    }
+  }
+}
+'
+```
+
+Search for all the kustomization files whose `fileType` field is `transformer`:
+```
+curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+  "query": {
+    "bool": {
+      "filter": [
+       { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }}, 
+       { "regexp": { "fileType": "transformer" }}
+      ]
+    }
+  }
+}
+'
+```
+
+Search for all the kustomize transformer files:
+```
+curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+  "query": {
+    "bool": {
+      "filter": [
+       { "regexp": { "fileType": "transformer" }}
+      ],
+      "must_not": {
+        "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }
+      }
+    }
+  }
+}
+'
+```
+
+Count distinct values of the `fileType` field:
+```
+curl -s -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+    "aggs" : {
+        "fileType_count" : {
+            "cardinality" : {
+                "field" : "fileType",
+                "precision_threshold": 40000
+            }
+        }
+    }
+}
+'
+```
+
+List all the values of the `fileType` field and the frequency of each value:
+```
+curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+    "aggs" : {
+        "fileType" : {
+            "terms" : {
+              "field" : "fileType"
+            }
+        }
+    }
+}
+'
+```
+
+
+For all the kustomization files in the index, list all the values of the
+`fileType` field and the frequency of each value:
+```
+curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+    "query": {
+        "bool": {
+             "filter": [
+                { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }}
+             ]
+        }
+    },
+    "aggs" : {
+        "fileType" : {
+            "terms" : {
+              "field" : "fileType"
+            }
+        }
+    }
+}
+'
+```
+
+For all the non-kustomization files in the index, list all the values of the
+`fileType` field and the frequency of each value:
+```
+curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+    "query": {
+        "bool": {
+            "must_not": {
+                "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }
+            }
+        }
+    },
+    "aggs" : {
+        "fileType" : {
+            "terms" : {
+              "field" : "fileType"
+            }
+        }
+    }
+}
+'
+```

api/internal/crawl/search_cmds/generator.md

@@ -0,0 +1,29 @@
+Find all the generator files whose `kinds` field includes `ChartRenderer`, and
+only output certain fields of each document:
+```
+curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+  "size": 200,
+  "_source": {
+    "includes": ["kinds", "repositoryUrl", "defaultBranch", "filePath"]
+  },
+  "query": {
+    "bool": {
+      "filter": [
+       { "regexp": { "fileType": "generator" }}
+      ],
+      "must_not": {
+        "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }
+      },
+      "must": {
+        "match" : {
+          "kinds" : {
+            "query" : "ChartRenderer"
+          }
+        }
+      }
+    }
+  }
+}
+'
+```

api/internal/crawl/search_cmds/id.md

@@ -0,0 +1,12 @@
+Find the document with the given `_id`:
+```
+curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+  "query": {
+    "terms": {
+      "_id": [ "b3a03f3327841617db696e2d6abc30e1a1bd653f1a2bbce05637f7dcae1a43f7" ] 
+    }
+  }
+}
+'
+```

api/internal/crawl/search_cmds/keyword_search.md

@@ -1,7 +1,7 @@
 Count the documents in the index whose `repositoryUrl` field starts with
 `https://github.com/`:
 ```
-curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type: application/json' -d'
+curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
 {
   "query": {
     "bool": {
@@ -17,7 +17,7 @@ curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type
 Count the documents in the index whose `repositoryUrl` field does not start with
 `https://github.com/`:
 ```
-curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type: application/json' -d'
+curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
 {
   "query": {
     "bool": {
@@ -33,7 +33,7 @@ curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type
 Search all the documents matching the given `repositoryUrl` and `filePath`, and return 
 a version for each search hit:
 ```
-curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type: application/json' -d'
+curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
 {
   "size": 10000,
   "version": true,
@@ -52,12 +52,28 @@ curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type
 Search all the documents whose filePath ends with one of these following three filenames:
 `kustomization.yaml`, `kustomization.yml`, `kustomization`:
 ```
-curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type: application/json' -d'
+curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
 {
   "query": {
     "bool": {
       "filter": [
-       { "regexp": { "filePath": ".*/kustomization((.yaml)?|(.yml)?)/*" }}
+       { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" }}
+      ]
+    }
+  }
+}
+'
+```
+
+Search all the documents whose filePath does not end with any of these following
+three filenames: `kustomization.yaml`, `kustomization.yml`, `kustomization`:
+```
+curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+  "query": {
+    "bool": {
+      "must_not": [
+       { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" }}
       ]
     }
   }

api/internal/crawl/search_cmds/misc.md

@@ -1,19 +1,32 @@
 Check the health status of an ElasticSearch cluster:
 ```
-curl -X GET "${ElasticSearchURL}:9200/_cat/health?v&pretty"
+curl -s -X GET "${ElasticSearchURL}:9200/_cat/health?v&pretty"
 ```
 
 Check the indices in an ElasticSearch cluster:
 ```
-curl "${ElasticSearchURL}:9200/_cat/indices?v"
+curl -s "${ElasticSearchURL}:9200/_cat/indices?v"
 ```
 
 Get the mapping of the index:
 ```
-curl -X GET "${ElasticSearchURL}:9200/kustomize/_mapping?pretty"
+curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_mapping?pretty"
 ```
 
 Delete the kustomize index from the ElasticSearch cluster (**Use this command with caution**):
 ```
-curl -X DELETE "${ElasticSearchURL}:9200/kustomize?pretty"
+curl -s -X DELETE "${ElasticSearchURL}:9200/${INDEXNAME}?pretty"
+```
+
+Add a new field into an existing index.
+```
+curl -s -X PUT "${ElasticSearchURL}:9200/${INDEXNAME}/_mapping/_doc?pretty" -H 'Content-Type: application/json' -d'
+{
+  "properties": {
+    "fileType": {
+      "type": "keyword"
+    }
+  }
+}
+'
 ```

api/internal/crawl/search_cmds/repositoryUrl.md

@@ -1,6 +1,6 @@
 Count distinct values of the `repositoryUrl` field:
 ```
-curl -X POST "${ElasticSearchURL}:9200/kustomize/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+curl -s -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
 {
     "aggs" : {
         "repositoryUrl_count" : {
@@ -16,12 +16,12 @@ curl -X POST "${ElasticSearchURL}:9200/kustomize/_search?size=0&pretty" -H 'Cont
 
 Count how many Github repositories include kustomization files:
 ```
-curl -X POST "${ElasticSearchURL}:9200/kustomize/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+curl -s -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
 {
     "query": {
         "bool": {
              "filter": [
-                { "regexp": { "filePath": ".*/kustomization((.yaml)?|(.yml)?)/*" }}
+                { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" }}
              ]
         }
     },
@@ -37,16 +37,143 @@ curl -X POST "${ElasticSearchURL}:9200/kustomize/_search?size=0&pretty" -H 'Cont
 '
 ```
 
-Count how many Github repositories include kustomize resource files:
+Count distinct values of the `repositoryUrl` field for all the kustomize resource files in the index:
 ```
-curl -X POST "${ElasticSearchURL}:9200/kustomize/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+curl -s -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
 {
     "query": {
         "bool": {
             "must_not": {
-                "regexp": { "filePath": ".*/kustomization((.yaml)?|(.yml)?)/*" }
+                "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }
+            },
+            "filter": [
+                { "regexp": { "fileType": "resource" }}
+            ]
+        }
+    },
+    "aggs" : {
+        "repositoryUrl_count" : {
+            "cardinality" : {
+                "field" : "repositoryUrl",
+                "precision_threshold": 40000
             }
         }
+    }
+}
+'
+```
+
+Count distinct values of the `repositoryUrl` field for all the kustomize generator files in the index:
+```
+curl -s -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+    "query": {
+        "bool": {
+            "must_not": {
+                "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }
+            },
+            "filter": [
+                { "regexp": { "fileType": "generator" }}
+            ]
+        }
+    },
+    "aggs" : {
+        "repositoryUrl_count" : {
+            "cardinality" : {
+                "field" : "repositoryUrl",
+                "precision_threshold": 40000
+            }
+        }
+    }
+}
+'
+```
+
+Count distinct values of the `repositoryUrl` field for all the kustomize transformer files in the index:
+```
+curl -s -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+    "query": {
+        "bool": {
+            "must_not": {
+                "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }
+            },
+            "filter": [
+                { "regexp": { "fileType": "transformer" }}
+            ]
+        }
+    },
+    "aggs" : {
+        "repositoryUrl_count" : {
+            "cardinality" : {
+                "field" : "repositoryUrl",
+                "precision_threshold": 40000
+            }
+        }
+    }
+}
+'
+```
+
+Count distinct values of the `repositoryUrl` field for all the kustomize resource dirs in the index:
+```
+curl -s -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+    "query": {
+        "bool": {
+            "filter": [
+                { "regexp": { "fileType": "resource" }},
+                { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }}
+            ]
+        }
+    },
+    "aggs" : {
+        "repositoryUrl_count" : {
+            "cardinality" : {
+                "field" : "repositoryUrl",
+                "precision_threshold": 40000
+            }
+        }
+    }
+}
+'
+```
+
+Count distinct values of the `repositoryUrl` field for all the kustomize generator dirs in the index:
+```
+curl -s -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+    "query": {
+        "bool": {
+            "filter": [
+                { "regexp": { "fileType": "generator" }},
+                { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }}
+            ]
+        }
+    },
+    "aggs" : {
+        "repositoryUrl_count" : {
+            "cardinality" : {
+                "field" : "repositoryUrl",
+                "precision_threshold": 40000
+            }
+        }
+    }
+}
+'
+```
+
+Count distinct values of the `repositoryUrl` field for all the kustomize transformer dirs in the index:
+```
+curl -s -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+    "query": {
+        "bool": {
+            "filter": [
+                { "regexp": { "fileType": "transformer" }},
+                { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }}
+            ]
+        }
     },
     "aggs" : {
         "repositoryUrl_count" : {
@@ -64,7 +191,7 @@ List all the github repositories including kustomization files and kustomize res
 and how many kustomization files and kustomize resource files each github repository includes
 (the github repository including the most kustomization files is listed first):
 ```
-curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
 {
     "aggs" : {
         "repositoryUrl" : {
@@ -80,12 +207,12 @@ curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?size=0&pretty" -H 'Conte
 
 List the top 20 Github repositories including the most amount of kustomization files:
 ```
-curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
 {
     "query": {
         "bool": {
              "filter": [
-                { "regexp": { "filePath": ".*/kustomization((.yaml)?|(.yml)?)/*" }}
+                { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" }}
              ]
         }
     },
@@ -103,13 +230,16 @@ curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?size=0&pretty" -H 'Conte
 
 List the top 20 Github repositories including the most amount of kustomize resource files:
 ```
-curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
 {
     "query": {
         "bool": {
             "must_not": {
-                "regexp": { "filePath": ".*/kustomization((.yaml)?|(.yml)?)/*" }
-            }
+                "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" }
+            },
+            "filter": [
+                { "regexp": { "fileType": "resource" }}
+            ]
         }
     },
     "aggs" : {

api/internal/crawl/search_cmds/snapshot.md

@@ -0,0 +1,29 @@
+Retrieve information about all registered snapshot repositories:
+```
+curl -s -X GET "${ElasticSearchURL}:9200/_snapshot?pretty"
+```
+
+Retrieve information about a given snapshot repository, `kustomize-backup`:
+```
+curl -s -X GET "${ElasticSearchURL}:9200/_snapshot/kustomize-backup?pretty"
+```
+
+Verify a snapshot repository, `kustomize-backup`, manually:
+```
+curl -s -X POST "${ElasticSearchURL}:9200/_snapshot/kustomize-backup/_verify?pretty"
+```
+
+List all the snapshots in a given snapshot repository:
+```
+curl -s -X GET "${ElasticSearchURL}:9200/_cat/snapshots/kustomize-backup?v&s=id&pretty"
+```
+
+Retrieve a summary information about a given snapshot:
+```
+curl -s -X GET "${ElasticSearchURL}:9200/_snapshot/kustomize-backup/kustomize-snapshot?pretty"
+```
+
+Retrieve a detailed information about a given snapshot:
+```
+curl -s -X GET "${ElasticSearchURL}:9200/_snapshot/kustomize-backup/kustomize-snapshot/_status?pretty"
+```

api/internal/crawl/search_cmds/text_search.md

@@ -1,6 +1,6 @@
 Search for all the kustomize resource files including a Deployment object:
 ```
-curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type: application/json' -d'
+curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
 {
   "query": {
     "match" : {
@@ -16,7 +16,7 @@ curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type
 Search for all the kustomize resource files including a Deployment object, but only
 including the `kinds` field in the result:
 ```
-curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type: application/json' -d'
+curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
 {
   "_source": {
     "includes": ["kinds"]
@@ -35,7 +35,7 @@ curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type
 Search for all the kustomize resource files including both a Deployment object and 
 a Service object:
 ```
-curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type: application/json' -d'
+curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
 {
   "query": {
     "match" : {
@@ -52,7 +52,7 @@ curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type
 Count the number of documents including Deployment and the number of documents 
 including Service:
 ```
-curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type: application/json' -d'
+curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
 {
   "size": 0,
   "aggs" : {
@@ -71,7 +71,7 @@ curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type
 
 Search for all the kustomization files involving CRDs:
 ```
-curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type: application/json' -d'
+curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
 {
   "size": 10000,
   "query": {
@@ -87,7 +87,7 @@ curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type
 
 Search for all the kustomization files defining configMapGenerator:
 ```
-curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type: application/json' -d'
+curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
 {
   "size": 10000,
   "query": {
@@ -103,7 +103,7 @@ curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type
 
 Search for all the documents having a `kind` field:
 ```
-curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type: application/json' -d'
+curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
 {
   "query": {
     "bool": {
@@ -118,7 +118,7 @@ curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type
 
 Search for all the kuostmization files having a `kind` field:
 ```
-curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type: application/json' -d'
+curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
 {
   "query": {
     "bool": {
@@ -134,7 +134,7 @@ curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type
 
 Search for all the kustomization files defining the `generatorOptions:disableNameSuffixHash` feature:
 ```
-curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type: application/json' -d'
+curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
 {
   "query": {
     "match" : {

api/internal/crawl/search_cmds/transformer.md

@@ -0,0 +1,29 @@
+Find all the transformer files whose `kinds` field includes `HelmValues`, and
+only output certain fields of each document:
+```
+curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+  "size": 200,
+  "_source": {
+    "includes": ["kinds", "repositoryUrl", "defaultBranch", "filePath"]
+  },
+  "query": {
+    "bool": {
+      "filter": [
+       { "regexp": { "fileType": "transformer" }}
+      ],
+      "must_not": {
+        "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }
+      },
+      "must": {
+        "match" : {
+          "kinds" : {
+            "query" : "HelmValues"
+          }
+        }
+      }
+    }
+  }
+}
+'
+```

api/internal/crawl/search_cmds/user.md

@@ -0,0 +1,380 @@
+Find all the documents having the `user` field set:
+```
+curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+    "query": {
+        "exists": {
+            "field": "user"
+        }
+    }
+}
+'
+```
+
+Find all the documents whose `user` field is not set:
+```
+curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+   "size": 10000,
+    "query": {
+        "bool": {
+            "must_not": {
+                "exists": {
+                    "field": "user"
+                }
+            }
+        }
+    }
+}
+'
+```
+
+Search for all the documents whose `user` field is `kubernetes-sigs`:
+```
+curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
+{
+  "query": {
+    "bool": {
+      "filter": [
+       { "regexp": { "user": "kubernetes-sigs" }}
+      ]
+    }
+  }
+}
+'
+```
+
+Count distinct values of the `user` field:
+```
+curl -s -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+    "aggs" : {
+        "user_count" : {
+            "cardinality" : {
+                "field" : "user",
+                "precision_threshold": 40000
+            }
+        }
+    }
+}
+'
+```
+
+List all the values of the `user` field and the frequency of each value:
+```
+curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+    "aggs" : {
+        "user" : {
+            "terms" : {
+                "field" : "user",
+                "size" : 20
+            }
+        }
+    }
+}
+'
+```
+
+Count distinct values of the `user` field for all the kustomization files in the index:
+```
+curl -s -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+    "query": {
+        "bool": {
+             "filter": [
+                { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }}
+             ]
+        }
+    },
+    "aggs" : {
+        "user_count" : {
+            "cardinality" : {
+                "field" : "user",
+                "precision_threshold": 40000
+            }
+        }
+    }
+}
+'
+```
+
+For all the kustomization files in the index, list all the values of the
+`user` field and the frequency of each value:
+```
+curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+    "query": {
+        "bool": {
+             "filter": [
+                { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }}
+             ]
+        }
+    },
+    "aggs" : {
+        "user" : {
+            "terms" : {
+                "field" : "user",
+                "size": 20
+            }
+        }
+    }
+}
+'
+```
+
+Count distinct values of the `user` field for all the kustomize resource files in the index:
+```
+curl -s -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+    "query": {
+        "bool": {
+            "must_not": {
+                "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }
+            },
+            "filter": [
+                { "regexp": { "fileType": "resource" }}
+            ]
+        }
+    },
+    "aggs" : {
+        "user_count" : {
+            "cardinality" : {
+                "field" : "user",
+                "precision_threshold": 40000
+            }
+        }
+    }
+}
+'
+```
+
+For all the kustomize resource files in the index, list all the values of the
+`user` field and the frequency of each value:
+```
+curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+    "query": {
+        "bool": {
+            "must_not": {
+                "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }
+            },
+            "filter": [
+                { "regexp": { "fileType": "resource" }}
+            ]
+        }
+    },
+    "aggs" : {
+        "user" : {
+            "terms" : {
+                "field" : "user",
+                "size": 20
+            }
+        }
+    }
+}
+'
+```
+
+Count distinct values of the `user` field for all the kustomize generator files in the index:
+```
+curl -s -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+    "query": {
+        "bool": {
+            "must_not": {
+                "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }
+            },
+            "filter": [
+                { "regexp": { "fileType": "generator" }}
+            ]
+        }
+    },
+    "aggs" : {
+        "user_count" : {
+            "cardinality" : {
+                "field" : "user",
+                "precision_threshold": 40000
+            }
+        }
+    }
+}
+'
+```
+
+For all the kustomize generator files in the index, list all the values of the
+`user` field and the frequency of each value:
+```
+curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+    "query": {
+        "bool": {
+            "must_not": {
+                "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }
+            },
+            "filter": [
+                { "regexp": { "fileType": "generator" }}
+            ]
+        }
+    },
+    "aggs" : {
+        "user" : {
+            "terms" : {
+              "field" : "user",
+              "size": 20  
+            }
+        }
+    }
+}
+'
+```
+
+Count distinct values of the `user` field for all the kustomize transformer files in the index:
+```
+curl -s -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+    "query": {
+        "bool": {
+            "must_not": {
+                "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }
+            },
+            "filter": [
+                { "regexp": { "fileType": "transformer" }}
+            ]
+        }
+    },
+    "aggs" : {
+        "user_count" : {
+            "cardinality" : {
+                "field" : "user",
+                "precision_threshold": 40000
+            }
+        }
+    }
+}
+'
+```
+
+For all the kustomize transformer files in the index, list all the values of the
+`user` field and the frequency of each value:
+```
+curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+    "query": {
+        "bool": {
+            "must_not": {
+                "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }
+            },
+            "filter": [
+                { "regexp": { "fileType": "transformer" }}
+            ]
+        }
+    },
+    "aggs" : {
+        "user" : {
+            "terms" : {
+                "field" : "user",
+                "size": 20
+            }
+        }
+    }
+}
+'
+```
+
+Count distinct values of the `user` field for all the kustomize generator dirs in the index:
+```
+curl -s -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+    "query": {
+        "bool": {
+            "filter": [
+                { "regexp": { "fileType": "generator" }},
+                { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }}
+            ]
+        }
+    },
+    "aggs" : {
+        "user_count" : {
+            "cardinality" : {
+                "field" : "user",
+                "precision_threshold": 40000
+            }
+        }
+    }
+}
+'
+```
+
+For all the kustomize generator dirs in the index, list all the values of the
+`user` field and the frequency of each value:
+```
+curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+    "query": {
+        "bool": {
+            "filter": [
+                { "regexp": { "fileType": "generator" }},
+                { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }}
+            ]
+        }
+    },
+    "aggs" : {
+        "user" : {
+            "terms" : {
+              "field" : "user",
+              "size": 20  
+            }
+        }
+    }
+}
+'
+```
+
+Count distinct values of the `user` field for all the kustomize transformer dirs in the index:
+```
+curl -s -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+    "query": {
+        "bool": {
+            "filter": [
+                { "regexp": { "fileType": "transformer" }},
+                { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }}
+            ]
+        }
+    },
+    "aggs" : {
+        "user_count" : {
+            "cardinality" : {
+                "field" : "user",
+                "precision_threshold": 40000
+            }
+        }
+    }
+}
+'
+```
+
+For all the kustomize transformer dirs in the index, list all the values of the
+`user` field and the frequency of each value:
+```
+curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
+{
+    "query": {
+        "bool": {
+            "filter": [
+                { "regexp": { "fileType": "transformer" }},
+                { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*"  }}
+            ]
+        }
+    },
+    "aggs" : {
+        "user" : {
+            "terms" : {
+              "field" : "user"
+            }
+        }
+    }
+}
+'
+```

api/internal/crawl/utils/utils.go

@@ -0,0 +1,21 @@
+package utils
+
+type SeenMap map[string]string
+
+func (seen SeenMap) Seen(item string) bool {
+	_, ok := seen[item]
+	return ok
+}
+
+func (seen SeenMap) Set(k, v string) {
+	seen[k] = v
+}
+
+// The caller should make sure that key is in the map.
+func (seen SeenMap) Value(k string) string {
+	return seen[k]
+}
+
+func NewSeenMap() SeenMap {
+	return make(map[string]string)
+}

api/internal/git/cloner.go

@@ -27,90 +27,28 @@ func ClonerUsingGitExec(repoSpec *RepoSpec) error {
 	if err != nil {
 		return err
 	}
+
+	if repoSpec.Ref == "" {
+		repoSpec.Ref = "master"
+	}
 	cmd := exec.Command(
 		gitProgram,
-		"init",
+		"clone",
+		"--depth=1",
+		repoSpec.CloneSpec(),
+		"-b",
+		repoSpec.Ref,
 		repoSpec.Dir.String())
 	var out bytes.Buffer
 	cmd.Stdout = &out
 	cmd.Stderr = &out
 	err = cmd.Run()
 	if err != nil {
-		log.Printf("Error initializing empty git repo: %s", out.String())
+		log.Printf("Error cloning git repo: %s", out.String())
 		return errors.Wrapf(
 			err,
-			"trouble initializing empty git repo in %s",
-			repoSpec.Dir.String())
-	}
-
-	cmd = exec.Command(
-		gitProgram,
-		"remote",
-		"add",
-		"origin",
-		repoSpec.CloneSpec())
-	cmd.Stdout = &out
-	cmd.Stderr = &out
-	cmd.Dir = repoSpec.Dir.String()
-	err = cmd.Run()
-	if err != nil {
-		log.Printf("Error setting git remote: %s", out.String())
-		return errors.Wrapf(
-			err,
-			"trouble adding remote %s",
-			repoSpec.CloneSpec())
-	}
-	if repoSpec.Ref == "" {
-		repoSpec.Ref = "master"
-	}
-	cmd = exec.Command(
-		gitProgram,
-		"fetch",
-		"--depth=1",
-		"origin",
-		repoSpec.Ref)
-	cmd.Stdout = &out
-	cmd.Stderr = &out
-	cmd.Dir = repoSpec.Dir.String()
-	err = cmd.Run()
-	if err != nil {
-		cmd = exec.Command(
-			gitProgram,
-			"pull",
-			"origin",
-			"master")
-		var out bytes.Buffer
-		cmd.Stdout = &out
-		cmd.Dir = repoSpec.Dir.String()
-		err := cmd.Run()
-		if err != nil {
-			return errors.Wrapf(err, "trouble pulling %s", repoSpec.OrgRepo)
-		}
-		if repoSpec.Ref == "" {
-			repoSpec.Ref = "master"
-		}
-		cmd = exec.Command(gitProgram, "checkout", repoSpec.Ref)
-		cmd.Dir = repoSpec.Dir.String()
-		err = cmd.Run()
-		if err != nil {
-			return errors.Wrapf(
-				err, "trouble checking out href %s", repoSpec.Ref)
-		}
-	}
-
-	cmd = exec.Command(
-		gitProgram,
-		"reset",
-		"--hard",
-		"FETCH_HEAD")
-	cmd.Stdout = &out
-	cmd.Stderr = &out
-	cmd.Dir = repoSpec.Dir.String()
-	err = cmd.Run()
-	if err != nil {
-		log.Printf("Error performing git reset: %s", out.String())
-		return errors.Wrapf(
-			err, "trouble hard resetting empty repository to %s", repoSpec.Ref)
+			"trouble cloning git repo %v in %s",
+			repoSpec.CloneSpec(), repoSpec.Dir.String())
 	}
 
 	cmd = exec.Command(
@@ -120,10 +58,11 @@ func ClonerUsingGitExec(repoSpec *RepoSpec) error {
 		"--init",
 		"--recursive")
 	cmd.Stdout = &out
+	cmd.Stderr = &out
 	cmd.Dir = repoSpec.Dir.String()
 	err = cmd.Run()
 	if err != nil {
-		return errors.Wrapf(err, "trouble fetching submodules for %s", repoSpec.Ref)
+		return errors.Wrapf(err, "trouble fetching submodules for %s", repoSpec.CloneSpec())
 	}
 
 	return nil

api/internal/k8sdeps/configmapandsecret/configmapfactory.go

@@ -38,10 +38,7 @@ func (f *Factory) MakeConfigMap(
 			return nil, errors.Wrap(err, "trouble mapping")
 		}
 	}
-	if f.options != nil {
-		cm.SetLabels(f.options.Labels)
-		cm.SetAnnotations(f.options.Annotations)
-	}
+	f.setLabelsAndAnnnotations(cm, args.GeneratorOptions)
 	return cm, nil
 }
 

api/internal/k8sdeps/configmapandsecret/configmapfactory_test.go

@@ -53,7 +53,7 @@ BAR=baz
 	}
 }
 
-func makeLiteralConfigMap(name string) *corev1.ConfigMap {
+func makeLiteralConfigMap(name string, labels, annotations map[string]string) *corev1.ConfigMap {
 	cm := &corev1.ConfigMap{
 		TypeMeta: metav1.TypeMeta{
 			APIVersion: "v1",
@@ -69,7 +69,12 @@ func makeLiteralConfigMap(name string) *corev1.ConfigMap {
 			"d": "true",
 		},
 	}
-	cm.SetLabels(map[string]string{"foo": "bar"})
+	if labels != nil {
+		cm.SetLabels(labels)
+	}
+	if annotations != nil {
+		cm.SetAnnotations(annotations)
+	}
 	return cm
 }
 
@@ -128,7 +133,49 @@ func TestConstructConfigMap(t *testing.T) {
 					"foo": "bar",
 				},
 			},
-			expected: makeLiteralConfigMap("literalConfigMap"),
+			expected: makeLiteralConfigMap("literalConfigMap", map[string]string{
+				"foo": "bar",
+			}, nil),
+		},
+		{
+			description: "construct config map from literal with GeneratorOptions in ConfigMapArgs",
+			input: types.ConfigMapArgs{
+				GeneratorArgs: types.GeneratorArgs{
+					Name: "literalConfigMap",
+					KvPairSources: types.KvPairSources{
+						LiteralSources: []string{"a=x", "b=y", "c=\"Hello World\"", "d='true'"},
+					},
+					GeneratorOptions: &types.GeneratorOptions{
+						Labels: map[string]string{
+							"foo": "changed",
+							"cat": "dog",
+						},
+						Annotations: map[string]string{
+							"foo": "changed",
+							"cat": "dog",
+						},
+					},
+				},
+			},
+			options: &types.GeneratorOptions{
+				Labels: map[string]string{
+					"foo": "bar",
+				},
+				Annotations: map[string]string{
+					"foo": "bar",
+				},
+			},
+			// GeneratorOptions from the ConfigMapArgs take precedence over the
+			// factory level GeneratorOptions and should overwrite
+			// labels/annotations set in the factory level if there are common
+			// labels/annotations
+			expected: makeLiteralConfigMap("literalConfigMap", map[string]string{
+				"foo": "changed",
+				"cat": "dog",
+			}, map[string]string{
+				"foo": "changed",
+				"cat": "dog",
+			}),
 		},
 	}
 

api/internal/k8sdeps/configmapandsecret/factory.go

@@ -4,6 +4,7 @@
 package configmapandsecret
 
 import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"sigs.k8s.io/kustomize/api/ifc"
 	"sigs.k8s.io/kustomize/api/types"
 )
@@ -20,4 +21,35 @@ func NewFactory(
 	return &Factory{kvLdr: kvLdr, options: o}
 }
 
+// setLabelsAndAnnnotations will take the labels and annotations from
+// global GeneratorOptions and resource level GeneratorOptions and merge them
+// with the resource level taking precedence, and then set them on the provided
+// obj.
+func (f *Factory) setLabelsAndAnnnotations(obj metav1.Object, opts *types.GeneratorOptions) {
+	labels := make(map[string]string)
+	annotations := make(map[string]string)
+	if f.options != nil {
+		for k, v := range f.options.Labels {
+			labels[k] = v
+		}
+		for k, v := range f.options.Annotations {
+			annotations[k] = v
+		}
+	}
+	if opts != nil {
+		for k, v := range opts.Labels {
+			labels[k] = v
+		}
+		for k, v := range opts.Annotations {
+			annotations[k] = v
+		}
+	}
+	if len(labels) != 0 {
+		obj.SetLabels(labels)
+	}
+	if len(annotations) != 0 {
+		obj.SetAnnotations(annotations)
+	}
+}
+
 const keyExistsErrorMsg = "cannot add key %s, another key by that name already exists: %v"

api/internal/k8sdeps/configmapandsecret/secretfactory.go

@@ -39,10 +39,7 @@ func (f *Factory) MakeSecret(
 			return nil, err
 		}
 	}
-	if f.options != nil {
-		s.SetLabels(f.options.Labels)
-		s.SetAnnotations(f.options.Annotations)
-	}
+	f.setLabelsAndAnnnotations(s, args.GeneratorOptions)
 	return s, nil
 }
 

api/internal/k8sdeps/configmapandsecret/secretfactory_test.go

@@ -51,7 +51,7 @@ BAR=baz
 	}
 }
 
-func makeLiteralSecret(name string) *corev1.Secret {
+func makeLiteralSecret(name string, labels, annotations map[string]string) *corev1.Secret {
 	s := &corev1.Secret{
 		TypeMeta: metav1.TypeMeta{
 			APIVersion: "v1",
@@ -66,7 +66,12 @@ func makeLiteralSecret(name string) *corev1.Secret {
 		},
 		Type: "Opaque",
 	}
-	s.SetLabels(map[string]string{"foo": "bar"})
+	if labels != nil {
+		s.SetLabels(labels)
+	}
+	if annotations != nil {
+		s.SetAnnotations(annotations)
+	}
 	return s
 }
 
@@ -120,7 +125,49 @@ func TestConstructSecret(t *testing.T) {
 					"foo": "bar",
 				},
 			},
-			expected: makeLiteralSecret("literalSecret"),
+			expected: makeLiteralSecret("literalSecret", map[string]string{
+				"foo": "bar",
+			}, nil),
+		},
+		{
+			description: "construct secret from literal with GeneratorOptions in SecretArgs",
+			input: types.SecretArgs{
+				GeneratorArgs: types.GeneratorArgs{
+					Name: "literalSecret",
+					KvPairSources: types.KvPairSources{
+						LiteralSources: []string{"a=x", "b=y"},
+					},
+					GeneratorOptions: &types.GeneratorOptions{
+						Labels: map[string]string{
+							"foo": "changed",
+							"cat": "dog",
+						},
+						Annotations: map[string]string{
+							"foo": "changed",
+							"cat": "dog",
+						},
+					},
+				},
+			},
+			options: &types.GeneratorOptions{
+				Labels: map[string]string{
+					"foo": "bar",
+				},
+				Annotations: map[string]string{
+					"foo": "bar",
+				},
+			},
+			// GeneratorOptions from the SecretArgs take precedence over the
+			// factory level GeneratorOptions and should overwrite
+			// labels/annotations set in the factory level if there are common
+			// labels/annotations
+			expected: makeLiteralSecret("literalSecret", map[string]string{
+				"foo": "changed",
+				"cat": "dog",
+			}, map[string]string{
+				"foo": "changed",
+				"cat": "dog",
+			}),
 		},
 	}
 

api/internal/target/kusttarget.go

@@ -7,7 +7,6 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
-	"log"
 	"strings"
 
 	"github.com/pkg/errors"
@@ -302,18 +301,15 @@ func (kt *KustTarget) configureExternalTransformers() ([]resmap.Transformer, err
 func (kt *KustTarget) accumulateResources(
 	ra *accumulator.ResAccumulator, paths []string) error {
 	for _, path := range paths {
-		ldr, err := kt.ldr.New(path)
-		if err == nil {
-			err = kt.accumulateDirectory(ra, ldr)
-			if err != nil {
-				return err
+		// try loading resource as file then as base (directory or git repository)
+		if errF := kt.accumulateFile(ra, path); errF != nil {
+			ldr, errL := kt.ldr.New(path)
+			if errL != nil {
+				return fmt.Errorf("accumulateFile %q, loader.New %q", errF, errL)
 			}
-		} else {
-			err2 := kt.accumulateFile(ra, path)
-			if err2 != nil {
-				// Log ldr.New() error to highlight git failures.
-				log.Print(err.Error())
-				return err2
+			errD := kt.accumulateDirectory(ra, ldr)
+			if errD != nil {
+				return fmt.Errorf("accumulateFile %q, accumulateDirector: %q", errF, errD)
 			}
 		}
 	}

api/internal/tools/tools.go

@@ -4,7 +4,7 @@
 // SPDX-License-Identifier: Apache-2.0
 
 // This file exists to automatically trigger installs
-// of the given tools, and is the offical 'unofficial'
+// of the given tools, and is the official 'unofficial'
 // way to declare a dependence on a Go binary until
 // some better technique comes along.
 

api/konfig/builtinpluginconsts/varreference.go

@@ -30,6 +30,9 @@ varReference:
 - path: spec/jobTemplate/spec/template/spec/initContainers/volumeMounts/mountPath
   kind: CronJob
 
+- path: spec/jobTemplate/spec/template/volumes/nfs/server
+  kind: CronJob
+
 - path: spec/template/spec/containers/args
   kind: DaemonSet
 
@@ -54,6 +57,9 @@ varReference:
 - path: spec/template/spec/initContainers/volumeMounts/mountPath
   kind: DaemonSet
 
+- path: spec/template/spec/volumes/nfs/server
+  kind: DaemonSet
+
 - path: spec/template/spec/containers/args
   kind: Deployment
 
@@ -78,6 +84,9 @@ varReference:
 - path: spec/template/spec/initContainers/volumeMounts/mountPath
   kind: Deployment
 
+- path: spec/template/spec/volumes/nfs/server
+  kind: Deployment
+
 - path: spec/rules/host
   kind: Ingress
 
@@ -111,6 +120,9 @@ varReference:
 - path: spec/template/spec/initContainers/volumeMounts/mountPath
   kind: Job
 
+- path: spec/template/spec/volumes/nfs/server
+  kind: Job
+
 - path: spec/containers/args
   kind: Pod
 
@@ -135,6 +147,9 @@ varReference:
 - path: spec/initContainers/volumeMounts/mountPath
   kind: Pod
 
+- path: spec/volumes/nfs/server
+  kind: Pod
+
 - path: spec/template/spec/containers/args
   kind: ReplicaSet
 
@@ -159,6 +174,9 @@ varReference:
 - path: spec/template/spec/initContainers/volumeMounts/mountPath
   kind: ReplicaSet
 
+- path: spec/template/spec/volumes/nfs/server
+  kind: ReplicaSet
+
 - path: spec/ports/port
   kind: Service
 
@@ -189,6 +207,12 @@ varReference:
 - path: spec/template/spec/initContainers/volumeMounts/mountPath
   kind: StatefulSet
 
+- path: spec/volumeClaimTemplates/spec/nfs/server
+  kind: StatefulSet
+
+- path: spec/nfs/server
+  kind: PersistentVolume
+
 - path: metadata/labels
 
 - path: metadata/annotations

api/krusty/accumulation_test.go

@@ -74,7 +74,7 @@ spec:
 	if err == nil {
 		t.Fatalf("expected an error")
 	}
-	if !IsMissingKustomizationFileError(err) {
+	if !strings.Contains(err.Error(), "accumulating resources") {
 		t.Fatalf("unexpected error: %q", err)
 	}
 }
@@ -89,7 +89,7 @@ resources:
 	if err == nil {
 		t.Fatalf("expected an error")
 	}
-	if !strings.Contains(err.Error(), "'/app/deployment.yaml' doesn't exist") {
+	if !strings.Contains(err.Error(), "accumulating resources") {
 		t.Fatalf("unexpected error: %q", err)
 	}
 }

api/krusty/baseandoverlaymedium_test.go

@@ -153,7 +153,8 @@ FRUIT=banana
 LEGUME=chickpea
 `)
 	th.WriteF("/app/overlay/configmap/dummy.txt",
-		`Lorem ipsum dolor sit amet, consectetur  
+		`Lorem ipsum dolor sit amet, consectetur
+ 		 
 adipiscing elit, sed do eiusmod tempor		
 incididunt ut labore et dolore magna aliqua.   
 `)
@@ -294,6 +295,7 @@ apiVersion: v1
 data:
   nonsense: |
     Lorem ipsum dolor sit amet, consectetur
+
     adipiscing elit, sed do eiusmod tempor
     incididunt ut labore et dolore magna aliqua.
 kind: ConfigMap
@@ -304,6 +306,6 @@ metadata:
     app: mungebot
     org: kubernetes
     repo: test-infra
-  name: test-infra-app-config-4mt28b5bg2
+  name: test-infra-app-config-hh272bg5d4
 `)
 }

api/krusty/variableref_test.go

@@ -1336,3 +1336,637 @@ spec:
     protocol: TCP
 `)
 }
+
+func TestVariableRefNFSServer(t *testing.T) {
+	th := kusttest_test.MakeHarness(t)
+	th.WriteK("/app/base", `
+resources:
+- pv_pvc.yaml
+- nfs_deployment.yaml
+- nfs_service.yaml
+- Deployment.yaml
+- CronJob.yaml
+- DaemonSet.yaml
+- ReplicaSet.yaml
+- StatefulSet.yaml
+- Pod.yaml
+- Job.yaml
+- nfs_pv.yaml
+
+vars:
+- name: NFS_SERVER_SERVICE_NAME
+  objref:
+    kind: Service
+    name: nfs-server-service
+    apiVersion: v1
+  fieldref:
+    fieldpath: metadata.name
+`)
+	th.WriteF("/app/base/pv_pvc.yaml", `
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: shared-volume-claim
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 10Gi
+`)
+	th.WriteF("/app/base/nfs_deployment.yaml", `
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  name: nfs-server
+spec:
+  replicas: 1
+  template:
+    spec:
+      metadata:
+        labels:
+          role: nfs-server
+      containers:
+        - name: nfs-server
+          image: gcr.io/google_containers/volume-nfs:0.8
+          ports:
+            - name: nfs
+              containerPort: 2049
+            - name: mountd
+              containerPort: 20048
+            - name: rpcbind
+              containerPort: 111
+          securityContext:
+            privileged: true
+          volumeMounts:
+            - mountPath: /exports
+              name: shared-files
+      volumes:
+        - name: shared-files
+          persistentVolumeClaim:
+            claimName: shared-volume-claim
+`)
+	th.WriteF("/app/base/nfs_service.yaml", `
+apiVersion: v1
+kind: Service
+metadata:
+  name: nfs-server-service
+spec:
+  ports:
+    - name: nfs
+      port: 2049
+    - name: mountd
+      port: 20048
+    - name: rpcbind
+      port: 111
+  selector:
+    role: nfs-server
+`)
+	th.WriteF("/app/base/Deployment.yaml", `
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: nginx
+  labels:
+    app.kubernetes.io/component: nginx
+spec:
+  selector:
+    matchLabels:
+      app.kubernetes.io/component: nginx
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/component: nginx
+    spec:
+      containers:
+      - name: nginx
+        image: nginx:1.15.7-alpine
+        ports:
+        - name: http
+          containerPort: 80
+        volumeMounts:
+          - mountPath: /app/shared-files
+            name: nfs-files-vol
+      volumes:
+        - name: nfs-files-vol
+          nfs:
+            server: $(NFS_SERVER_SERVICE_NAME).default.srv.cluster.local
+            path: /
+            readOnly: false
+`)
+	th.WriteF("/app/base/CronJob.yaml", `
+apiVersion: batch/v1beta1
+kind: CronJob
+metadata:
+  name: hello
+spec:
+  schedule: "*/1 * * * *"
+  jobTemplate:
+    spec:
+      template:
+        spec:
+          containers:
+          - name: hello
+            image: busybox
+            args:
+            - /bin/sh
+            - -c
+            - date; echo Hello from the Kubernetes cluster
+          restartPolicy: OnFailure
+          volumeMounts:
+          - mountPath: /app/shared-files
+            name: nfs-files-vol
+        volumes:
+        - name: nfs-files-vol
+          nfs:
+            server: $(NFS_SERVER_SERVICE_NAME).default.srv.cluster.local
+            path: /
+            readOnly: false
+`)
+	th.WriteF("/app/base/DaemonSet.yaml", `
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: fluentd-elasticsearch
+  namespace: kube-system
+  labels:
+    k8s-app: fluentd-logging
+spec:
+  selector:
+    matchLabels:
+      name: fluentd-elasticsearch
+  template:
+    metadata:
+      labels:
+        name: fluentd-elasticsearch
+    spec:
+      tolerations:
+      - key: node-role.kubernetes.io/master
+        effect: NoSchedule
+      containers:
+      - name: fluentd-elasticsearch
+        image: quay.io/fluentd_elasticsearch/fluentd:v2.5.2
+        resources:
+          limits:
+            memory: 200Mi
+          requests:
+            cpu: 100m
+            memory: 200Mi
+        volumeMounts:
+        - name: varlog
+          mountPath: /var/log
+        - name: varlibdockercontainers
+          mountPath: /var/lib/docker/containers
+          readOnly: true
+        - mountPath: /app/shared-files
+          name: nfs-files-vol
+      terminationGracePeriodSeconds: 30
+      volumes:
+      - name: varlog
+        hostPath:
+          path: /var/log
+      - name: varlibdockercontainers
+        hostPath:
+          path: /var/lib/docker/containers
+      - name: nfs-files-vol
+        nfs:
+          server: $(NFS_SERVER_SERVICE_NAME).default.srv.cluster.local
+          path: /
+          readOnly: false
+`)
+	th.WriteF("/app/base/ReplicaSet.yaml", `
+apiVersion: apps/v1
+kind: ReplicaSet
+metadata:
+  name: frontend
+  labels:
+    app: guestbook
+    tier: frontend
+spec:
+  # modify replicas according to your case
+  replicas: 3
+  selector:
+    matchLabels:
+      tier: frontend
+  template:
+    metadata:
+      labels:
+        tier: frontend
+    spec:
+      containers:
+      - name: php-redis
+        image: gcr.io/google_samples/gb-frontend:v3
+        volumeMounts:
+        - mountPath: /app/shared-files
+          name: nfs-files-vol
+      volumes:
+      - name: nfs-files-vol
+        nfs:
+          server: $(NFS_SERVER_SERVICE_NAME).default.srv.cluster.local
+          path: /
+          readOnly: false
+`)
+
+	th.WriteF("/app/base/Job.yaml", `
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: pi
+spec:
+  template:
+    spec:
+      containers:
+      - name: pi
+        image: perl
+        command: ["perl",  "-Mbignum=bpi", "-wle", "print bpi(2000)"]
+        volumeMounts:
+        - mountPath: /app/shared-files
+          name: nfs-files-vol
+      restartPolicy: Never
+      volumes:
+      - name: nfs-files-vol
+        nfs:
+          server: $(NFS_SERVER_SERVICE_NAME).default.srv.cluster.local
+          path: /
+          readOnly: false
+  backoffLimit: 4
+`)
+	th.WriteF("/app/base/StatefulSet.yaml", `
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: web
+spec:
+  selector:
+    matchLabels:
+      app: nginx # has to match .spec.template.metadata.labels
+  serviceName: "nginx"
+  replicas: 3 # by default is 1
+  template:
+    metadata:
+      labels:
+        app: nginx # has to match .spec.selector.matchLabels
+    spec:
+      terminationGracePeriodSeconds: 10
+      containers:
+      - name: nginx
+        image: k8s.gcr.io/nginx-slim:0.8
+        ports:
+        - containerPort: 80
+          name: web
+        volumeMounts:
+        - name: www
+          mountPath: /usr/share/nginx/html
+  volumeClaimTemplates:
+  - metadata:
+      name: www
+    spec:
+      accessModes: [ "ReadWriteMany" ]
+      nfs:
+        server: $(NFS_SERVER_SERVICE_NAME).default.srv.cluster.local
+        path: /
+        readOnly: false
+`)
+	th.WriteF("/app/base/Pod.yaml", `
+apiVersion: v1
+kind: Pod
+metadata:
+  name: myapp-pod
+  labels:
+    app: myapp
+spec:
+  containers:
+  - name: nginx
+    image: nginx:1.15.7-alpine
+    ports:
+    - name: http
+      containerPort: 80
+  volumeMounts:
+  - name: nfs-files-vol
+    mountPath: /app/shared-files
+  volumes:
+  - name: nfs-files-vol
+    nfs:
+      server: $(NFS_SERVER_SERVICE_NAME).default.srv.cluster.local
+      path: /
+      readOnly: false
+`)
+	th.WriteF("/app/base/nfs_pv.yaml", `
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+  name: nfs-files-pv
+spec:
+  capacity:
+    storage: 10Gi
+  accessModes:
+    - ReadWriteMany
+  nfs:
+    server: $(NFS_SERVER_SERVICE_NAME).default.srv.cluster.local
+    path: /
+    readOnly: false
+`)
+	th.WriteK("/app/overlay", `
+nameprefix: kustomized-
+resources:
+- ../base
+`)
+	m := th.Run("/app/overlay", th.MakeDefaultOptions())
+	th.AssertActualEqualsExpected(m, `
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: kustomized-shared-volume-claim
+spec:
+  accessModes:
+  - ReadWriteOnce
+  resources:
+    requests:
+      storage: 10Gi
+---
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  name: kustomized-nfs-server
+spec:
+  replicas: 1
+  template:
+    spec:
+      containers:
+      - image: gcr.io/google_containers/volume-nfs:0.8
+        name: nfs-server
+        ports:
+        - containerPort: 2049
+          name: nfs
+        - containerPort: 20048
+          name: mountd
+        - containerPort: 111
+          name: rpcbind
+        securityContext:
+          privileged: true
+        volumeMounts:
+        - mountPath: /exports
+          name: shared-files
+      metadata:
+        labels:
+          role: nfs-server
+      volumes:
+      - name: shared-files
+        persistentVolumeClaim:
+          claimName: kustomized-shared-volume-claim
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: kustomized-nfs-server-service
+spec:
+  ports:
+  - name: nfs
+    port: 2049
+  - name: mountd
+    port: 20048
+  - name: rpcbind
+    port: 111
+  selector:
+    role: nfs-server
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app.kubernetes.io/component: nginx
+  name: kustomized-nginx
+spec:
+  selector:
+    matchLabels:
+      app.kubernetes.io/component: nginx
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/component: nginx
+    spec:
+      containers:
+      - image: nginx:1.15.7-alpine
+        name: nginx
+        ports:
+        - containerPort: 80
+          name: http
+        volumeMounts:
+        - mountPath: /app/shared-files
+          name: nfs-files-vol
+      volumes:
+      - name: nfs-files-vol
+        nfs:
+          path: /
+          readOnly: false
+          server: kustomized-nfs-server-service.default.srv.cluster.local
+---
+apiVersion: batch/v1beta1
+kind: CronJob
+metadata:
+  name: kustomized-hello
+spec:
+  jobTemplate:
+    spec:
+      template:
+        spec:
+          containers:
+          - args:
+            - /bin/sh
+            - -c
+            - date; echo Hello from the Kubernetes cluster
+            image: busybox
+            name: hello
+          restartPolicy: OnFailure
+          volumeMounts:
+          - mountPath: /app/shared-files
+            name: nfs-files-vol
+        volumes:
+        - name: nfs-files-vol
+          nfs:
+            path: /
+            readOnly: false
+            server: kustomized-nfs-server-service.default.srv.cluster.local
+  schedule: '*/1 * * * *'
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  labels:
+    k8s-app: fluentd-logging
+  name: kustomized-fluentd-elasticsearch
+  namespace: kube-system
+spec:
+  selector:
+    matchLabels:
+      name: fluentd-elasticsearch
+  template:
+    metadata:
+      labels:
+        name: fluentd-elasticsearch
+    spec:
+      containers:
+      - image: quay.io/fluentd_elasticsearch/fluentd:v2.5.2
+        name: fluentd-elasticsearch
+        resources:
+          limits:
+            memory: 200Mi
+          requests:
+            cpu: 100m
+            memory: 200Mi
+        volumeMounts:
+        - mountPath: /var/log
+          name: varlog
+        - mountPath: /var/lib/docker/containers
+          name: varlibdockercontainers
+          readOnly: true
+        - mountPath: /app/shared-files
+          name: nfs-files-vol
+      terminationGracePeriodSeconds: 30
+      tolerations:
+      - effect: NoSchedule
+        key: node-role.kubernetes.io/master
+      volumes:
+      - hostPath:
+          path: /var/log
+        name: varlog
+      - hostPath:
+          path: /var/lib/docker/containers
+        name: varlibdockercontainers
+      - name: nfs-files-vol
+        nfs:
+          path: /
+          readOnly: false
+          server: kustomized-nfs-server-service.default.srv.cluster.local
+---
+apiVersion: apps/v1
+kind: ReplicaSet
+metadata:
+  labels:
+    app: guestbook
+    tier: frontend
+  name: kustomized-frontend
+spec:
+  replicas: 3
+  selector:
+    matchLabels:
+      tier: frontend
+  template:
+    metadata:
+      labels:
+        tier: frontend
+    spec:
+      containers:
+      - image: gcr.io/google_samples/gb-frontend:v3
+        name: php-redis
+        volumeMounts:
+        - mountPath: /app/shared-files
+          name: nfs-files-vol
+      volumes:
+      - name: nfs-files-vol
+        nfs:
+          path: /
+          readOnly: false
+          server: kustomized-nfs-server-service.default.srv.cluster.local
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: kustomized-web
+spec:
+  replicas: 3
+  selector:
+    matchLabels:
+      app: nginx
+  serviceName: nginx
+  template:
+    metadata:
+      labels:
+        app: nginx
+    spec:
+      containers:
+      - image: k8s.gcr.io/nginx-slim:0.8
+        name: nginx
+        ports:
+        - containerPort: 80
+          name: web
+        volumeMounts:
+        - mountPath: /usr/share/nginx/html
+          name: www
+      terminationGracePeriodSeconds: 10
+  volumeClaimTemplates:
+  - metadata:
+      name: www
+    spec:
+      accessModes:
+      - ReadWriteMany
+      nfs:
+        path: /
+        readOnly: false
+        server: kustomized-nfs-server-service.default.srv.cluster.local
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  labels:
+    app: myapp
+  name: kustomized-myapp-pod
+spec:
+  containers:
+  - image: nginx:1.15.7-alpine
+    name: nginx
+    ports:
+    - containerPort: 80
+      name: http
+  volumeMounts:
+  - mountPath: /app/shared-files
+    name: nfs-files-vol
+  volumes:
+  - name: nfs-files-vol
+    nfs:
+      path: /
+      readOnly: false
+      server: kustomized-nfs-server-service.default.srv.cluster.local
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: kustomized-pi
+spec:
+  backoffLimit: 4
+  template:
+    spec:
+      containers:
+      - command:
+        - perl
+        - -Mbignum=bpi
+        - -wle
+        - print bpi(2000)
+        image: perl
+        name: pi
+        volumeMounts:
+        - mountPath: /app/shared-files
+          name: nfs-files-vol
+      restartPolicy: Never
+      volumes:
+      - name: nfs-files-vol
+        nfs:
+          path: /
+          readOnly: false
+          server: kustomized-nfs-server-service.default.srv.cluster.local
+---
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+  name: kustomized-nfs-files-pv
+spec:
+  accessModes:
+  - ReadWriteMany
+  capacity:
+    storage: 10Gi
+  nfs:
+    path: /
+    readOnly: false
+    server: kustomized-nfs-server-service.default.srv.cluster.local
+`)
+}

api/kv/kv.go

@@ -93,7 +93,7 @@ func (kvl *loader) keyValuesFromFileSources(sources []string) ([]types.Pair, err
 
 // trimTrailingSpacesInLines takes string with multiple lines and trims the trailing white spaces and tabs from each line.
 func trimTrailingSpacesInLines(str string) string {
-	re := regexp.MustCompile(`\s*\n`)
+	re := regexp.MustCompile(`[ \t]*\n`)
 	return re.ReplaceAllString(str, "\n")
 }
 

api/kv/kv_test.go

@@ -97,8 +97,8 @@ func TestKeyValuesFromFileSources(t *testing.T) {
 }
 
 func TestTrimTrailingSpacesInLines(t *testing.T) {
-	input := "\"fooKey\": \"fooValue\"   \t\n\t\"barKey\": \"barValue\""
-	expected := "\"fooKey\": \"fooValue\"\n\t\"barKey\": \"barValue\""
+	input := "\"fooKey\": \"fooValue\"   \t\n \t\t  \n\t\"barKey\": \"barValue\""
+	expected := "\"fooKey\": \"fooValue\"\n\n\t\"barKey\": \"barValue\""
 	res := trimTrailingSpacesInLines(input)
 	if !reflect.DeepEqual(res, expected) {
 		t.Errorf("Trim trailing spaces in lines should succeed, got: %s exptected: %s", res, expected)

api/loader/fileloader.go

@@ -5,7 +5,10 @@ package loader
 
 import (
 	"fmt"
+	"io/ioutil"
 	"log"
+	"net/http"
+	"net/url"
 	"path/filepath"
 	"strings"
 
@@ -86,6 +89,9 @@ type fileLoader struct {
 	// File system utilities.
 	fSys filesys.FileSystem
 
+	// Used to load from HTTP
+	http *http.Client
+
 	// Used to clone repositories.
 	cloner git.Cloner
 
@@ -293,6 +299,25 @@ func (fl *fileLoader) errIfRepoCycle(newRepoSpec *git.RepoSpec) error {
 // else an error.  Relative paths are taken relative
 // to the root.
 func (fl *fileLoader) Load(path string) ([]byte, error) {
+	if u, err := url.Parse(path); err == nil && (u.Scheme == "http" || u.Scheme == "https") {
+		var hc *http.Client
+		if fl.http != nil {
+			hc = fl.http
+		} else {
+			hc = &http.Client{}
+		}
+		resp, err := hc.Get(path)
+		if err != nil {
+			return nil, err
+		}
+		defer resp.Body.Close()
+		body, err := ioutil.ReadAll(resp.Body)
+		if err != nil {
+			return nil, err
+		}
+		return body, nil
+	}
+
 	if !filepath.IsAbs(path) {
 		path = fl.root.Join(path)
 	}

api/loader/fileloader_test.go

@@ -4,7 +4,9 @@
 package loader
 
 import (
+	"bytes"
 	"io/ioutil"
+	"net/http"
 	"os"
 	"path"
 	"path/filepath"
@@ -54,6 +56,7 @@ func makeLoader() *fileLoader {
 	return NewFileLoaderAtRoot(MakeFakeFs(testCases))
 
 }
+
 func TestLoaderLoad(t *testing.T) {
 	l1 := makeLoader()
 	if "/" != l1.Root() {
@@ -579,3 +582,93 @@ func TestRepoIndirectCycleDetection(t *testing.T) {
 		t.Fatalf("unexpected err: %v", err)
 	}
 }
+
+// Inspired by https://hassansin.github.io/Unit-Testing-http-client-in-Go
+type fakeRoundTripper func(req *http.Request) *http.Response
+
+func (f fakeRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
+	return f(req), nil
+}
+
+func makeFakeHTTPClient(fn fakeRoundTripper) *http.Client {
+	return &http.Client{
+		Transport: fn,
+	}
+}
+
+// TestLoaderHTTP test http file loader
+func TestLoaderHTTP(t *testing.T) {
+	var testCasesFile = []testData{
+		{
+			path:            "http/file.yaml",
+			expectedContent: "file content",
+		},
+	}
+
+	l1 := NewFileLoaderAtRoot(MakeFakeFs(testCasesFile))
+	if "/" != l1.Root() {
+		t.Fatalf("incorrect root: '%s'\n", l1.Root())
+	}
+	for _, x := range testCasesFile {
+		b, err := l1.Load(x.path)
+		if err != nil {
+			t.Fatalf("unexpected load error: %v", err)
+		}
+		if !reflect.DeepEqual([]byte(x.expectedContent), b) {
+			t.Fatalf("in load expected %s, but got %s", x.expectedContent, b)
+		}
+	}
+
+	var testCasesHTTP = []testData{
+		{
+			path:            "http://example.com/resource.yaml",
+			expectedContent: "http content",
+		},
+		{
+			path:            "https://example.com/resource.yaml",
+			expectedContent: "https content",
+		},
+	}
+
+	for _, x := range testCasesHTTP {
+		hc := makeFakeHTTPClient(func(req *http.Request) *http.Response {
+			u := req.URL.String()
+			if x.path != u {
+				t.Fatalf("expected URL %s, but got %s", x.path, u)
+			}
+			return &http.Response{
+				StatusCode: 200,
+				Body:       ioutil.NopCloser(bytes.NewBufferString(x.expectedContent)),
+				Header:     make(http.Header),
+			}
+		})
+		l2 := l1
+		l2.http = hc
+		b, err := l2.Load(x.path)
+		if err != nil {
+			t.Fatalf("unexpected load error: %v", err)
+		}
+		if !reflect.DeepEqual([]byte(x.expectedContent), b) {
+			t.Fatalf("in load expected %s, but got %s", x.expectedContent, b)
+		}
+	}
+
+	var testCaseUnsupported = []testData{
+		{
+			path:            "httpsnotreal://example.com/resource.yaml",
+			expectedContent: "invalid",
+		},
+	}
+	for _, x := range testCaseUnsupported {
+		hc := makeFakeHTTPClient(func(req *http.Request) *http.Response {
+			t.Fatalf("unexpected request to URL %s", req.URL.String())
+			return nil
+		})
+		l2 := l1
+		l2.http = hc
+		_, err := l2.Load(x.path)
+		if err == nil {
+			t.Fatalf("expect error but get %v", err)
+		}
+	}
+}

api/resmap/resmap.go

@@ -114,7 +114,7 @@ type ResMap interface {
 	Append(*resource.Resource) error
 
 	// AppendAll appends another ResMap to self,
-	// failing on any OrgId collision.
+	// failing on any CurId collision.
 	AppendAll(ResMap) error
 
 	// AbsorbAll appends, replaces or merges the contents

api/types/generatorargs.go

@@ -21,4 +21,7 @@ type GeneratorArgs struct {
 
 	// KvPairSources for the generator.
 	KvPairSources `json:",inline,omitempty" yaml:",inline,omitempty"`
+
+	// GeneratorOptions modify this generator
+	GeneratorOptions *GeneratorOptions `json:"generatorOptions,omitempty" yaml:"generatorOptions,omitempty"`
 }

cmd/config/.golangci.yml

@@ -23,7 +23,7 @@ linters:
     - gofmt
     - goimports
 #    - golint
-    - gosec
+#    - gosec
     - gosimple
     - govet
     - ineffassign

cmd/config/configcobra/cmds.go

@@ -43,6 +43,30 @@ Advanced Documentation Topics:
 `,
 }
 
+// Export commands publicly for composition
+var (
+	Annotate           = commands.AnnotateCommand
+	Cat                = commands.CatCommand
+	Count              = commands.CountCommand
+	CreateSetter       = commands.CreateSetterCommand
+	CreateSubstitution = commands.CreateSubstitutionCommand
+	Fmt                = commands.FmtCommand
+	Grep               = commands.GrepCommand
+	ListSetters        = commands.ListSettersCommand
+	Merge              = commands.MergeCommand
+	Merge3             = commands.Merge3Command
+	RunFn              = commands.RunFnCommand
+	Set                = commands.SetCommand
+	Sink               = commands.SinkCommand
+	Source             = commands.SourceCommand
+	Tree               = commands.TreeCommand
+	Wrap               = commands.WrapCommand
+	XArgs              = commands.XArgsCommand
+
+	StackOnError = &commands.StackOnError
+	ExitOnError  = &commands.ExitOnError
+)
+
 // NewConfigCommand returns a new *cobra.Command for the config command group.  This may
 // be embedded into other go binaries as a way of packaging the "config" command as part
 // of another binary.
@@ -69,6 +93,7 @@ func NewConfigCommand(name string) *cobra.Command {
 
 	name = strings.TrimSpace(name + " config")
 	commands.ExitOnError = true
+	root.AddCommand(commands.AnnotateCommand(name))
 	root.AddCommand(commands.GrepCommand(name))
 	root.AddCommand(commands.TreeCommand(name))
 	root.AddCommand(commands.CatCommand(name))
@@ -77,9 +102,13 @@ func NewConfigCommand(name string) *cobra.Command {
 	root.AddCommand(commands.Merge3Command(name))
 	root.AddCommand(commands.CountCommand(name))
 	root.AddCommand(commands.RunFnCommand(name))
+	root.AddCommand(commands.XArgsCommand())
+	root.AddCommand(commands.WrapCommand())
+
 	root.AddCommand(commands.SetCommand(name))
 	root.AddCommand(commands.ListSettersCommand(name))
 	root.AddCommand(commands.CreateSetterCommand(name))
+	root.AddCommand(commands.CreateSubstitutionCommand(name))
 	root.AddCommand(commands.SinkCommand(name))
 	root.AddCommand(commands.SourceCommand(name))
 

cmd/config/docs/api-conventions/config-io.md

@@ -29,7 +29,7 @@ metadata:
 ### `config.kubernetes.io/index`
 
 Records the index of a Resource in file. In a multi-object YAML file, Resources are separated
-by three dashes (`---`), and the index represents the positon of the Resource starting from zero.
+by three dashes (`---`), and the index represents the position of the Resource starting from zero.
 
 This annotation **SHOULD** be set when reading Resources from files.
 It **SHOULD** be unset when writing Resources to files.

cmd/config/docs/api-conventions/functions-impl.md

@@ -129,7 +129,7 @@ spec:
 
 ### Output
 
-The function is invoked using by runing `kustomize config run dir/`.
+The function is invoked using byrunning `kustomize config run dir/`.
 
 `dir/my-instance_deployment.yaml` contains the Deployment:
 

cmd/config/docs/commands/annotate.md

@@ -0,0 +1,18 @@
+## annotate
+
+[Alpha] Set an annotation on Resources.
+
+### Synopsis
+
+[Alpha]  Set an annotation on Resources.
+
+  DIR:
+    Path to local directory.
+
+### Examples
+
+    kustomize config annotate my-dir/ --kv foo=bar
+
+    kustomize config annotate my-dir/ --kv foo=bar --kv a=b
+
+    kustomize config annotate my-dir/ --kv foo=bar --kind Deployment --name foo

cmd/config/docs/commands/sink.md

@@ -6,10 +6,10 @@
 
 [Alpha] Implement a Sink by writing input to a local directory.
 
-    kustomize config sink DIR
+    kustomize config sink [DIR]
 
   DIR:
-    Path to local directory.
+    Path to local directory.  If unspecified, sink will write to stdout as if it were a single file.
 
 `sink` writes its input to a directory
 

cmd/config/docs/commands/source.md

@@ -6,10 +6,11 @@
 
 [Alpha] Implement a Source by reading a local directory.
 
-    kustomize config source DIR
+    kustomize config source DIR...
 
   DIR:
-    Path to local directory.
+    One or more paths to local directories.  Contents from directories will be concatenated.
+    If no directories are provided, source will read from stdin as if it were a single file.
 
 `source` emits configuration to act as input to a function
 

cmd/config/ext/ext.go

@@ -0,0 +1,12 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package ext
+
+import "path/filepath"
+
+// GetOpenAPIFile returns the path to the file containing supplementary OpenAPI definitions.
+// Maybe be overridden to configure which file to read OpenAPI definitions from.
+var GetOpenAPIFile = func(args []string) (string, error) {
+	return filepath.Join(args[0], "kustomization"), nil
+}

cmd/config/go.mod

@@ -10,7 +10,5 @@ require (
 	github.com/spf13/pflag v1.0.5
 	github.com/stretchr/testify v1.4.0
 	k8s.io/apimachinery v0.17.0
-	sigs.k8s.io/kustomize/kyaml v0.0.0
+	sigs.k8s.io/kustomize/kyaml v0.1.2
 )
-
-replace sigs.k8s.io/kustomize/kyaml v0.0.0 => ../../kyaml

cmd/config/go.sum

@@ -1,12 +1,18 @@
+github.com/360EntSecGroup-Skylar/excelize v1.4.1/go.mod h1:vnax29X2usfl7HHkBrX5EvSCJcmH3dT9luvxzu8iGAE=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ=
+github.com/PuerkitoBio/goquery v1.5.0/go.mod h1:qD2PgZ9lccMbQlc7eEOjaeRlFQON7xY8kdmcsrnKqMg=
 github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
 github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI=
 github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
 github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
 github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M=
 github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
+github.com/andybalholm/cascadia v1.0.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
+github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
+github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
+github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
 github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
 github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=
 github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
@@ -16,6 +22,7 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM=
+github.com/dustmop/soup v1.1.2-0.20190516214245-38228baa104e/go.mod h1:CgNC6SGbT+Xb8wGGvzilttZL1mc5sQ/5KkcxsZttMIk=
 github.com/elazarl/goproxy v0.0.0-20170405201442-c4fc26588b6e/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
 github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
 github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
@@ -82,6 +89,7 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJ
 github.com/modern-go/reflect2 v0.0.0-20180320133207-05fbef0ca5da/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8=
 github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw=
 github.com/olekukonko/tablewriter v0.0.4 h1:vHD/YYe1Wolo78koG299f7V/VAS08c6IpCLn+Ejf/w8=
@@ -91,6 +99,7 @@ github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+W
 github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
 github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/paulmach/orb v0.1.3/go.mod h1:VFlX/8C+IQ1p6FTRRKzKoOPJnvEtA5G0Veuqwbu//Vk=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
@@ -100,6 +109,7 @@ github.com/posener/complete/v2 v2.0.1-alpha.12 h1:0wvkuDfHb5vSZlNBYgpEH4XQHpF46M
 github.com/posener/complete/v2 v2.0.1-alpha.12/go.mod h1://JlL91cS2JV7rOl6LVHrRqBXoBUecJu3ILQPgbJiMQ=
 github.com/posener/script v1.0.4 h1:nSuXW5ZdmFnQIueLB2s0qvs4oNsUloM1Zydzh75v42w=
 github.com/posener/script v1.0.4/go.mod h1:Rg3ijooqulo05aGLyGsHoLmIOUzHUVK19WVgrYBPU/E=
+github.com/qri-io/starlib v0.4.2-0.20200213133954-ff2e8cd5ef8d/go.mod h1:7DPO4domFU579Ga6E61sB9VFNaniPVwJP5C4bBCu3wA=
 github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
 github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0=
 github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
@@ -116,6 +126,7 @@ github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DM
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v0.0.0-20151208002404-e3a8ff8ce365/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/testify v1.2.3-0.20181224173747-660f15d67dbb/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
@@ -123,10 +134,15 @@ github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljT
 github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca h1:1CFlNzQhALwjS9mBAUkycX616GzgsuYUOCHA5+HSlXI=
 github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg=
 github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
+go.starlark.net v0.0.0-20190528202925-30ae18b8564f/go.mod h1:c1/X6cHgvdXj6pUlmWKMkuqRnW4K8x2vwt6JAaaircg=
+go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5/go.mod h1:nmDLcffg48OtT/PSW0Hg7FvpRQsQh5OSqIylirxKC7o=
 golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
 golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20191004110552-13f9640d40b9 h1:rjwSpXsdiK0dV8/Naq3kAw9ymfAeJIyd0upUIElB+lI=
@@ -137,6 +153,7 @@ golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5h
 golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191002063906-3421d5a6bb1c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
@@ -156,13 +173,17 @@ gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I=
 gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v3 v3.0.0-20191026110619-0b21df46bc1d h1:LCPbGQ34PMrwad11aMZ+dbz5SAsq/0ySjRwQ8I9Qwd8=
-gopkg.in/yaml.v3 v3.0.0-20191026110619-0b21df46bc1d/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v2 v2.2.7 h1:VUgggvou5XRW9mHwD/yXxIYSMtY0zoKQf/v226p2nyo=
+gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v3 v3.0.0-20191120175047-4206685974f2 h1:XZx7nhd5GMaZpmDaEHFVafUZC7ya0fuo7cSJ3UCKYmM=
+gopkg.in/yaml.v3 v3.0.0-20191120175047-4206685974f2/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 k8s.io/apimachinery v0.17.0 h1:xRBnuie9rXcPxUkDizUsGvPf1cnlZCFu210op7J7LJo=
 k8s.io/apimachinery v0.17.0/go.mod h1:b9qmWdKlLuU9EBh+06BtLcSf/Mu89rWL33naRxs1uZg=
 k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
 k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk=
 k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I=
 k8s.io/kube-openapi v0.0.0-20191107075043-30be4d16710a/go.mod h1:1TqjTSzOxsLGIKfj0lK8EeCP7K1iUG65v09OM0/WG5E=
+sigs.k8s.io/kustomize/kyaml v0.1.2 h1:l12+QGl+ETUHhP8/bZAi6TknU7H194fXL/9b2gUxZFY=
+sigs.k8s.io/kustomize/kyaml v0.1.2/go.mod h1:461i94nj0h0ylJ6w83jLkR4SqqVhn1iY6fjD0JSTQeE=
 sigs.k8s.io/structured-merge-diff v0.0.0-20190525122527-15d366b2352e/go.mod h1:wWxsB5ozmmv/SG7nM11ayaAW51xMvak/t1r0CSlcokI=
 sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=

cmd/config/internal/commands/annotate.go

@@ -0,0 +1,103 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package commands
+
+import (
+	"strings"
+
+	"github.com/spf13/cobra"
+	"sigs.k8s.io/kustomize/cmd/config/internal/generateddocs/commands"
+	"sigs.k8s.io/kustomize/kyaml/errors"
+	"sigs.k8s.io/kustomize/kyaml/kio"
+	"sigs.k8s.io/kustomize/kyaml/yaml"
+)
+
+// NewAnnotateRunner returns a command runner.
+func NewAnnotateRunner(parent string) *AnnotateRunner {
+	r := &AnnotateRunner{}
+	c := &cobra.Command{
+		Use:     "annotate [DIR]",
+		Args:    cobra.MaximumNArgs(1),
+		Short:   commands.AnnotateShort,
+		Long:    commands.AnnotateLong,
+		Example: commands.AnnotateExamples,
+		RunE:    r.runE,
+	}
+	fixDocs(parent, c)
+	r.Command = c
+	c.Flags().StringVar(&r.Kind, "kind", "", "Resource kind to annotate")
+	c.Flags().StringVar(&r.ApiVersion, "apiVersion", "", "Resource apiVersion to annotate")
+	c.Flags().StringVar(&r.Name, "name", "", "Resource name to annotate")
+	c.Flags().StringVar(&r.Namespace, "namespace", "", "Resource namespace to annotate")
+	c.Flags().StringSliceVar(&r.Values, "kv", []string{}, "annotation as KEY=VALUE")
+	return r
+}
+
+func AnnotateCommand(parent string) *cobra.Command {
+	return NewAnnotateRunner(parent).Command
+}
+
+type AnnotateRunner struct {
+	Command    *cobra.Command
+	Values     []string
+	Kind       string
+	Name       string
+	ApiVersion string
+	Namespace  string
+	Path       string
+}
+
+func (r *AnnotateRunner) runE(c *cobra.Command, args []string) error {
+	var input []kio.Reader
+	var output []kio.Writer
+	if len(args) == 0 {
+		rw := &kio.ByteReadWriter{Reader: c.InOrStdin(), Writer: c.OutOrStdout()}
+		input = []kio.Reader{rw}
+		output = []kio.Writer{rw}
+	} else {
+		rw := &kio.LocalPackageReadWriter{PackagePath: args[0], NoDeleteFiles: true}
+		input = []kio.Reader{rw}
+		output = []kio.Writer{rw}
+	}
+	return handleError(c, kio.Pipeline{
+		Inputs:  input,
+		Filters: []kio.Filter{r},
+		Outputs: output,
+	}.Execute())
+}
+
+func (r *AnnotateRunner) Filter(nodes []*yaml.RNode) ([]*yaml.RNode, error) {
+	for i := range nodes {
+		n := nodes[i]
+		m, err := n.GetMeta()
+		if err != nil {
+			return nil, err
+		}
+		if r.Kind != "" && r.Kind != m.Kind {
+			continue
+		}
+		if r.ApiVersion != "" && r.ApiVersion != m.APIVersion {
+			continue
+		}
+		if r.Namespace != "" && r.Namespace != m.Namespace {
+			continue
+		}
+		if r.Name != "" && r.Name != m.Name {
+			continue
+		}
+
+		for i := range r.Values {
+			// split key, value pairs
+			kv := strings.SplitN(r.Values[i], "=", 2)
+			if len(kv) != 2 {
+				return nil, errors.Errorf("must specify --kv as KEY=VALUE: %s", r.Values[i])
+			}
+			if err := n.PipeE(yaml.SetAnnotation(kv[0], kv[1])); err != nil {
+				return nil, err
+			}
+		}
+
+	}
+	return nodes, nil
+}

cmd/config/internal/commands/annotate_test.go

@@ -0,0 +1,488 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package commands
+
+import (
+	"bytes"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"sigs.k8s.io/kustomize/kyaml/kio"
+)
+
+func TestAnnotateCommand(t *testing.T) {
+	var tests = []struct {
+		name     string
+		args     []string
+		expected string
+	}{
+		{
+			name:     "single value",
+			args:     []string{"--kv", "a=b"},
+			expected: expectedSingleValue,
+		},
+		{
+			name:     "multi value",
+			args:     []string{"--kv", "a=b", "--kv", "c=d"},
+			expected: expectedMultiValue,
+		},
+		{
+			name:     "filter kind",
+			args:     []string{"--kv", "a=b", "--kind", "Service"},
+			expected: expectedFilterKindService,
+		},
+		{
+			name:     "filter apiVersion",
+			args:     []string{"--kv", "a=b", "--apiVersion", "v1"},
+			expected: expectedFilterApiVersionV1,
+		},
+		{
+			name:     "filter name",
+			args:     []string{"--kv", "a=b", "--name", "bar"},
+			expected: expectedFilterNameBar,
+		},
+		{
+			name:     "filter namespace",
+			args:     []string{"--kv", "a=b", "--namespace", "bar"},
+			expected: expectedFilterNamespaceBar,
+		},
+	}
+	for i := range tests {
+		tt := tests[i]
+		t.Run(tt.name, func(t *testing.T) {
+			d := initTestDir(t)
+			defer os.RemoveAll(d)
+
+			a := NewAnnotateRunner("")
+			a.Command.SetArgs(append([]string{d}, tt.args...))
+			a.Command.SilenceUsage = true
+			a.Command.SilenceErrors = true
+
+			err := a.Command.Execute()
+			if !assert.NoError(t, err) {
+				t.FailNow()
+			}
+
+			actual := &bytes.Buffer{}
+			err = kio.Pipeline{
+				Inputs:  []kio.Reader{kio.LocalPackageReader{PackagePath: d}},
+				Outputs: []kio.Writer{kio.ByteWriter{Writer: actual, KeepReaderAnnotations: true}},
+			}.Execute()
+			if !assert.NoError(t, err) {
+				t.FailNow()
+			}
+			if !assert.Equal(t,
+				strings.TrimSpace(tt.expected),
+				strings.TrimSpace(actual.String())) {
+				t.FailNow()
+			}
+		})
+	}
+}
+
+func initTestDir(t *testing.T) string {
+	d, err := ioutil.TempDir("", "kustomize-annotate-test")
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
+	err = ioutil.WriteFile(filepath.Join(d, "f1.yaml"), []byte(f1Input), 0600)
+	if !assert.NoError(t, err) {
+		defer os.RemoveAll(d)
+		t.FailNow()
+	}
+	err = ioutil.WriteFile(filepath.Join(d, "f2.yaml"), []byte(f2Input), 0600)
+	if !assert.NoError(t, err) {
+		defer os.RemoveAll(d)
+		t.FailNow()
+	}
+	return d
+}
+
+var (
+	f1Input = `
+kind: Deployment
+metadata:
+  labels:
+    app: nginx2
+  name: foo
+  annotations:
+    app: nginx2
+spec:
+  replicas: 1
+---
+kind: Service
+metadata:
+  name: foo
+  annotations:
+    app: nginx
+spec:
+  selector:
+    app: nginx
+`
+
+	f2Input = `
+apiVersion: v1
+kind: Abstraction
+metadata:
+  name: foo
+  configFn:
+    container:
+      image: gcr.io/example/reconciler:v1
+  annotations:
+    config.kubernetes.io/local-config: "true"
+  namespace: bar
+spec:
+  replicas: 3
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: nginx
+  name: bar
+  annotations:
+    app: nginx
+  namespace: foo
+spec:
+  replicas: 3
+`
+
+	expectedSingleValue = `kind: Deployment
+metadata:
+  labels:
+    app: nginx2
+  name: foo
+  annotations:
+    app: nginx2
+    a: 'b'
+    config.kubernetes.io/index: '0'
+    config.kubernetes.io/path: 'f1.yaml'
+spec:
+  replicas: 1
+---
+kind: Service
+metadata:
+  name: foo
+  annotations:
+    app: nginx
+    a: 'b'
+    config.kubernetes.io/index: '1'
+    config.kubernetes.io/path: 'f1.yaml'
+spec:
+  selector:
+    app: nginx
+---
+apiVersion: v1
+kind: Abstraction
+metadata:
+  name: foo
+  configFn:
+    container:
+      image: gcr.io/example/reconciler:v1
+  annotations:
+    config.kubernetes.io/local-config: "true"
+    a: 'b'
+    config.kubernetes.io/index: '0'
+    config.kubernetes.io/path: 'f2.yaml'
+  namespace: bar
+spec:
+  replicas: 3
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: nginx
+  name: bar
+  annotations:
+    app: nginx
+    a: 'b'
+    config.kubernetes.io/index: '1'
+    config.kubernetes.io/path: 'f2.yaml'
+  namespace: foo
+spec:
+  replicas: 3
+`
+
+	expectedMultiValue = `kind: Deployment
+metadata:
+  labels:
+    app: nginx2
+  name: foo
+  annotations:
+    app: nginx2
+    a: 'b'
+    c: 'd'
+    config.kubernetes.io/index: '0'
+    config.kubernetes.io/path: 'f1.yaml'
+spec:
+  replicas: 1
+---
+kind: Service
+metadata:
+  name: foo
+  annotations:
+    app: nginx
+    a: 'b'
+    c: 'd'
+    config.kubernetes.io/index: '1'
+    config.kubernetes.io/path: 'f1.yaml'
+spec:
+  selector:
+    app: nginx
+---
+apiVersion: v1
+kind: Abstraction
+metadata:
+  name: foo
+  configFn:
+    container:
+      image: gcr.io/example/reconciler:v1
+  annotations:
+    config.kubernetes.io/local-config: "true"
+    a: 'b'
+    c: 'd'
+    config.kubernetes.io/index: '0'
+    config.kubernetes.io/path: 'f2.yaml'
+  namespace: bar
+spec:
+  replicas: 3
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: nginx
+  name: bar
+  annotations:
+    app: nginx
+    a: 'b'
+    c: 'd'
+    config.kubernetes.io/index: '1'
+    config.kubernetes.io/path: 'f2.yaml'
+  namespace: foo
+spec:
+  replicas: 3
+`
+
+	expectedFilterKindService = `kind: Deployment
+metadata:
+  labels:
+    app: nginx2
+  name: foo
+  annotations:
+    app: nginx2
+    config.kubernetes.io/index: '0'
+    config.kubernetes.io/path: 'f1.yaml'
+spec:
+  replicas: 1
+---
+kind: Service
+metadata:
+  name: foo
+  annotations:
+    app: nginx
+    a: 'b'
+    config.kubernetes.io/index: '1'
+    config.kubernetes.io/path: 'f1.yaml'
+spec:
+  selector:
+    app: nginx
+---
+apiVersion: v1
+kind: Abstraction
+metadata:
+  name: foo
+  configFn:
+    container:
+      image: gcr.io/example/reconciler:v1
+  annotations:
+    config.kubernetes.io/local-config: "true"
+    config.kubernetes.io/index: '0'
+    config.kubernetes.io/path: 'f2.yaml'
+  namespace: bar
+spec:
+  replicas: 3
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: nginx
+  name: bar
+  annotations:
+    app: nginx
+    config.kubernetes.io/index: '1'
+    config.kubernetes.io/path: 'f2.yaml'
+  namespace: foo
+spec:
+  replicas: 3
+`
+
+	expectedFilterApiVersionV1 = `kind: Deployment
+metadata:
+  labels:
+    app: nginx2
+  name: foo
+  annotations:
+    app: nginx2
+    config.kubernetes.io/index: '0'
+    config.kubernetes.io/path: 'f1.yaml'
+spec:
+  replicas: 1
+---
+kind: Service
+metadata:
+  name: foo
+  annotations:
+    app: nginx
+    config.kubernetes.io/index: '1'
+    config.kubernetes.io/path: 'f1.yaml'
+spec:
+  selector:
+    app: nginx
+---
+apiVersion: v1
+kind: Abstraction
+metadata:
+  name: foo
+  configFn:
+    container:
+      image: gcr.io/example/reconciler:v1
+  annotations:
+    config.kubernetes.io/local-config: "true"
+    a: 'b'
+    config.kubernetes.io/index: '0'
+    config.kubernetes.io/path: 'f2.yaml'
+  namespace: bar
+spec:
+  replicas: 3
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: nginx
+  name: bar
+  annotations:
+    app: nginx
+    config.kubernetes.io/index: '1'
+    config.kubernetes.io/path: 'f2.yaml'
+  namespace: foo
+spec:
+  replicas: 3
+`
+
+	expectedFilterNameBar = `kind: Deployment
+metadata:
+  labels:
+    app: nginx2
+  name: foo
+  annotations:
+    app: nginx2
+    config.kubernetes.io/index: '0'
+    config.kubernetes.io/path: 'f1.yaml'
+spec:
+  replicas: 1
+---
+kind: Service
+metadata:
+  name: foo
+  annotations:
+    app: nginx
+    config.kubernetes.io/index: '1'
+    config.kubernetes.io/path: 'f1.yaml'
+spec:
+  selector:
+    app: nginx
+---
+apiVersion: v1
+kind: Abstraction
+metadata:
+  name: foo
+  configFn:
+    container:
+      image: gcr.io/example/reconciler:v1
+  annotations:
+    config.kubernetes.io/local-config: "true"
+    config.kubernetes.io/index: '0'
+    config.kubernetes.io/path: 'f2.yaml'
+  namespace: bar
+spec:
+  replicas: 3
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: nginx
+  name: bar
+  annotations:
+    app: nginx
+    a: 'b'
+    config.kubernetes.io/index: '1'
+    config.kubernetes.io/path: 'f2.yaml'
+  namespace: foo
+spec:
+  replicas: 3
+`
+
+	expectedFilterNamespaceBar = `kind: Deployment
+metadata:
+  labels:
+    app: nginx2
+  name: foo
+  annotations:
+    app: nginx2
+    config.kubernetes.io/index: '0'
+    config.kubernetes.io/path: 'f1.yaml'
+spec:
+  replicas: 1
+---
+kind: Service
+metadata:
+  name: foo
+  annotations:
+    app: nginx
+    config.kubernetes.io/index: '1'
+    config.kubernetes.io/path: 'f1.yaml'
+spec:
+  selector:
+    app: nginx
+---
+apiVersion: v1
+kind: Abstraction
+metadata:
+  name: foo
+  configFn:
+    container:
+      image: gcr.io/example/reconciler:v1
+  annotations:
+    config.kubernetes.io/local-config: "true"
+    a: 'b'
+    config.kubernetes.io/index: '0'
+    config.kubernetes.io/path: 'f2.yaml'
+  namespace: bar
+spec:
+  replicas: 3
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: nginx
+  name: bar
+  annotations:
+    app: nginx
+    config.kubernetes.io/index: '1'
+    config.kubernetes.io/path: 'f2.yaml'
+  namespace: foo
+spec:
+  replicas: 3
+`
+)

cmd/config/internal/commands/cmdcreatesetter.go

@@ -5,9 +5,11 @@ package commands
 
 import (
 	"github.com/spf13/cobra"
+	"sigs.k8s.io/kustomize/cmd/config/ext"
 	"sigs.k8s.io/kustomize/cmd/config/internal/generateddocs/commands"
 	"sigs.k8s.io/kustomize/kyaml/kio"
 	"sigs.k8s.io/kustomize/kyaml/setters"
+	"sigs.k8s.io/kustomize/kyaml/setters2/settersutil"
 )
 
 // NewCreateSetterRunner returns a command runner.
@@ -23,22 +25,27 @@ func NewCreateSetterRunner(parent string) *CreateSetterRunner {
 		RunE:    r.runE,
 	}
 	set.Flags().StringVar(&r.Set.SetPartialField.SetBy, "set-by", "",
-		"set the setBy annotation.")
+		"record who the field was default by.")
 	set.Flags().StringVar(&r.Set.SetPartialField.Description, "description", "",
-		"set the description of the field value.")
+		"record a description for the current setter value.")
 	set.Flags().StringVar(&r.Set.SetPartialField.Field, "field", "",
-		"name of the field to set -- e.g. --field port")
+		"name of the field to set -- e.g. --field port.  defaults to all fields match"+
+			"VALUE.  maybe be the field name, field path, or partial field path (suffix)")
 	set.Flags().StringVar(&r.Set.ResourceMeta.Name, "name", "",
 		"name of the Resource on which to create the setter.")
+	set.Flags().MarkHidden("name")
 	set.Flags().StringVar(&r.Set.ResourceMeta.Kind, "kind", "",
 		"kind of the Resource on which to create the setter.")
+	set.Flags().MarkHidden("kind")
 	set.Flags().StringVar(&r.Set.SetPartialField.Type, "type", "",
-		"valid OpenAPI field type -- e.g. integer,boolean,string.")
+		"OpenAPI field type for the setter -- e.g. integer,boolean,string.")
 	set.Flags().BoolVar(&r.Set.SetPartialField.Partial, "partial", false,
 		"create a partial setter for only part of the field value.")
+	set.Flags().MarkHidden("partial")
+	set.Flags().StringVar(&setterVersion, "version", "",
+		"use this version of the setter format")
+	set.Flags().MarkHidden("version")
 	fixDocs(parent, set)
-	set.MarkFlagRequired("type")
-	set.MarkFlagRequired("field")
 	r.Command = set
 	return r
 }
@@ -48,8 +55,10 @@ func CreateSetterCommand(parent string) *cobra.Command {
 }
 
 type CreateSetterRunner struct {
-	Command *cobra.Command
-	Set     setters.CreateSetter
+	Command      *cobra.Command
+	Set          setters.CreateSetter
+	CreateSetter settersutil.SetterCreator
+	OpenAPIFile  string
 }
 
 func (r *CreateSetterRunner) runE(c *cobra.Command, args []string) error {
@@ -57,12 +66,41 @@ func (r *CreateSetterRunner) runE(c *cobra.Command, args []string) error {
 }
 
 func (r *CreateSetterRunner) preRunE(c *cobra.Command, args []string) error {
+	var err error
 	r.Set.SetPartialField.Setter.Name = args[1]
 	r.Set.SetPartialField.Setter.Value = args[2]
+	r.CreateSetter.Name = args[1]
+	r.CreateSetter.FieldValue = args[2]
+	r.CreateSetter.FieldName, err = c.Flags().GetString("field")
+	if err != nil {
+		return err
+	}
+
+	if setterVersion == "" {
+		if len(args) < 3 {
+			setterVersion = "v1"
+		} else if err := initSetterVersion(c, args); err != nil {
+			return err
+		}
+	}
+	if setterVersion == "v2" {
+		var err error
+		r.OpenAPIFile, err = ext.GetOpenAPIFile(args)
+		r.CreateSetter.Description = r.Set.SetPartialField.Description
+		r.CreateSetter.SetBy = r.Set.SetPartialField.SetBy
+		r.CreateSetter.Type = r.Set.SetPartialField.Type
+		if err != nil {
+			return err
+		}
+	}
 	return nil
 }
 
 func (r *CreateSetterRunner) set(c *cobra.Command, args []string) error {
+	if setterVersion == "v2" {
+		return r.CreateSetter.Create(r.OpenAPIFile, args[0])
+	}
+
 	rw := &kio.LocalPackageReadWriter{PackagePath: args[0]}
 	err := kio.Pipeline{
 		Inputs:  []kio.Reader{rw},

cmd/config/internal/commands/cmdcreatesetter_test.go

@@ -0,0 +1,131 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package commands_test
+
+import (
+	"bytes"
+	"io/ioutil"
+	"os"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"sigs.k8s.io/kustomize/cmd/config/ext"
+	"sigs.k8s.io/kustomize/cmd/config/internal/commands"
+	"sigs.k8s.io/kustomize/kyaml/openapi"
+)
+
+func TestCreateSetterCommand(t *testing.T) {
+	var tests = []struct {
+		name              string
+		input             string
+		args              []string
+		out               string
+		expectedOpenAPI   string
+		expectedResources string
+	}{
+		{
+			name: "add replicas",
+			args: []string{"replicas", "3", "--description", "hello world", "--set-by", "me"},
+			input: `
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: nginx-deployment
+spec:
+  replicas: 3
+ `,
+			expectedOpenAPI: `
+apiVersion: v1alpha1
+kind: Example
+openAPI:
+  definitions:
+    io.k8s.cli.setters.replicas:
+      description: hello world
+      x-k8s-cli:
+        setter:
+          name: replicas
+          value: "3"
+          setBy: me
+ `,
+			expectedResources: `
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: nginx-deployment
+spec:
+  replicas: 3 # {"$ref":"#/definitions/io.k8s.cli.setters.replicas"}
+ `,
+		},
+	}
+	for i := range tests {
+		test := tests[i]
+		t.Run(test.name, func(t *testing.T) {
+			// reset the openAPI afterward
+			openapi.ResetOpenAPI()
+			defer openapi.ResetOpenAPI()
+
+			f, err := ioutil.TempFile("", "k8s-cli-")
+			if !assert.NoError(t, err) {
+				t.FailNow()
+			}
+			defer os.Remove(f.Name())
+			err = ioutil.WriteFile(f.Name(), []byte(`
+apiVersion: v1alpha1
+kind: Example
+`), 0600)
+			if !assert.NoError(t, err) {
+				t.FailNow()
+			}
+			old := ext.GetOpenAPIFile
+			defer func() { ext.GetOpenAPIFile = old }()
+			ext.GetOpenAPIFile = func(args []string) (s string, err error) {
+				return f.Name(), nil
+			}
+
+			r, err := ioutil.TempFile("", "k8s-cli-*.yaml")
+			if !assert.NoError(t, err) {
+				t.FailNow()
+			}
+			defer os.Remove(r.Name())
+			err = ioutil.WriteFile(r.Name(), []byte(test.input), 0600)
+			if !assert.NoError(t, err) {
+				t.FailNow()
+			}
+
+			runner := commands.NewCreateSetterRunner("")
+			out := &bytes.Buffer{}
+			runner.Command.SetOut(out)
+			runner.Command.SetArgs(append([]string{r.Name()}, test.args...))
+			err = runner.Command.Execute()
+			if !assert.NoError(t, err) {
+				t.FailNow()
+			}
+
+			if !assert.Equal(t, test.out, out.String()) {
+				t.FailNow()
+			}
+
+			actualResources, err := ioutil.ReadFile(r.Name())
+			if !assert.NoError(t, err) {
+				t.FailNow()
+			}
+			if !assert.Equal(t,
+				strings.TrimSpace(test.expectedResources),
+				strings.TrimSpace(string(actualResources))) {
+				t.FailNow()
+			}
+
+			actualOpenAPI, err := ioutil.ReadFile(f.Name())
+			if !assert.NoError(t, err) {
+				t.FailNow()
+			}
+			if !assert.Equal(t,
+				strings.TrimSpace(test.expectedOpenAPI),
+				strings.TrimSpace(string(actualOpenAPI))) {
+				t.FailNow()
+			}
+		})
+	}
+}

cmd/config/internal/commands/cmdcreatesubstitution.go

@@ -0,0 +1,81 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package commands
+
+import (
+	"strings"
+
+	"github.com/spf13/cobra"
+	"sigs.k8s.io/kustomize/cmd/config/ext"
+	"sigs.k8s.io/kustomize/kyaml/errors"
+	"sigs.k8s.io/kustomize/kyaml/setters2"
+	"sigs.k8s.io/kustomize/kyaml/setters2/settersutil"
+)
+
+// NewCreateSubstitutionRunner returns a command runner.
+func NewCreateSubstitutionRunner(parent string) *CreateSubstitutionRunner {
+	r := &CreateSubstitutionRunner{}
+	cs := &cobra.Command{
+		Use:     "create-subst DIR NAME VALUE",
+		Args:    cobra.ExactArgs(3),
+		PreRunE: r.preRunE,
+		RunE:    r.runE,
+	}
+	cs.Flags().StringVar(&r.CreateSubstitution.FieldName, "field", "",
+		"name of the field to set -- e.g. --field port")
+	cs.Flags().StringVar(&r.CreateSubstitution.Pattern, "pattern", "",
+		"substitution pattern")
+	cs.Flags().StringSliceVar(&r.Values, "value", []string{""},
+		"substitution values for the pattern.  format is PATTERN_MARKER=SETTER_NAME"+
+			"where PATTERN_MARKER is the pattern substring to replace, and SETTER_NAME is the"+
+			"setter from which to take the replacement value.")
+	_ = cs.MarkFlagRequired("pattern")
+	fixDocs(parent, cs)
+	r.Command = cs
+	return r
+}
+
+func CreateSubstitutionCommand(parent string) *cobra.Command {
+	return NewCreateSubstitutionRunner(parent).Command
+}
+
+type CreateSubstitutionRunner struct {
+	Command            *cobra.Command
+	CreateSubstitution settersutil.SubstitutionCreator
+	OpenAPIFile        string
+	Values             []string
+}
+
+func (r *CreateSubstitutionRunner) runE(c *cobra.Command, args []string) error {
+	return handleError(c, r.CreateSubstitution.Create(r.OpenAPIFile, args[0]))
+}
+
+func (r *CreateSubstitutionRunner) preRunE(c *cobra.Command, args []string) error {
+	var err error
+	r.CreateSubstitution.Name = args[1]
+	r.CreateSubstitution.FieldValue = args[2]
+	if err != nil {
+		return err
+	}
+
+	r.OpenAPIFile, err = ext.GetOpenAPIFile(args)
+	if err != nil {
+		return err
+	}
+
+	// parse the marker values
+	for i := range r.Values {
+		parts := strings.SplitN(r.Values[i], "=", 2)
+		if len(parts) < 2 {
+			return errors.Errorf("values must be specified as PATTERN_MARKER=SETTER_NAME")
+		}
+		ref := setters2.DefinitionsPrefix + setters2.SetterDefinitionPrefix + parts[1]
+		r.CreateSubstitution.Values = append(
+			r.CreateSubstitution.Values,
+			setters2.Value{Marker: parts[0], Ref: ref},
+		)
+	}
+
+	return nil
+}

cmd/config/internal/commands/cmdcreatesubstitution_test.go

@@ -0,0 +1,240 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package commands_test
+
+import (
+	"bytes"
+	"io/ioutil"
+	"os"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"sigs.k8s.io/kustomize/cmd/config/ext"
+	"sigs.k8s.io/kustomize/cmd/config/internal/commands"
+	"sigs.k8s.io/kustomize/kyaml/openapi"
+)
+
+func TestCreateSubstitutionCommand(t *testing.T) {
+	var tests = []struct {
+		name              string
+		inputOpenAPI      string
+		input             string
+		args              []string
+		out               string
+		expectedOpenAPI   string
+		expectedResources string
+	}{
+		{
+			name: "substitution replicas",
+			args: []string{
+				"image", "nginx:1.7.9", "--pattern", "IMAGE:TAG",
+				"--value", "IMAGE=image", "--value", "TAG=tag"},
+			input: `
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: nginx-deployment
+spec:
+  replicas: 3
+  template:
+    spec:
+      containers:
+      - name: nginx
+        image: nginx:1.7.9
+      - name: sidecar
+        image: sidecar:1.7.9
+ `,
+			inputOpenAPI: `
+apiVersion: v1alpha1
+kind: Example
+openAPI:
+  definitions:
+    io.k8s.cli.setters.image:
+      x-k8s-cli:
+        setter:
+          name: image
+          value: "nginx"
+    io.k8s.cli.setters.tag:
+      x-k8s-cli:
+        setter:
+          name: tag
+          value: "1.7.9"
+ `,
+			expectedOpenAPI: `
+apiVersion: v1alpha1
+kind: Example
+openAPI:
+  definitions:
+    io.k8s.cli.setters.image:
+      x-k8s-cli:
+        setter:
+          name: image
+          value: "nginx"
+    io.k8s.cli.setters.tag:
+      x-k8s-cli:
+        setter:
+          name: tag
+          value: "1.7.9"
+    io.k8s.cli.substitutions.image:
+      x-k8s-cli:
+        substitution:
+          name: image
+          pattern: IMAGE:TAG
+          values:
+          - marker: IMAGE
+            ref: '#/definitions/io.k8s.cli.setters.image'
+          - marker: TAG
+            ref: '#/definitions/io.k8s.cli.setters.tag'
+ `,
+			expectedResources: `
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: nginx-deployment
+spec:
+  replicas: 3
+  template:
+    spec:
+      containers:
+      - name: nginx
+        image: nginx:1.7.9 # {"$ref":"#/definitions/io.k8s.cli.substitutions.image"}
+      - name: sidecar
+        image: sidecar:1.7.9
+ `,
+		},
+		{
+			name: "substitution and create setters 1",
+			args: []string{
+				"image", "something/nginx::1.7.9/nginxotherthing", "--pattern", "something/IMAGE::TAG/nginxotherthing",
+				"--value", "IMAGE=image", "--value", "TAG=tag"},
+			input: `
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: nginx-deployment
+spec:
+  replicas: 3
+  template:
+    spec:
+      containers:
+      - name: nginx
+        image: something/nginx::1.7.9/nginxotherthing
+      - name: sidecar
+        image: sidecar:1.7.9
+ `,
+			inputOpenAPI: `
+apiVersion: v1alpha1
+kind: Example
+ `,
+			expectedOpenAPI: `
+apiVersion: v1alpha1
+kind: Example
+openAPI:
+  definitions:
+    io.k8s.cli.setters.image:
+      x-k8s-cli:
+        setter:
+          name: image
+          value: nginx
+    io.k8s.cli.setters.tag:
+      x-k8s-cli:
+        setter:
+          name: tag
+          value: 1.7.9
+    io.k8s.cli.substitutions.image:
+      x-k8s-cli:
+        substitution:
+          name: image
+          pattern: something/IMAGE::TAG/nginxotherthing
+          values:
+          - marker: IMAGE
+            ref: '#/definitions/io.k8s.cli.setters.image'
+          - marker: TAG
+            ref: '#/definitions/io.k8s.cli.setters.tag'
+ `,
+			expectedResources: `
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: nginx-deployment
+spec:
+  replicas: 3
+  template:
+    spec:
+      containers:
+      - name: nginx
+        image: something/nginx::1.7.9/nginxotherthing # {"$ref":"#/definitions/io.k8s.cli.substitutions.image"}
+      - name: sidecar
+        image: sidecar:1.7.9
+ `,
+		},
+	}
+	for i := range tests {
+		test := tests[i]
+		t.Run(test.name, func(t *testing.T) {
+			// reset the openAPI afterward
+			openapi.ResetOpenAPI()
+			defer openapi.ResetOpenAPI()
+
+			f, err := ioutil.TempFile("", "k8s-cli-")
+			if !assert.NoError(t, err) {
+				t.FailNow()
+			}
+			defer os.Remove(f.Name())
+			err = ioutil.WriteFile(f.Name(), []byte(test.inputOpenAPI), 0600)
+			if !assert.NoError(t, err) {
+				t.FailNow()
+			}
+			old := ext.GetOpenAPIFile
+			defer func() { ext.GetOpenAPIFile = old }()
+			ext.GetOpenAPIFile = func(args []string) (s string, err error) {
+				return f.Name(), nil
+			}
+
+			r, err := ioutil.TempFile("", "k8s-cli-*.yaml")
+			if !assert.NoError(t, err) {
+				t.FailNow()
+			}
+			defer os.Remove(r.Name())
+			err = ioutil.WriteFile(r.Name(), []byte(test.input), 0600)
+			if !assert.NoError(t, err) {
+				t.FailNow()
+			}
+
+			runner := commands.NewCreateSubstitutionRunner("")
+			out := &bytes.Buffer{}
+			runner.Command.SetOut(out)
+			runner.Command.SetArgs(append([]string{r.Name()}, test.args...))
+			err = runner.Command.Execute()
+			if !assert.NoError(t, err) {
+				t.FailNow()
+			}
+
+			if !assert.Equal(t, test.out, out.String()) {
+				t.FailNow()
+			}
+
+			actualResources, err := ioutil.ReadFile(r.Name())
+			if !assert.NoError(t, err) {
+				t.FailNow()
+			}
+			if !assert.Equal(t,
+				strings.TrimSpace(test.expectedResources),
+				strings.TrimSpace(string(actualResources))) {
+				t.FailNow()
+			}
+
+			actualOpenAPI, err := ioutil.ReadFile(f.Name())
+			if !assert.NoError(t, err) {
+				t.FailNow()
+			}
+			if !assert.Equal(t,
+				strings.TrimSpace(test.expectedOpenAPI),
+				strings.TrimSpace(string(actualOpenAPI))) {
+				t.FailNow()
+			}
+		})
+	}
+}

cmd/config/internal/commands/cmdlistsetters.go

@@ -4,9 +4,17 @@
 package commands
 
 import (
+	"fmt"
+	"io"
+	"os"
+	"strings"
+
+	"github.com/olekukonko/tablewriter"
 	"github.com/spf13/cobra"
+	"sigs.k8s.io/kustomize/cmd/config/ext"
 	"sigs.k8s.io/kustomize/cmd/config/internal/generateddocs/commands"
 	"sigs.k8s.io/kustomize/kyaml/setters"
+	"sigs.k8s.io/kustomize/kyaml/setters2"
 )
 
 // NewListSettersRunner returns a command runner.
@@ -33,15 +41,64 @@ func ListSettersCommand(parent string) *cobra.Command {
 type ListSettersRunner struct {
 	Command *cobra.Command
 	Lookup  setters.LookupSetters
+	List    setters2.List
 }
 
 func (r *ListSettersRunner) preRunE(c *cobra.Command, args []string) error {
 	if len(args) > 1 {
 		r.Lookup.Name = args[1]
+		r.List.Name = args[1]
 	}
+
+	initSetterVersion(c, args)
 	return nil
 }
 
 func (r *ListSettersRunner) runE(c *cobra.Command, args []string) error {
+	if setterVersion == "v2" {
+		// use setters v2
+		path, err := ext.GetOpenAPIFile(args)
+		if err != nil {
+			return err
+		}
+		if err := r.List.List(path, args[0]); err != nil {
+			return err
+		}
+		table := newTable(c.OutOrStdout())
+		table.SetHeader([]string{"NAME", "VALUE", "SET BY", "DESCRIPTION", "COUNT"})
+		for i := range r.List.Setters {
+			s := r.List.Setters[i]
+			v := s.Value
+
+			// if the setter is for a list, populate the values
+			if len(s.ListValues) > 0 {
+				v = strings.Join(s.ListValues, ",")
+				v = fmt.Sprintf("[%s]", v)
+			}
+			table.Append([]string{
+				s.Name, v, s.SetBy, s.Description, fmt.Sprintf("%d", s.Count)})
+		}
+		table.Render()
+
+		if len(r.List.Setters) == 0 {
+			// exit non-0 if no matching setters are found
+			if ExitOnError {
+				os.Exit(1)
+			}
+		}
+		return nil
+	}
+
 	return handleError(c, lookup(r.Lookup, c, args))
 }
+
+func newTable(o io.Writer) *tablewriter.Table {
+	table := tablewriter.NewWriter(o)
+	table.SetRowLine(false)
+	table.SetBorder(false)
+	table.SetHeaderLine(false)
+	table.SetColumnSeparator(" ")
+	table.SetCenterSeparator(" ")
+	table.SetAlignment(tablewriter.ALIGN_LEFT)
+	return table
+}

cmd/config/internal/commands/cmdlistsetters_test.go

@@ -0,0 +1,299 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package commands_test
+
+import (
+	"bytes"
+	"io/ioutil"
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"sigs.k8s.io/kustomize/cmd/config/ext"
+	"sigs.k8s.io/kustomize/cmd/config/internal/commands"
+	"sigs.k8s.io/kustomize/kyaml/openapi"
+)
+
+func TestListSettersCommand(t *testing.T) {
+	var tests = []struct {
+		name     string
+		openapi  string
+		input    string
+		args     []string
+		expected string
+	}{
+		{
+			name: "list-replicas",
+			openapi: `
+openAPI:
+  definitions:
+    io.k8s.cli.setters.replicas:
+      x-k8s-cli:
+        setter:
+          name: replicas
+          value: "3"
+          setBy: me
+      description: "hello world"
+ `,
+			input: `
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: nginx-deployment
+spec:
+  replicas: 3 # {"$ref": "#/definitions/io.k8s.cli.setters.replicas"}
+ `,
+			expected: `    NAME     VALUE   SET BY   DESCRIPTION   COUNT  
+  replicas   3       me       hello world   1      
+`,
+		},
+		{
+			name: "list-multiple",
+			openapi: `
+openAPI:
+  definitions:
+    io.k8s.cli.setters.replicas:
+      description: "hello world 1"
+      x-k8s-cli:
+        setter:
+          name: replicas
+          value: "3"
+          setBy: me1
+    io.k8s.cli.setters.image:
+      description: "hello world 2"
+      x-k8s-cli:
+        setter:
+          name: image
+          value: "nginx"
+          setBy: me2
+    io.k8s.cli.setters.tag:
+      description: "hello world 3"
+      x-k8s-cli:
+        setter:
+          name: tag
+          value: "1.7.9"
+          setBy: me3
+    io.k8s.cli.substitutions.image:
+      x-k8s-cli:
+        substitution:
+          name: image
+          pattern: IMAGE:TAG
+          values:
+          - marker: IMAGE
+            ref: '#/definitions/io.k8s.cli.setters.image'
+          - marker: TAG
+            ref: '#/definitions/io.k8s.cli.setters.tag'
+ `,
+			input: `
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: nginx-deployment
+spec:
+  replicas: 3 # {"$ref": "#/definitions/io.k8s.cli.setters.replicas"}
+  template:
+    spec:
+      containers:
+      - name: nginx
+        image: nginx:1.7.9 # {"$ref": "#/definitions/io.k8s.cli.substitutions.image"}
+      - name: nginx2
+        image: nginx # {"$ref": "#/definitions/io.k8s.cli.setters.image"}
+ `,
+			expected: `    NAME     VALUE   SET BY    DESCRIPTION    COUNT  
+  image      nginx   me2      hello world 2   2      
+  replicas   3       me1      hello world 1   1      
+  tag        1.7.9   me3      hello world 3   1      
+`,
+		},
+		{
+			name: "list-multiple-resources",
+			openapi: `
+openAPI:
+  definitions:
+    io.k8s.cli.setters.replicas:
+      description: "hello world 1"
+      x-k8s-cli:
+        setter:
+          name: replicas
+          value: "3"
+          setBy: me1
+    io.k8s.cli.setters.image:
+      description: "hello world 2"
+      x-k8s-cli:
+        setter:
+          name: image
+          value: "nginx"
+          setBy: me2
+    io.k8s.cli.setters.tag:
+      description: "hello world 3"
+      x-k8s-cli:
+        setter:
+          name: tag
+          value: "1.7.9"
+          setBy: me3
+    io.k8s.cli.substitutions.image:
+      x-k8s-cli:
+        substitution:
+          name: image
+          pattern: IMAGE:TAG
+          values:
+          - marker: IMAGE
+            ref: '#/definitions/io.k8s.cli.setters.image'
+          - marker: TAG
+            ref: '#/definitions/io.k8s.cli.setters.tag'
+ `,
+			input: `
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: nginx-deployment-1
+spec:
+  replicas: 3 # {"$ref": "#/definitions/io.k8s.cli.setters.replicas"}
+  template:
+    spec:
+      containers:
+      - name: nginx
+        image: nginx:1.7.9 # {"$ref": "#/definitions/io.k8s.cli.substitutions.image"}
+      - name: nginx2
+        image: nginx # {"$ref": "#/definitions/io.k8s.cli.setters.image"}
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: nginx-deployment-2
+spec:
+  replicas: 3 # {"$ref": "#/definitions/io.k8s.cli.setters.replicas"}
+  template:
+    spec:
+      containers:
+      - name: nginx
+        image: nginx:1.7.9 # {"$ref": "#/definitions/io.k8s.cli.substitutions.image"}
+      - name: nginx2
+        image: nginx
+`,
+			expected: `    NAME     VALUE   SET BY    DESCRIPTION    COUNT  
+  image      nginx   me2      hello world 2   3      
+  replicas   3       me1      hello world 1   2      
+  tag        1.7.9   me3      hello world 3   2      
+`,
+		},
+		{
+			name: "list-name",
+			args: []string{"image"},
+			openapi: `
+openAPI:
+  definitions:
+    io.k8s.cli.setters.replicas:
+      description: "hello world 1"
+      x-k8s-cli:
+        setter:
+          name: replicas
+          value: "3"
+          setBy: me1
+    io.k8s.cli.setters.image:
+      description: "hello world 2"
+      x-k8s-cli:
+        setter:
+          name: image
+          value: "nginx"
+          setBy: me2
+    io.k8s.cli.setters.tag:
+      description: "hello world 3"
+      x-k8s-cli:
+        setter:
+          name: tag
+          value: "1.7.9"
+          setBy: me3
+    io.k8s.cli.substitutions.image:
+      x-k8s-cli:
+        substitution:
+          name: image
+          pattern: IMAGE:TAG
+          values:
+          - marker: IMAGE
+            ref: '#/definitions/io.k8s.cli.setters.image'
+          - marker: TAG
+            ref: '#/definitions/io.k8s.cli.setters.tag'
+ `,
+			input: `
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: nginx-deployment-1
+spec:
+  replicas: 3 # {"$ref": "#/definitions/io.k8s.cli.setters.replicas"}
+  template:
+    spec:
+      containers:
+      - name: nginx
+        image: nginx:1.7.9 # {"$ref": "#/definitions/io.k8s.cli.substitutions.image"}
+      - name: nginx2
+        image: nginx # {"$ref": "#/definitions/io.k8s.cli.setters.image"}
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: nginx-deployment-2
+spec:
+  replicas: 3 # {"$ref": "#/definitions/io.k8s.cli.setters.replicas"}
+  template:
+    spec:
+      containers:
+      - name: nginx
+        image: nginx:1.7.9 # {"$ref": "#/definitions/io.k8s.cli.substitutions.image"}
+      - name: nginx2
+        image: nginx
+`,
+			expected: `  NAME    VALUE   SET BY    DESCRIPTION    COUNT  
+  image   nginx   me2      hello world 2   3      
+`,
+		},
+	}
+	for i := range tests {
+		test := tests[i]
+		t.Run(test.name, func(t *testing.T) {
+			// reset the openAPI afterward
+			openapi.ResetOpenAPI()
+			defer openapi.ResetOpenAPI()
+
+			f, err := ioutil.TempFile("", "k8s-cli-")
+			if !assert.NoError(t, err) {
+				t.FailNow()
+			}
+			defer os.Remove(f.Name())
+			old := ext.GetOpenAPIFile
+			defer func() { ext.GetOpenAPIFile = old }()
+			ext.GetOpenAPIFile = func(args []string) (s string, err error) {
+				err = ioutil.WriteFile(f.Name(), []byte(test.openapi), 0600)
+				if !assert.NoError(t, err) {
+					t.FailNow()
+				}
+				return f.Name(), nil
+			}
+
+			r, err := ioutil.TempFile("", "k8s-cli-*.yaml")
+			if !assert.NoError(t, err) {
+				t.FailNow()
+			}
+			defer os.Remove(r.Name())
+			err = ioutil.WriteFile(r.Name(), []byte(test.input), 0600)
+			if !assert.NoError(t, err) {
+				t.FailNow()
+			}
+
+			runner := commands.NewListSettersRunner("")
+			actual := &bytes.Buffer{}
+			runner.Command.SetOut(actual)
+			runner.Command.SetArgs(append([]string{r.Name()}, test.args...))
+			err = runner.Command.Execute()
+			if !assert.NoError(t, err) {
+				t.FailNow()
+			}
+
+			if !assert.Equal(t, test.expected, actual.String()) {
+				t.FailNow()
+			}
+		})
+	}
+}

cmd/config/internal/commands/cmdset.go

@@ -9,17 +9,19 @@ import (
 
 	"github.com/olekukonko/tablewriter"
 	"github.com/spf13/cobra"
+	"sigs.k8s.io/kustomize/cmd/config/ext"
 	"sigs.k8s.io/kustomize/cmd/config/internal/generateddocs/commands"
 	"sigs.k8s.io/kustomize/kyaml/kio"
 	"sigs.k8s.io/kustomize/kyaml/setters"
+	"sigs.k8s.io/kustomize/kyaml/setters2/settersutil"
 )
 
 // NewSetRunner returns a command runner.
 func NewSetRunner(parent string) *SetRunner {
 	r := &SetRunner{}
 	c := &cobra.Command{
-		Use:     "set DIR [NAME] [VALUE]",
-		Args:    cobra.RangeArgs(1, 3),
+		Use:     "set DIR NAME [VALUE]",
+		Args:    cobra.MinimumNArgs(3),
 		Short:   commands.SetShort,
 		Long:    commands.SetLong,
 		Example: commands.SetExamples,
@@ -32,18 +34,44 @@ func NewSetRunner(parent string) *SetRunner {
 		"annotate the field with who set it")
 	c.Flags().StringVar(&r.Perform.Description, "description", "",
 		"annotate the field with a description of its value")
+	c.Flags().StringVar(&setterVersion, "version", "",
+		"use this version of the setter format")
+	c.Flags().MarkHidden("version")
 
 	return r
 }
 
+var setterVersion string
+
 func SetCommand(parent string) *cobra.Command {
 	return NewSetRunner(parent).Command
 }
 
 type SetRunner struct {
-	Command *cobra.Command
-	Lookup  setters.LookupSetters
-	Perform setters.PerformSetters
+	Command     *cobra.Command
+	Lookup      setters.LookupSetters
+	Perform     setters.PerformSetters
+	Set         settersutil.FieldSetter
+	OpenAPIFile string
+}
+
+func initSetterVersion(c *cobra.Command, args []string) error {
+	setterVersion = "v2"
+	l := setters.LookupSetters{}
+
+	// backwards compatibility for resources with setter v1
+	err := kio.Pipeline{
+		Inputs:  []kio.Reader{&kio.LocalPackageReader{PackagePath: args[0]}},
+		Filters: []kio.Filter{&l},
+	}.Execute()
+	if err != nil {
+		return err
+	}
+	if len(l.SetterCounts) > 0 {
+		setterVersion = "v1"
+	}
+
+	return nil
 }
 
 func (r *SetRunner) preRunE(c *cobra.Command, args []string) error {
@@ -55,15 +83,42 @@ func (r *SetRunner) preRunE(c *cobra.Command, args []string) error {
 		r.Perform.Value = args[2]
 	}
 
+	if setterVersion == "" {
+		if len(args) < 3 {
+			setterVersion = "v1"
+		} else if err := initSetterVersion(c, args); err != nil {
+			return err
+		}
+	}
+	if setterVersion == "v2" {
+		var err error
+		r.Set.Name = args[1]
+		r.Set.Value = args[2]
+
+		// set remaining values as list values
+		if len(args) > 3 {
+			r.Set.ListValues = args[3:]
+		}
+		r.Set.Description = r.Perform.Description
+		r.Set.SetBy = r.Perform.SetBy
+		r.OpenAPIFile, err = ext.GetOpenAPIFile(args)
+		if err != nil {
+			return err
+		}
+	}
+
 	return nil
 }
 
 func (r *SetRunner) runE(c *cobra.Command, args []string) error {
-
+	if setterVersion == "v2" {
+		count, err := r.Set.Set(r.OpenAPIFile, args[0])
+		fmt.Fprintf(c.OutOrStdout(), "set %d fields\n", count)
+		return handleError(c, err)
+	}
 	if len(args) == 3 {
 		return handleError(c, r.perform(c, args))
 	}
-
 	return handleError(c, lookup(r.Lookup, c, args))
 }
 

cmd/config/internal/commands/cmdset_test.go

@@ -0,0 +1,277 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package commands_test
+
+import (
+	"bytes"
+	"io/ioutil"
+	"os"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"sigs.k8s.io/kustomize/cmd/config/ext"
+	"sigs.k8s.io/kustomize/cmd/config/internal/commands"
+	"sigs.k8s.io/kustomize/kyaml/openapi"
+)
+
+func TestSetCommand(t *testing.T) {
+	var tests = []struct {
+		name              string
+		inputOpenAPI      string
+		input             string
+		args              []string
+		out               string
+		expectedOpenAPI   string
+		expectedResources string
+	}{
+		{
+			name: "set replicas",
+			args: []string{"replicas", "4", "--description", "hi there", "--set-by", "pw"},
+			out:  "set 1 fields\n",
+			inputOpenAPI: `
+apiVersion: v1alpha1
+kind: Example
+openAPI:
+  definitions:
+    io.k8s.cli.setters.replicas:
+      description: hello world
+      x-k8s-cli:
+        setter:
+          name: replicas
+          value: "3"
+          setBy: me
+ `,
+			input: `
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: nginx-deployment
+spec:
+  replicas: 3 # {"$ref":"#/definitions/io.k8s.cli.setters.replicas"}
+ `,
+			expectedOpenAPI: `
+apiVersion: v1alpha1
+kind: Example
+openAPI:
+  definitions:
+    io.k8s.cli.setters.replicas:
+      description: hi there
+      x-k8s-cli:
+        setter:
+          name: replicas
+          value: "4"
+          setBy: pw
+ `,
+			expectedResources: `
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: nginx-deployment
+spec:
+  replicas: 4 # {"$ref":"#/definitions/io.k8s.cli.setters.replicas"}
+ `,
+		},
+		{
+			name: "set replicas no description",
+			args: []string{"replicas", "4"},
+			out:  "set 1 fields\n",
+			inputOpenAPI: `
+apiVersion: v1alpha1
+kind: Example
+openAPI:
+  definitions:
+    io.k8s.cli.setters.replicas:
+      description: hello world
+      x-k8s-cli:
+        setter:
+          name: replicas
+          value: "3"
+          setBy: me
+ `,
+			input: `
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: nginx-deployment
+spec:
+  replicas: 3 # {"$ref":"#/definitions/io.k8s.cli.setters.replicas"}
+ `,
+			expectedOpenAPI: `
+apiVersion: v1alpha1
+kind: Example
+openAPI:
+  definitions:
+    io.k8s.cli.setters.replicas:
+      description: hello world
+      x-k8s-cli:
+        setter:
+          name: replicas
+          value: "4"
+ `,
+			expectedResources: `
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: nginx-deployment
+spec:
+  replicas: 4 # {"$ref":"#/definitions/io.k8s.cli.setters.replicas"}
+ `,
+		},
+		{
+			name: "set image",
+			args: []string{"tag", "1.8.1"},
+			out:  "set 1 fields\n",
+			inputOpenAPI: `
+apiVersion: v1alpha1
+kind: Example
+openAPI:
+  definitions:
+    io.k8s.cli.setters.image:
+      x-k8s-cli:
+        setter:
+          name: image
+          value: "nginx"
+    io.k8s.cli.setters.tag:
+      x-k8s-cli:
+        setter:
+          name: tag
+          value: "1.7.9"
+    io.k8s.cli.substitutions.image:
+      x-k8s-cli:
+        substitution:
+          name: image
+          pattern: IMAGE:TAG
+          values:
+          - marker: IMAGE
+            ref: '#/definitions/io.k8s.cli.setters.image'
+          - marker: TAG
+            ref: '#/definitions/io.k8s.cli.setters.tag'
+ `,
+			input: `
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: nginx-deployment
+spec:
+  replicas: 3
+  template:
+    spec:
+      containers:
+      - name: nginx
+        image: nginx:1.7.9 # {"$ref":"#/definitions/io.k8s.cli.substitutions.image"}
+      - name: sidecar
+        image: sidecar:1.7.9
+ `,
+			expectedOpenAPI: `
+apiVersion: v1alpha1
+kind: Example
+openAPI:
+  definitions:
+    io.k8s.cli.setters.image:
+      x-k8s-cli:
+        setter:
+          name: image
+          value: "nginx"
+    io.k8s.cli.setters.tag:
+      x-k8s-cli:
+        setter:
+          name: tag
+          value: "1.8.1"
+    io.k8s.cli.substitutions.image:
+      x-k8s-cli:
+        substitution:
+          name: image
+          pattern: IMAGE:TAG
+          values:
+          - marker: IMAGE
+            ref: '#/definitions/io.k8s.cli.setters.image'
+          - marker: TAG
+            ref: '#/definitions/io.k8s.cli.setters.tag'
+
+ `,
+			expectedResources: `
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: nginx-deployment
+spec:
+  replicas: 3
+  template:
+    spec:
+      containers:
+      - name: nginx
+        image: nginx:1.8.1 # {"$ref":"#/definitions/io.k8s.cli.substitutions.image"}
+      - name: sidecar
+        image: sidecar:1.7.9
+`,
+		},
+	}
+	for i := range tests {
+		test := tests[i]
+		t.Run(test.name, func(t *testing.T) {
+			// reset the openAPI afterward
+			openapi.ResetOpenAPI()
+			defer openapi.ResetOpenAPI()
+
+			f, err := ioutil.TempFile("", "k8s-cli-")
+			if !assert.NoError(t, err) {
+				t.FailNow()
+			}
+			defer os.Remove(f.Name())
+			err = ioutil.WriteFile(f.Name(), []byte(test.inputOpenAPI), 0600)
+			if !assert.NoError(t, err) {
+				t.FailNow()
+			}
+			old := ext.GetOpenAPIFile
+			defer func() { ext.GetOpenAPIFile = old }()
+			ext.GetOpenAPIFile = func(args []string) (s string, err error) {
+				return f.Name(), nil
+			}
+
+			r, err := ioutil.TempFile("", "k8s-cli-*.yaml")
+			if !assert.NoError(t, err) {
+				t.FailNow()
+			}
+			defer os.Remove(r.Name())
+			err = ioutil.WriteFile(r.Name(), []byte(test.input), 0600)
+			if !assert.NoError(t, err) {
+				t.FailNow()
+			}
+
+			runner := commands.NewSetRunner("")
+			out := &bytes.Buffer{}
+			runner.Command.SetOut(out)
+			runner.Command.SetArgs(append([]string{r.Name()}, test.args...))
+			err = runner.Command.Execute()
+			if !assert.NoError(t, err) {
+				t.FailNow()
+			}
+
+			if !assert.Equal(t, test.out, out.String()) {
+				t.FailNow()
+			}
+
+			actualResources, err := ioutil.ReadFile(r.Name())
+			if !assert.NoError(t, err) {
+				t.FailNow()
+			}
+			if !assert.Equal(t,
+				strings.TrimSpace(test.expectedResources),
+				strings.TrimSpace(string(actualResources))) {
+				t.FailNow()
+			}
+
+			actualOpenAPI, err := ioutil.ReadFile(f.Name())
+			if !assert.NoError(t, err) {
+				t.FailNow()
+			}
+			if !assert.Equal(t,
+				strings.TrimSpace(test.expectedOpenAPI),
+				strings.TrimSpace(string(actualOpenAPI))) {
+				t.FailNow()
+			}
+		})
+	}
+}

cmd/config/internal/commands/fmt.go

@@ -31,6 +31,8 @@ formatting substitution verbs {'%n': 'metadata.name', '%s': 'metadata.namespace'
 		`if true, keep index and filename annotations set on Resources.`)
 	c.Flags().BoolVar(&r.Override, "override", false,
 		`if true, override existing filepath annotations.`)
+	c.Flags().BoolVar(&r.UseSchema, "use-schema", false,
+		`if true, uses openapi resource schema to format resources.`)
 	r.Command = c
 	return r
 }
@@ -46,6 +48,7 @@ type FmtRunner struct {
 	SetFilenames    bool
 	KeepAnnotations bool
 	Override        bool
+	UseSchema       bool
 }
 
 func (r *FmtRunner) preRunE(c *cobra.Command, args []string) error {
@@ -56,7 +59,9 @@ func (r *FmtRunner) preRunE(c *cobra.Command, args []string) error {
 }
 
 func (r *FmtRunner) runE(c *cobra.Command, args []string) error {
-	f := []kio.Filter{filters.FormatFilter{}}
+	f := []kio.Filter{filters.FormatFilter{
+		UseSchema: r.UseSchema,
+	}}
 
 	// format with file names
 	if r.SetFilenames {

cmd/config/internal/commands/fmt_test.go

@@ -143,6 +143,8 @@ func TestCmd_failFiles(t *testing.T) {
 	// fmt the files
 	r := commands.GetFmtRunner("")
 	r.Command.SetArgs([]string{"notrealfile"})
+	r.Command.SilenceUsage = true
+	r.Command.SilenceErrors = true
 	err := r.Command.Execute()
 	assert.EqualError(t, err, "lstat notrealfile: no such file or directory")
 }

cmd/config/internal/commands/merge3.go

@@ -25,6 +25,8 @@ func GetMerge3Runner(name string) *Merge3Runner {
 		"Path to updated package")
 	c.Flags().StringVar(&r.toDir, "to", "",
 		"Path to destination package")
+	c.Flags().BoolVar(&r.path, "path-merge-key", false,
+		"Use the path as part of the merge key when merging resources")
 
 	r.Command = c
 	return r
@@ -40,6 +42,7 @@ type Merge3Runner struct {
 	ancestor string
 	fromDir  string
 	toDir    string
+	path     bool
 }
 
 func (r *Merge3Runner) runE(c *cobra.Command, args []string) error {
@@ -47,6 +50,7 @@ func (r *Merge3Runner) runE(c *cobra.Command, args []string) error {
 		OriginalPath: r.ancestor,
 		UpdatedPath:  r.fromDir,
 		DestPath:     r.toDir,
+		MergeOnPath:  r.path,
 	}.Merge()
 	if err != nil {
 		return err

cmd/config/internal/commands/run-fns.go

@@ -4,22 +4,27 @@
 package commands
 
 import (
+	"fmt"
+	"io"
+	"strings"
+
 	"github.com/spf13/cobra"
 	"sigs.k8s.io/kustomize/cmd/config/internal/generateddocs/commands"
+	"sigs.k8s.io/kustomize/kyaml/errors"
 	"sigs.k8s.io/kustomize/kyaml/runfn"
+	"sigs.k8s.io/kustomize/kyaml/yaml"
 )
 
 // GetCatRunner returns a RunFnRunner.
 func GetRunFnRunner(name string) *RunFnRunner {
 	r := &RunFnRunner{}
 	c := &cobra.Command{
-		Use:     "run DIR",
-		Aliases: []string{"run-fns"},
+		Use:     "run [DIR]",
 		Short:   commands.RunFnsShort,
 		Long:    commands.RunFnsLong,
 		Example: commands.RunFnsExamples,
 		RunE:    r.runE,
-		Args:    cobra.ExactArgs(1),
+		PreRunE: r.preRunE,
 	}
 	fixDocs(name, c)
 	c.Flags().BoolVar(&r.IncludeSubpackages, "include-subpackages", true,
@@ -27,11 +32,18 @@ func GetRunFnRunner(name string) *RunFnRunner {
 	r.Command = c
 	r.Command.Flags().BoolVar(
 		&r.DryRun, "dry-run", false, "print results to stdout")
+	r.Command.Flags().BoolVar(
+		&r.GlobalScope, "global-scope", false, "set global scope for functions.")
 	r.Command.Flags().StringSliceVar(
 		&r.FnPaths, "fn-path", []string{},
-		"directories containing functions without configuration")
-	r.Command.AddCommand(XArgsCommand())
-	r.Command.AddCommand(WrapCommand())
+		"read functions from these directories instead of the configuration directory.")
+	r.Command.Flags().StringVar(
+		&r.Image, "image", "",
+		"run this image as a function instead of discovering them.")
+	r.Command.Flags().BoolVar(
+		&r.Network, "network", false, "enable network access for functions that declare it")
+	r.Command.Flags().StringVar(
+		&r.NetworkName, "network-name", "bridge", "the docker network to run the container in")
 	return r
 }
 
@@ -44,13 +56,156 @@ type RunFnRunner struct {
 	IncludeSubpackages bool
 	Command            *cobra.Command
 	DryRun             bool
+	GlobalScope        bool
 	FnPaths            []string
+	Image              string
+	RunFns             runfn.RunFns
+	Network            bool
+	NetworkName        string
 }
 
 func (r *RunFnRunner) runE(c *cobra.Command, args []string) error {
-	rec := runfn.RunFns{Path: args[0], FunctionPaths: r.FnPaths}
-	if r.DryRun {
-		rec.Output = c.OutOrStdout()
-	}
-	return handleError(c, rec.Execute())
+	return handleError(c, r.RunFns.Execute())
+}
+
+// getFunctions parses the commandline flags and arguments into explicit
+// Functions to run.
+func (r *RunFnRunner) getFunctions(c *cobra.Command, args, dataItems []string) (
+	[]*yaml.RNode, error) {
+	// if image isn't specified, then Functions is empty
+	if r.Image == "" {
+		return nil, nil
+	}
+
+	// create the function spec to set as an annotation
+	fn, err := yaml.Parse(`container: {}`)
+	if err != nil {
+		return nil, err
+	}
+	// TODO: add support network, volumes, etc based on flag values
+	err = fn.PipeE(
+		yaml.Lookup("container"),
+		yaml.SetField("image", yaml.NewScalarRNode(r.Image)))
+	if err != nil {
+		return nil, err
+	}
+	if r.Network {
+		err = fn.PipeE(
+			yaml.LookupCreate(yaml.MappingNode, "container", "network"),
+			yaml.SetField("required", yaml.NewScalarRNode("true")))
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	// create the function config
+	rc, err := yaml.Parse(`
+metadata:
+  name: function-input
+data: {}
+`)
+	if err != nil {
+		return nil, err
+	}
+
+	// set the function annotation on the function config so it
+	// is parsed by RunFns
+	value, err := fn.String()
+	if err != nil {
+		return nil, err
+	}
+	err = rc.PipeE(
+		yaml.LookupCreate(yaml.MappingNode, "metadata", "annotations"),
+		yaml.SetField("config.kubernetes.io/function", yaml.NewScalarRNode(value)))
+	if err != nil {
+		return nil, err
+	}
+
+	// default the function config kind to ConfigMap, this may be overridden
+	var kind = "ConfigMap"
+	var version = "v1"
+
+	// populate the function config with data.  this is a convention for functions
+	// to be more commandline friendly
+	if len(dataItems) > 0 {
+		dataField, err := rc.Pipe(yaml.Lookup("data"))
+		if err != nil {
+			return nil, err
+		}
+		for i, s := range dataItems {
+			kv := strings.SplitN(s, "=", 2)
+			if i == 0 && len(kv) == 1 {
+				// first argument may be the kind
+				kind = s
+				continue
+			}
+			if len(kv) != 2 {
+				return nil, fmt.Errorf("args must have keys and values separated by =")
+			}
+			err := dataField.PipeE(yaml.SetField(kv[0], yaml.NewScalarRNode(kv[1])))
+			if err != nil {
+				return nil, err
+			}
+		}
+	}
+	err = rc.PipeE(yaml.SetField("kind", yaml.NewScalarRNode(kind)))
+	if err != nil {
+		return nil, err
+	}
+	err = rc.PipeE(yaml.SetField("apiVersion", yaml.NewScalarRNode(version)))
+	if err != nil {
+		return nil, err
+	}
+	return []*yaml.RNode{rc}, nil
+}
+
+func (r *RunFnRunner) preRunE(c *cobra.Command, args []string) error {
+	if c.ArgsLenAtDash() >= 0 && r.Image == "" {
+		return errors.Errorf("must specify --image")
+	}
+
+	var dataItems []string
+	if c.ArgsLenAtDash() >= 0 {
+		dataItems = args[c.ArgsLenAtDash():]
+		args = args[:c.ArgsLenAtDash()]
+	}
+	if len(args) > 1 {
+		return errors.Errorf("0 or 1 arguments supported, function arguments go after '--'")
+	}
+
+	fns, err := r.getFunctions(c, args, dataItems)
+	if err != nil {
+		return err
+	}
+
+	// set the output to stdout if in dry-run mode or no arguments are specified
+	var output io.Writer
+	var input io.Reader
+	if len(args) == 0 {
+		output = c.OutOrStdout()
+		input = c.InOrStdin()
+	} else if r.DryRun {
+		output = c.OutOrStdout()
+	}
+
+	// set the path if specified as an argument
+	var path string
+	if len(args) == 1 {
+		// argument is the directory
+		path = args[0]
+	}
+
+	r.RunFns = runfn.RunFns{
+		FunctionPaths: r.FnPaths,
+		GlobalScope:   r.GlobalScope,
+		Functions:     fns,
+		Output:        output,
+		Input:         input,
+		Path:          path,
+		Network:       r.Network,
+		NetworkName:   r.NetworkName,
+	}
+
+	// don't consider args for the function
+	return nil
 }

cmd/config/internal/commands/run_test.go

@@ -0,0 +1,282 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package commands
+
+import (
+	"io"
+	"os"
+	"strings"
+	"testing"
+
+	"github.com/spf13/cobra"
+	"github.com/stretchr/testify/assert"
+)
+
+// TestRunFnCommand_preRunE verifies that preRunE correctly parses the commandline
+// flags and arguments into the RunFns structure to be executed.
+func TestRunFnCommand_preRunE(t *testing.T) {
+	tests := []struct {
+		name          string
+		args          []string
+		expected      string
+		err           string
+		path          string
+		input         io.Reader
+		output        io.Writer
+		functionPaths []string
+		network       bool
+		networkName   string
+	}{
+		{
+			name: "config map",
+			args: []string{"run", "dir", "--image", "foo:bar", "--", "a=b", "c=d", "e=f"},
+			path: "dir",
+			expected: `
+metadata:
+  name: function-input
+  annotations:
+    config.kubernetes.io/function: |
+      container: {image: 'foo:bar'}
+data: {a: b, c: d, e: f}
+kind: ConfigMap
+apiVersion: v1
+`,
+		},
+		{
+			name:   "config map stdin / stdout",
+			args:   []string{"run", "--image", "foo:bar", "--", "a=b", "c=d", "e=f"},
+			input:  os.Stdin,
+			output: os.Stdout,
+			expected: `
+metadata:
+  name: function-input
+  annotations:
+    config.kubernetes.io/function: |
+      container: {image: 'foo:bar'}
+data: {a: b, c: d, e: f}
+kind: ConfigMap
+apiVersion: v1
+`,
+		},
+		{
+			name:   "config map dry-run",
+			args:   []string{"run", "dir", "--image", "foo:bar", "--dry-run", "--", "a=b", "c=d", "e=f"},
+			output: os.Stdout,
+			path:   "dir",
+			expected: `
+metadata:
+  name: function-input
+  annotations:
+    config.kubernetes.io/function: |
+      container: {image: 'foo:bar'}
+data: {a: b, c: d, e: f}
+kind: ConfigMap
+apiVersion: v1
+`,
+		},
+		{
+			name: "config map no args",
+			args: []string{"run", "dir", "--image", "foo:bar"},
+			path: "dir",
+			expected: `
+metadata:
+  name: function-input
+  annotations:
+    config.kubernetes.io/function: |
+      container: {image: 'foo:bar'}
+data: {}
+kind: ConfigMap
+apiVersion: v1
+`,
+		},
+		{
+			name:        "network enabled",
+			args:        []string{"run", "dir", "--image", "foo:bar", "--network"},
+			path:        "dir",
+			network:     true,
+			networkName: "bridge",
+			expected: `
+metadata:
+  name: function-input
+  annotations:
+    config.kubernetes.io/function: |
+      container: {image: 'foo:bar', network: {required: true}}
+data: {}
+kind: ConfigMap
+apiVersion: v1
+`,
+		},
+		{
+			name:        "with network name",
+			args:        []string{"run", "dir", "--image", "foo:bar", "--network", "--network-name", "foo"},
+			path:        "dir",
+			network:     true,
+			networkName: "foo",
+			expected: `
+metadata:
+  name: function-input
+  annotations:
+    config.kubernetes.io/function: |
+      container: {image: 'foo:bar', network: {required: true}}
+data: {}
+kind: ConfigMap
+apiVersion: v1
+`,
+		},
+		{
+			name: "custom kind",
+			args: []string{"run", "dir", "--image", "foo:bar", "--", "Foo", "g=h"},
+			path: "dir",
+			expected: `
+metadata:
+  name: function-input
+  annotations:
+    config.kubernetes.io/function: |
+      container: {image: 'foo:bar'}
+data: {g: h}
+kind: Foo
+apiVersion: v1
+`,
+		},
+		{
+			name: "custom kind '=' in data",
+			args: []string{"run", "dir", "--image", "foo:bar", "--", "Foo", "g=h", "i=j=k"},
+			path: "dir",
+			expected: `
+metadata:
+  name: function-input
+  annotations:
+    config.kubernetes.io/function: |
+      container: {image: 'foo:bar'}
+data: {g: h, i: j=k}
+kind: Foo
+apiVersion: v1
+`,
+		},
+		{
+			name:          "function paths",
+			args:          []string{"run", "dir", "--fn-path", "path1", "--fn-path", "path2"},
+			path:          "dir",
+			functionPaths: []string{"path1", "path2"},
+		},
+		{
+			name: "custom kind with function paths",
+			args: []string{
+				"run", "dir", "--fn-path", "path", "--image", "foo:bar", "--", "Foo", "g=h", "i=j=k"},
+			path:          "dir",
+			functionPaths: []string{"path"},
+			expected: `
+metadata:
+  name: function-input
+  annotations:
+    config.kubernetes.io/function: |
+      container: {image: 'foo:bar'}
+data: {g: h, i: j=k}
+kind: Foo
+apiVersion: v1
+`,
+		},
+		{
+			name: "config map multi args",
+			args: []string{"run", "dir", "dir2", "--image", "foo:bar", "--", "a=b", "c=d", "e=f"},
+			err:  "0 or 1 arguments supported",
+		},
+		{
+			name: "config map not image",
+			args: []string{"run", "dir", "--", "a=b", "c=d", "e=f"},
+			err:  "must specify --image",
+		},
+		{
+			name: "config map bad data",
+			args: []string{"run", "dir", "--image", "foo:bar", "--", "a=b", "c", "e=f"},
+			err:  "must have keys and values separated by",
+		},
+	}
+
+	for i := range tests {
+		tt := tests[i]
+		t.Run(tt.name, func(t *testing.T) {
+			r := GetRunFnRunner("kustomize")
+			// Don't run the actual command
+			r.Command.Run = nil
+			r.Command.RunE = func(cmd *cobra.Command, args []string) error { return nil }
+			r.Command.SilenceErrors = true
+			r.Command.SilenceUsage = true
+
+			// hack due to https://github.com/spf13/cobra/issues/42
+			root := &cobra.Command{Use: "root"}
+			root.AddCommand(r.Command)
+			root.SetArgs(tt.args)
+
+			// error case
+			err := r.Command.Execute()
+			if tt.err != "" {
+				if !assert.Error(t, err) {
+					t.FailNow()
+				}
+				if !assert.Contains(t, err.Error(), tt.err) {
+					t.FailNow()
+				}
+				// don't check anything else in error case
+				return
+			}
+
+			// non-error case
+			if !assert.NoError(t, err) {
+				t.FailNow()
+			}
+
+			// check if Input was set
+			if !assert.Equal(t, tt.input, r.RunFns.Input) {
+				t.FailNow()
+			}
+
+			// check if Output was set
+			if !assert.Equal(t, tt.output, r.RunFns.Output) {
+				t.FailNow()
+			}
+
+			// check if Path was set
+			if !assert.Equal(t, tt.path, r.RunFns.Path) {
+				t.FailNow()
+			}
+
+			// check if Network was set
+			if tt.network {
+				if !assert.Equal(t, tt.network, r.RunFns.Network) {
+					t.FailNow()
+				}
+				if !assert.Equal(t, tt.networkName, r.RunFns.NetworkName) {
+					t.FailNow()
+				}
+			} else {
+				if !assert.Equal(t, false, r.RunFns.Network) {
+					t.FailNow()
+				}
+			}
+
+			// check if FunctionPaths were set
+			if tt.functionPaths == nil {
+				// make Equal work against flag default
+				tt.functionPaths = []string{}
+			}
+			if !assert.Equal(t, tt.functionPaths, r.RunFns.FunctionPaths) {
+				t.FailNow()
+			}
+
+			// check if Functions were set
+			if tt.expected != "" {
+				if !assert.Len(t, r.RunFns.Functions, 1) {
+					t.FailNow()
+				}
+				actual := strings.TrimSpace(r.RunFns.Functions[0].MustString())
+				if !assert.Equal(t, strings.TrimSpace(tt.expected), actual) {
+					t.FailNow()
+				}
+			}
+
+		})
+	}
+
+}

cmd/config/internal/commands/sink.go

@@ -7,6 +7,7 @@ import (
 	"github.com/spf13/cobra"
 	"sigs.k8s.io/kustomize/cmd/config/internal/generateddocs/commands"
 	"sigs.k8s.io/kustomize/kyaml/kio"
+	"sigs.k8s.io/kustomize/kyaml/kio/kioutil"
 )
 
 // GetSinkRunner returns a command for Sink.
@@ -18,7 +19,7 @@ func GetSinkRunner(name string) *SinkRunner {
 		Long:    commands.SinkLong,
 		Example: commands.SinkExamples,
 		RunE:    r.runE,
-		Args:    cobra.ExactArgs(1),
+		Args:    cobra.MaximumNArgs(1),
 	}
 	fixDocs(name, c)
 	r.Command = c
@@ -35,14 +36,18 @@ type SinkRunner struct {
 }
 
 func (r *SinkRunner) runE(c *cobra.Command, args []string) error {
+	var outputs []kio.Writer
+	if len(args) == 1 {
+		outputs = []kio.Writer{&kio.LocalPackageWriter{PackagePath: args[0]}}
+	} else {
+		outputs = []kio.Writer{&kio.ByteWriter{
+			Writer:           c.OutOrStdout(),
+			ClearAnnotations: []string{kioutil.PathAnnotation}},
+		}
+	}
+
 	err := kio.Pipeline{
-		Inputs: []kio.Reader{&kio.ByteReader{Reader: c.InOrStdin()}},
-		Outputs: []kio.Writer{
-			&kio.LocalPackageWriter{
-				PackagePath:      args[0],
-				ClearAnnotations: []string{"config.kubernetes.io/path"},
-			},
-		},
-	}.Execute()
+		Inputs:  []kio.Reader{&kio.ByteReader{Reader: c.InOrStdin()}},
+		Outputs: outputs}.Execute()
 	return handleError(c, err)
 }

cmd/config/internal/commands/sink_test.go

@@ -21,8 +21,6 @@ func TestSinkCommand(t *testing.T) {
 	}
 	defer os.RemoveAll(d)
 
-	// fmt the files
-	b := &bytes.Buffer{}
 	r := commands.GetSinkRunner("")
 	r.Command.SetIn(bytes.NewBufferString(`apiVersion: config.kubernetes.io/v1alpha1
 kind: ResourceList
@@ -75,7 +73,6 @@ items:
     replicas: 3
 `))
 	r.Command.SetArgs([]string{d})
-	r.Command.SetOut(b)
 	if !assert.NoError(t, r.Command.Execute()) {
 		t.FailNow()
 	}
@@ -138,3 +135,117 @@ spec:
 		t.FailNow()
 	}
 }
+
+func TestSinkCommand_Stdout(t *testing.T) {
+	d, err := ioutil.TempDir("", "kustomize-source-test")
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
+	defer os.RemoveAll(d)
+
+	// fmt the files
+	out := &bytes.Buffer{}
+	r := commands.GetSinkRunner("")
+	r.Command.SetIn(bytes.NewBufferString(`apiVersion: config.kubernetes.io/v1alpha1
+kind: ResourceList
+items:
+- kind: Deployment
+  metadata:
+    labels:
+      app: nginx2
+    name: foo
+    annotations:
+      app: nginx2
+      config.kubernetes.io/index: '0'
+      config.kubernetes.io/path: 'f1.yaml'
+  spec:
+    replicas: 1
+- kind: Service
+  metadata:
+    name: foo
+    annotations:
+      app: nginx
+      config.kubernetes.io/index: '1'
+      config.kubernetes.io/path: 'f1.yaml'
+  spec:
+    selector:
+      app: nginx
+- apiVersion: v1
+  kind: Abstraction
+  metadata:
+    name: foo
+    annotations:
+      config.kubernetes.io/function: |
+        container:
+          image: gcr.io/example/reconciler:v1
+      config.kubernetes.io/local-config: "true"
+      config.kubernetes.io/index: '0'
+      config.kubernetes.io/path: 'f2.yaml'
+  spec:
+    replicas: 3
+- apiVersion: apps/v1
+  kind: Deployment
+  metadata:
+    labels:
+      app: nginx
+    name: bar
+    annotations:
+      app: nginx
+      config.kubernetes.io/index: '1'
+      config.kubernetes.io/path: 'f2.yaml'
+  spec:
+    replicas: 3
+`))
+
+	r.Command.SetOut(out)
+	r.Command.SetArgs([]string{})
+	if !assert.NoError(t, r.Command.Execute()) {
+		t.FailNow()
+	}
+
+	expected := `kind: Deployment
+metadata:
+  labels:
+    app: nginx2
+  name: foo
+  annotations:
+    app: nginx2
+spec:
+  replicas: 1
+---
+kind: Service
+metadata:
+  name: foo
+  annotations:
+    app: nginx
+spec:
+  selector:
+    app: nginx
+---
+apiVersion: v1
+kind: Abstraction
+metadata:
+  name: foo
+  annotations:
+    config.kubernetes.io/function: |
+      container:
+        image: gcr.io/example/reconciler:v1
+    config.kubernetes.io/local-config: "true"
+spec:
+  replicas: 3
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: nginx
+  name: bar
+  annotations:
+    app: nginx
+spec:
+  replicas: 3
+`
+	if !assert.Equal(t, expected, out.String()) {
+		t.FailNow()
+	}
+}

cmd/config/internal/commands/source.go

@@ -21,7 +21,6 @@ func GetSourceRunner(name string) *SourceRunner {
 		Long:    commands.SourceLong,
 		Example: commands.SourceExamples,
 		RunE:    r.runE,
-		Args:    cobra.ExactArgs(1),
 	}
 	fixDocs(name, c)
 	c.Flags().StringVar(&r.WrapKind, "wrap-kind", kio.ResourceListKind,
@@ -70,8 +69,14 @@ func (r *SourceRunner) runE(c *cobra.Command, args []string) error {
 		FunctionConfig:        functionConfig,
 	})
 
-	err := kio.Pipeline{
-		Inputs:  []kio.Reader{kio.LocalPackageReader{PackagePath: args[0]}},
-		Outputs: outputs}.Execute()
+	var inputs []kio.Reader
+	for _, a := range args {
+		inputs = append(inputs, kio.LocalPackageReader{PackagePath: a})
+	}
+	if len(inputs) == 0 {
+		inputs = []kio.Reader{&kio.ByteReader{Reader: c.InOrStdin()}}
+	}
+
+	err := kio.Pipeline{Inputs: inputs, Outputs: outputs}.Execute()
 	return handleError(c, err)
 }

cmd/config/internal/commands/source_test.go

@@ -134,3 +134,67 @@ items:
 		return
 	}
 }
+
+func TestSourceCommand_Stdin(t *testing.T) {
+	d, err := ioutil.TempDir("", "kustomize-source-test")
+	if !assert.NoError(t, err) {
+		return
+	}
+	defer os.RemoveAll(d)
+
+	in := bytes.NewBufferString(`
+kind: Deployment
+metadata:
+  labels:
+    app: nginx2
+  name: foo
+  annotations:
+    app: nginx2
+spec:
+  replicas: 1
+---
+kind: Service
+metadata:
+  name: foo
+  annotations:
+    app: nginx
+spec:
+  selector:
+    app: nginx
+`)
+
+	out := &bytes.Buffer{}
+	r := commands.GetSourceRunner("")
+	r.Command.SetArgs([]string{})
+	r.Command.SetIn(in)
+	r.Command.SetOut(out)
+	if !assert.NoError(t, r.Command.Execute()) {
+		return
+	}
+
+	if !assert.Equal(t, `apiVersion: config.kubernetes.io/v1alpha1
+kind: ResourceList
+items:
+- kind: Deployment
+  metadata:
+    labels:
+      app: nginx2
+    name: foo
+    annotations:
+      app: nginx2
+      config.kubernetes.io/index: '0'
+  spec:
+    replicas: 1
+- kind: Service
+  metadata:
+    name: foo
+    annotations:
+      app: nginx
+      config.kubernetes.io/index: '1'
+  spec:
+    selector:
+      app: nginx
+`, out.String()) {
+		return
+	}
+}

cmd/config/internal/generateddocs/api/docs.go

@@ -33,7 +33,7 @@ Example:
 ### ` + "`" + `config.kubernetes.io/index` + "`" + `
 
 Records the index of a Resource in file. In a multi-object YAML file, Resources are separated
-by three dashes (` + "`" + `---` + "`" + `), and the index represents the positon of the Resource starting from zero.
+by three dashes (` + "`" + `---` + "`" + `), and the index represents the position of the Resource starting from zero.
 
 This annotation **SHOULD** be set when reading Resources from files.
 It **SHOULD** be unset when writing Resources to files.
@@ -187,7 +187,7 @@ are passed to the Function through the ` + "`" + `ResourceList.functionConfig` +
 
 ### Output
 
-The function is invoked using by runing ` + "`" + `kustomize config run dir/` + "`" + `.
+The function is invoked using byrunning ` + "`" + `kustomize config run dir/` + "`" + `.
 
 ` + "`" + `dir/my-instance_deployment.yaml` + "`" + ` contains the Deployment:
 

cmd/config/internal/generateddocs/commands/docs.go

@@ -4,6 +4,20 @@
 // Code generated by "mdtogo"; DO NOT EDIT.
 package commands
 
+var AnnotateShort = `[Alpha] Set an annotation on Resources.`
+var AnnotateLong = `
+[Alpha]  Set an annotation on Resources.
+
+  DIR:
+    Path to local directory.
+`
+var AnnotateExamples = `
+    kustomize config annotate my-dir/ --kv foo=bar
+
+    kustomize config annotate my-dir/ --kv foo=bar --kv a=b
+
+    kustomize config annotate my-dir/ --kv foo=bar --kind Deployment --name foo`
+
 var CatShort = `[Alpha] Print Resource Config from a local directory.`
 var CatLong = `
 [Alpha]  Print Resource Config from a local directory.
@@ -336,10 +350,10 @@ var SinkShort = `[Alpha] Implement a Sink by writing input to a local directory.
 var SinkLong = `
 [Alpha] Implement a Sink by writing input to a local directory.
 
-    kustomize config sink DIR
+    kustomize config sink [DIR]
 
   DIR:
-    Path to local directory.
+    Path to local directory.  If unspecified, sink will write to stdout as if it were a single file.
 
 ` + "`" + `sink` + "`" + ` writes its input to a directory
 `
@@ -350,10 +364,11 @@ var SourceShort = `[Alpha] Implement a Source by reading a local directory.`
 var SourceLong = `
 [Alpha] Implement a Source by reading a local directory.
 
-    kustomize config source DIR
+    kustomize config source DIR...
 
   DIR:
-    Path to local directory.
+    One or more paths to local directories.  Contents from directories will be concatenated.
+    If no directories are provided, source will read from stdin as if it were a single file.
 
 ` + "`" + `source` + "`" + ` emits configuration to act as input to a function
 `

cmd/kubectl/go.mod

@@ -3,13 +3,21 @@ module sigs.k8s.io/kustomize/cmd/kubectl
 go 1.13
 
 require (
+	github.com/go-errors/errors v1.0.1
 	github.com/spf13/cobra v0.0.5
+	github.com/stretchr/testify v1.4.0
+	k8s.io/api v0.17.0
 	k8s.io/apimachinery v0.17.0
 	k8s.io/cli-runtime v0.17.0
 	k8s.io/client-go v0.17.0
 	k8s.io/component-base v0.17.0 // indirect
 	k8s.io/kubectl v0.0.0-20191219154910-1528d4eea6dd
-	sigs.k8s.io/kustomize/kyaml v0.0.0
+	sigs.k8s.io/controller-runtime v0.4.0
+	sigs.k8s.io/kustomize/kstatus v0.0.1
+	sigs.k8s.io/kustomize/kyaml v0.0.2
 )
 
-replace sigs.k8s.io/kustomize/kyaml v0.0.0 => ../../kyaml
+replace (
+	sigs.k8s.io/kustomize/kstatus v0.0.1 => ../../kstatus
+	sigs.k8s.io/kustomize/kyaml v0.0.0 => ../../kyaml
+)