Copy pluginator to its own module.

add release note for v3.2.0

.gitignore

@@ -4,6 +4,7 @@
 *.dll
 *.so
 *.dylib
+/kustomize
 
 # Test binary, build with `go test -c`
 *.test

Makefile

@@ -0,0 +1,37 @@
+BIN_NAME=kustomize
+
+COVER_FILE=coverage.out
+
+export GO111MODULE=on
+
+all: test build
+
+test: generate-code test-lint test-go
+
+test-go:
+	go test -v ./...
+
+test-lint:
+	golangci-lint run ./...
+
+generate-code:
+	./plugin/generateBuiltins.sh $(GOPATH)
+
+build:
+	go build -o $(BIN_NAME) cmd/kustomize/main.go
+
+install:
+	go install $(PWD)/cmd/kustomize
+
+cover:
+	# The plugin directory eludes coverage, and is therefore omitted
+	go test ./pkg/... ./k8sdeps/... ./internal/... -coverprofile=$(COVER_FILE) && \
+	go tool cover -html=$(COVER_FILE)
+
+
+clean:
+	go clean
+	rm -f $(BIN_NAME)
+	rm -f $(COVER_FILE)
+
+.PHONY: test build install clean generate-code test-go test-lint cover

README.md

@@ -22,8 +22,16 @@ these [instructions](docs/INSTALL.md).
 Browse the [docs](docs) or jump right into the
 tested [examples](examples).
 
-kustomize [v2.0.3] is available in [kubectl v1.15][kubectl].
+## kubectl integration
 
+Since [v1.14][kubectl announcement] the kustomize build system has been included in kubectl.
+
+| kubectl version | kustomize version |
+|---------|--------|
+| v1.15.x | [v2.0.3](https://github.com/kubernetes-sigs/kustomize/tree/v2.0.3) |
+| v1.14.x | [v2.0.3](https://github.com/kubernetes-sigs/kustomize/tree/v2.0.3) |
+
+For examples and guides for using the kubectl integration please see the [kubectl book] or the [kubernetes documentation].
 
 ## Usage
 
@@ -159,7 +167,9 @@ is governed by the [Kubernetes Code of Conduct].
 [imageBase]: docs/images/base.jpg
 [imageOverlay]: docs/images/overlay.jpg
 [kind/feature]: https://github.com/kubernetes-sigs/kustomize/labels/kind%2Ffeature
-[kubectl]: https://kubernetes.io/blog/2019/03/25/kubernetes-1-14-release-announcement
+[kubectl announcement]: https://kubernetes.io/blog/2019/03/25/kubernetes-1-14-release-announcement
+[kubectl book]: https://kubectl.docs.kubernetes.io/pages/app_customization/introduction.html
+[kubernetes documentation]: https://kubernetes.io/docs/tasks/manage-kubernetes-objects/kustomization/
 [kubernetes style]: docs/glossary.md#kubernetes-style-object
 [kustomization]: docs/glossary.md#kustomization
 [overlay]: docs/glossary.md#overlay

cmd/pluginator/main.go

@@ -16,6 +16,15 @@ import (
 	"sigs.k8s.io/kustomize/v3/pkg/plugins"
 )
 
+//go:generate stringer -type=pluginType
+type pluginType int
+
+const (
+	unknown pluginType = iota
+	Transformer
+	Generator
+)
+
 func main() {
 	root := inputFileRoot()
 	file, err := os.Open(root + ".go")
@@ -34,24 +43,41 @@ func main() {
 		fmt.Sprintf(
 			"// Code generated by pluginator on %s; DO NOT EDIT.",
 			root))
-	w.write("package builtin")
+	w.write("package " + plugins.BuiltinPluginPackage)
+
+	pType := unknown
 
 	for scanner.Scan() {
 		l := scanner.Text()
 		if strings.HasPrefix(l, "//go:generate") {
 			continue
 		}
-		if l == "var "+plugins.PluginSymbol+" plugin" {
-			w.write("func New" + root + "Plugin() *" + root + "Plugin {")
-			w.write("  return &" + root + "Plugin{}")
-			w.write("}")
+		if strings.HasPrefix(l, "//noinspection") {
 			continue
 		}
+		if l == "var "+plugins.PluginSymbol+" plugin" {
+			continue
+		}
+		if strings.Contains(l, " Transform(") {
+			if pType != unknown {
+				log.Fatal("unexpected Transform(")
+			}
+			pType = Transformer
+		} else if strings.Contains(l, " Generate(") {
+			if pType != unknown {
+				log.Fatal("unexpected Generate(")
+			}
+			pType = Generator
+		}
 		w.write(l)
 	}
 	if err := scanner.Err(); err != nil {
 		log.Fatal(err)
 	}
+	w.write("")
+	w.write("func New" + root + "Plugin() resmap." + pType.String() + "Plugin {")
+	w.write("  return &" + root + "Plugin{}")
+	w.write("}")
 }
 
 func inputFileRoot() string {
@@ -93,7 +119,7 @@ func makeOutputFileName(root string) string {
 		pgmconfig.DomainName,
 		pgmconfig.ProgramName,
 		pgmconfig.PluginRoot,
-		"builtin",
+		plugins.BuiltinPluginPackage,
 		root+".go")
 }
 

cmd/pluginator/plugintype_string.go

@@ -0,0 +1,25 @@
+// Code generated by "stringer -type=pluginType"; DO NOT EDIT.
+
+package main
+
+import "strconv"
+
+func _() {
+	// An "invalid array index" compiler error signifies that the constant values have changed.
+	// Re-run the stringer command to generate them again.
+	var x [1]struct{}
+	_ = x[unknown-0]
+	_ = x[Transformer-1]
+	_ = x[Generator-2]
+}
+
+const _pluginType_name = "unknownTransformerGenerator"
+
+var _pluginType_index = [...]uint8{0, 7, 18, 27}
+
+func (i pluginType) String() string {
+	if i < 0 || i >= pluginType(len(_pluginType_index)-1) {
+		return "pluginType(" + strconv.FormatInt(int64(i), 10) + ")"
+	}
+	return _pluginType_name[_pluginType_index[i]:_pluginType_index[i+1]]
+}

docs/FAQ.md

@@ -28,3 +28,31 @@ To disable this, use v3, and the `load_restrictor` flag:
 ```
 kustomize build --load_restrictor none $target
 ```
+
+## Some field is not transformed by kustomize
+
+Example: [#1319](https://github.com/kubernetes-sigs/kustomize/issues/1319), [#1322](https://github.com/kubernetes-sigs/kustomize/issues/1322), [#1347](https://github.com/kubernetes-sigs/kustomize/issues/1347) and etc.
+
+The fields transformed by kustomize is configured explicitly in [defaultconfig](https://github.com/kubernetes-sigs/kustomize/tree/master/pkg/transformers/config/defaultconfig). The configuration itself can be customized by including `configurations` in `kustomization.yaml`, e.g.
+
+```yaml
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+configurations:
+- kustomizeconfig.yaml
+```
+
+The configuration directive allows customization of the following transformers:
+
+```yaml
+commonAnnotations: []
+commonLabels: []
+nameprefix: []
+namespace: []
+varreference: []
+namereference: []
+images: []
+replicas: []
+```
+
+To persist the changes to default configuration, submit a PR like [#1338](https://github.com/kubernetes-sigs/kustomize/pull/1338), [#1348](https://github.com/kubernetes-sigs/kustomize/pull/1348) and etc.

docs/INSTALL.md

@@ -25,11 +25,17 @@ chmod u+x kustomize
 
 Requires [Go] v1.12 or higher:
 
-<!-- @installkustomize @test -->
+<!-- @installkustomize @testAgainstLatestRelease -->
 ```
 go install sigs.k8s.io/kustomize/v3/cmd/kustomize
 ```
 
+> With [Go v1.12](https://golang.org/doc/go1.12#modules), prefix the above command with `GO111MODULE=on`, e.g.
+> ```
+> GO111MODULE=on go install sigs.k8s.io/kustomize/v3/cmd/kustomize
+> ```
+> This shouldn't be necessary with [Go v1.13](https://golang.org/doc/go1.13#modules).
+
 ### Other methods
 
 #### macOS

docs/README.md

@@ -23,7 +23,9 @@ English | [简体中文](zh/README.md)
 
 ## Release notes
 
- * [3.0](v3.0.0.md) - Late June 2019.  Plugin developer release.
+ * [3.1](v3.1.0.md) - Late July 2019. Extended patches and improved resource matching.
+
+ * [3.0](v3.0.0.md) - Late June 2019. Plugin developer release.
 
  * [2.1](v2.1.0.md) - 18 June 2019.  Plugins, ordered resources, etc.
 

docs/fields.md

@@ -1,5 +1,19 @@
 # Kustomization File Fields
 
+[field-name-namespace]: plugins/builtins.md#field-name-namespace
+[field-name-images]: plugins/builtins.md#field-name-images
+[field-name-namePrefix]: plugins/builtins.md#field-name-prefix
+[field-name-nameSuffix]: plugins/builtins.md#field-name-prefix
+[field-name-patches]: plugins/builtins.md#field-name-patches
+[field-name-patchesStrategicMerge]: plugins/builtins.md#field-name-patchesStrategicMerge
+[field-name-patchesJson6902]: plugins/builtins.md#field-name-patchesJson6902
+[field-name-replicas]: plugins/builtins.md#field-name-replicas
+[field-name-secretGenerator]: plugins/builtins.md#field-name-secretGenerator
+[field-name-commonLabels]: plugins/builtins.md#field-name-commonLabels
+[field-name-commonAnnotations]: plugins/builtins.md#field-name-commonAnnotations
+[field-name-configMapGenerator]: plugins/builtins.md#field-name-configMapGenerator
+
+
 An explanation of the fields in a [kustomization.yaml](glossary.md#kustomization) file.
 
 
@@ -38,6 +52,7 @@ What transformations (customizations) should be applied?
 | [namePrefix](#nameprefix) | string | Prepends value to the names of all resources |
 | [nameSuffix](#namesuffix) | string | The value is appended to the names of all resources. |
 | [replicas](#replicas) | list | Replicas modifies the number of replicas of a resource. |
+| [patches](#patches) | list | Each entry should resolve to a patch that can be applied to multiple targets. |
 |[patchesStrategicMerge](#patchesstrategicmerge)| list |Each entry in this list should resolve to a partial or complete resource definition file.|
 |[patchesJson6902](#patchesjson6902)| list  |Each entry in this list should resolve to a kubernetes object and a JSON patch that will be applied to the object.|
 |[transformers](#transformers)|list|[plugin](plugins) configuration files|
@@ -64,7 +79,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 
 ### bases
 
-The `bases` field was deprecated in v2.1.0.
+_The `bases` field was deprecated in v2.1.0._
 
 Move entries into the [resources](#resources)
 field.  This allows bases - which are still a
@@ -72,51 +87,13 @@ field.  This allows bases - which are still a
 ordered relative to other input resources.
 
 ### commonLabels
-
-Adds labels to all resources and selectors
-```
-commonLabels:
-  someName: someValue
-  owner: alice
-  app: bingo
-```
+See [field-name-commonLabels].
 
 ### commonAnnotations
-
-Adds annotions (non-identifying metadata) to add
-all resources. Like labels, these are key value
-pairs.
-
-```
-commonAnnotations:
-  oncallPager: 800-555-1212
-```
+See [field-name-commonAnnotations].
 
 ### configMapGenerator
-
-Each entry in this list results in the creation of
-one ConfigMap resource (it's a generator of n maps).
-
-The example below creates two ConfigMaps. One with the
-names and contents of the given files, the other with
-key/value as data.
-
-Each configMapGenerator item accepts a parameter of
-`behavior: [create|replace|merge]`.
-This allows an overlay to modify or
-replace an existing configMap from the parent.
-
-```
-configMapGenerator:
-- name: myJavaServerProps
-  files:
-  - application.properties
-  - more.properties
-- name: myJavaServerEnvVars
-  literals:	
-  - JAVA_HOME=/opt/java/jdk
-  - JAVA_TOOL_OPTIONS=-agentlib:hprof
-```
+See [field-name-configMapGenerator].
 
 ### crds
 
@@ -143,9 +120,7 @@ The annotations can be put into openAPI definitions are:
  -  "x-kubernetes-object-ref-kind": "Secret",
  -  "x-kubernetes-object-ref-name-key": "name",
 
-
 ```
-
 crds:
 - crds/typeA.yaml
 - crds/typeB.yaml
@@ -183,42 +158,7 @@ generators:
 
 ### images
 
-Images modify the name, tags and/or digest for images without creating patches.
-E.g. Given this kubernetes Deployment fragment:
-
-```
-containers:
- - name: mypostgresdb
-   image: postgres:8
- - name: nginxapp
-   image: nginx:1.7.9
- - name: myapp
-   image: my-demo-app:latest
- - name: alpine-app
-   image: alpine:3.7
-```
-
-one can change the `image` in the following ways:
- 
- - `postgres:8` to `my-registry/my-postgres:v1`,
- - nginx tag `1.7.9` to `1.8.0`,
- - image name `my-demo-app` to `my-app`,
- - alpine's tag `3.7` to a digest value
-
-all with the following *kustomization*:
-
-```
-images:
-- name: postgres
-  newName: my-registry/my-postgres
-  newTag: v1
-- name: nginx
-  newTag: 1.8.0
-- name: my-demo-app
-  newName: my-app
-- name: alpine
-  digest: sha256:24a0c4b4a4c0eb97a1aabb8e29f18e917d05abfe1b7a7c07857230879ce7d3d3
-```
+See [field-name-images].
 
 ### inventory
 
@@ -232,143 +172,33 @@ If missing, this field's value defaults to
 kind: Kustomization
 ```
 
-
 ### namespace
 
-Adds namespace to all resources
-
-```
-namespace: my-namespace
-```
+See [field-name-namespace].
 
 ### namePrefix
 
-Prepends value to the names of all resources
-Ex. a deployment named `wordpress` would become `alices-wordpress`
-
-```
-namePrefix: alices-
-```
+See [field-name-namePrefix].
 
 ### nameSuffix
 
-The value is appended to the names of all
-resources.  Ex. A deployment named `wordpress`
-would become `wordpress-v2`.
+See [field-name-nameSuffix].
 
-The suffix is appended before content has if
-resource type is ConfigMap or Secret.
+### patches
 
-```
-nameSuffix: -v2
-```
+See [field-name-patches].
 
 ### patchesStrategicMerge
 
-Each entry in this list should be a relative path
-resolving to a partial or complete resource
-definition file.
-
-The names in these (possibly partial) resource
-files must match names already loaded via the
-`resources` field.  These entries are used to
-_patch_ (modify) the known resources.
-
-Small patches that do one thing are best, e.g. modify
-a memory request/limit, change an env var in a
-ConfigMap, etc.  Small patches are easy to review and
-easy to mix together in overlays.
-
-```
-patchesStrategicMerge:
-- service_port_8888.yaml
-- deployment_increase_replicas.yaml
-- deployment_increase_memory.yaml
-```
+See [field-name-patchesStrategicMerge].
 
 ### patchesJson6902
 
-Each entry in this list should resolve to
-a kubernetes object and a JSON patch that will be applied
-to the object.
-The JSON patch is documented at https://tools.ietf.org/html/rfc6902
-
-target field points to a kubernetes object within the same kustomization
-by the object's group, version, kind, name and namespace.
-path field is a relative file path of a JSON patch file.
-The content in this patch file can be either in JSON format as
-
-```
- [
-   {"op": "add", "path": "/some/new/path", "value": "value"},
-   {"op": "replace", "path": "/some/existing/path", "value": "new value"}
- ]
- ```
-
-or in YAML format as
-
-```
-- op: add
-  path: /some/new/path
-  value: value
-- op: replace
-  path: /some/existing/path
-  value: new value
-```
-
-```
-patchesJson6902:
-- target:
-    version: v1
-    kind: Deployment
-    name: my-deployment
-  path: add_init_container.yaml
-- target:
-    version: v1
-    kind: Service
-    name: my-service
-  path: add_service_annotation.yaml
-```
+See [field-name-patchesJson6902].
 
 ### replicas
 
-Replicas modified the number of replicas for a resource.
-
-E.g. Given this kubernetes Deployment fragment:
-
-```
-kind: Deployment
-metadata:
-  name: deployment-name
-spec:
-  replicas: 3
-```
-
-one can change the number of replicas to 5
-by adding the following to your kustomization:
-
-```
-replicas:
-- name: deployment-name
-  count: 5
-```
-
-This field accepts a list, so many resources can
-be modified at the same time.
-
-
-#### Limitation
-
-As this declaration does not take in a `kind:` nor a `group:`
-it will match any `group` and `kind` that has a matching name and
-that is one of:
-- `Deployment`
-- `ReplicationController`
-- `ReplicaSet`
-- `StatefulSet`
-
-For more complex use cases, revert to using a patch.
-
+See [field-name-replicas].
 
 ### resources
 
@@ -406,28 +236,7 @@ must contain a `kustomization.yaml` file.
 
 ### secretGenerator
 
-Each entry in this list results in the creation of
-one Secret resource (it's a generator of n secrets).
-
-```
-secretGenerator:
-- name: app-tls
-  files:
-  - secret/tls.cert
-  - secret/tls.key
-  type: "kubernetes.io/tls"
-- name: app-tls-namespaced
-  # you can define a namespace to generate secret in, defaults to: "default"
-  namespace: apps
-  files:
-  - tls.crt=catsecret/tls.cert
-  - tls.key=secret/tls.key
-  type: "kubernetes.io/tls"
-- name: env_file_secret
-  envs:
-  - env.txt
-  type: Opaque
-```
+See [field-name-secretGenerator].
 
 ### vars
 

docs/glossary.md

@@ -376,9 +376,8 @@ value is a list.
 
 To change this
 default behavior, add a _directive_.  Recognized
-directives include _replace_ (the default), _merge_
-(avoid replacing a list), _delete_ and a few more
-(see [these notes][strategic-merge]).
+directives in YAML patches are _replace_ (the default)
+and _delete_ (see [these notes][strategic-merge]).
 
 Note that for custom resources, SMPs are treated as
 [json merge patches][JSONMergePatch].

docs/plugins/README.md

@@ -223,6 +223,7 @@ provided in the kustomization file).
 [helm chart inflator]: ../../plugin/someteam.example.com/v1/chartinflator
 [bashed config map]: ../../plugin/someteam.example.com/v1/bashedconfigmap
 [sed transformer]: ../../plugin/someteam.example.com/v1/sedtransformer
+[hashicorp go-getter]: ../../plugin/someteam.example.com/v1/gogetter
 
 #### Examples
 
@@ -230,7 +231,7 @@ provided in the kustomization file).
  * [bashed config map] -  Super simple configMap generation from bash.
  * [sed transformer] - Define your unstructured edits using a
    plugin like this one.
-
+ * [hashicorp go-getter] - Download kustomize layes and build it to generate resources
 
 A generator plugin accepts nothing on `stdin`, but emits
 generated resources to `stdout`.

docs/plugins/builtins.md

@@ -0,0 +1,683 @@
+<!-- 
+  TODO: Generate this file (or files) from
+        data in directories under plugin/builtin.
+        
+        This file too hard to maintain distinctly
+        from what's going on in those directories.
+        We could expand pluginator to do this, since
+        it already scans the relevant files in the
+        relevant directory to generate the static
+        factory methods for plugins.
+-->
+
+# Builtin Plugins
+
+A list of kustomize's builtin plugins (both
+generators and transformers).
+
+For each plugin, an example is given for
+
+* implicitly triggering
+the plugin via a dedicated kustomization
+file field (e.g. the `AnnotationsTransformer` is 
+triggered by the `commonAnnotations` field).
+
+* explicitly triggering the plugin
+via the `generators` or `transformers` field
+(by providing a config file specifying the
+plugin).
+
+The former method is convenient but limited in 
+power as most of the plugins arguments must
+be defaulted.  The latter method allows for
+complete plugin argument specification.
+
+
+[types.GeneratorOptions]: ../../pkg/types/generatoroptions.go
+[types.SecretArgs]: ../../pkg/types/secretargs.go
+[types.ConfigMapArgs]: ../../pkg/types/configmapargs.go
+[config.FieldSpec]: ../../pkg/transformers/config/fieldspec.go
+[types.ObjectMeta]: ../../pkg/types/objectmeta.go
+[types.Selector]: ../../pkg/types/selector.go
+[types.Replica]: ../../pkg/types/replica.go
+[types.PatchStrategicMerge]: ../../pkg/types/patchstrategicmerge.go
+[types.PatchTarget]: ../../pkg/types/patchtarget.go
+[image.Image]: ../../pkg/image/image.go
+
+## _AnnotationTransformer_
+### Usage via `kustomization.yaml`
+
+#### field name: `commonAnnotations`
+
+Adds annotions (non-identifying metadata) to add
+all resources. Like labels, these are key value
+pairs.
+
+```
+commonAnnotations:
+  oncallPager: 800-555-1212
+```
+
+### Usage via plugin
+#### Arguments
+
+> Annotations map\[string\]string
+>
+> FieldSpecs  \[\][config.FieldSpec]
+
+#### Example
+> ```
+> apiVersion: builtin
+> kind: AnnotationsTransformer
+> metadata:
+>   name: not-important-to-example
+> annotations:
+>   app: myApp
+>   greeting/morning: a string with blanks
+> fieldSpecs:
+> - path: metadata/annotations
+>   create: true
+> ```
+
+
+
+## _ConfigMapGenerator_
+
+### Usage via `kustomization.yaml`
+
+#### field name: `configMapGenerator`
+
+Each entry in this list results in the creation of
+one ConfigMap resource (it's a generator of n maps).
+
+The example below creates two ConfigMaps. One with the
+names and contents of the given files, the other with
+key/value as data.
+
+Each configMapGenerator item accepts a parameter of
+`behavior: [create|replace|merge]`.
+This allows an overlay to modify or
+replace an existing configMap from the parent.
+
+```
+configMapGenerator:
+- name: my-java-server-props
+  files:
+  - application.properties
+  - more.properties
+- name: my-java-server-env-vars
+  literals:	
+  - JAVA_HOME=/opt/java/jdk
+  - JAVA_TOOL_OPTIONS=-agentlib:hprof
+```
+
+It is also possible to
+[define a key](https://kubernetes.io/docs/tasks/configure-pod-container/configure-pod-configmap/#define-the-key-to-use-when-creating-a-configmap-from-a-file)
+to set a name different than the filename.
+
+The example below creates a ConfigMap
+with the name of file as `myFileName.ini`
+while the _actual_ filename from which the
+configmap is created is `whatever.ini`.
+
+```
+configMapGenerator:
+- name: app-whatever
+  files:
+  - myFileName.ini=whatever.ini
+```
+
+### Usage via plugin
+#### Arguments
+
+> [types.GeneratorOptions]
+>
+> [types.ConfigMapArgs]
+
+#### Example
+> ```
+> apiVersion: builtin
+> kind: ConfigMapGenerator
+> metadata:
+>   name: mymap
+> envs:
+> - devops.env
+> - uxteam.env
+> literals:
+> - FRUIT=apple
+> - VEGETABLE=carrot
+> ```
+
+
+## _ImageTagTransformer_
+### Usage via `kustomization.yaml`
+
+#### field name: `image`
+
+Images modify the name, tags and/or digest for images
+without creating patches.  E.g. Given this
+kubernetes Deployment fragment:
+
+```
+containers:
+- name: mypostgresdb
+  image: postgres:8
+- name: nginxapp
+  image: nginx:1.7.9
+- name: myapp
+  image: my-demo-app:latest
+- name: alpine-app
+  image: alpine:3.7
+```
+
+one can change the `image` in the following ways:
+ 
+ - `postgres:8` to `my-registry/my-postgres:v1`,
+ - nginx tag `1.7.9` to `1.8.0`,
+ - image name `my-demo-app` to `my-app`,
+ - alpine's tag `3.7` to a digest value
+
+all with the following *kustomization*:
+
+```
+images:
+- name: postgres
+  newName: my-registry/my-postgres
+  newTag: v1
+- name: nginx
+  newTag: 1.8.0
+- name: my-demo-app
+  newName: my-app
+- name: alpine
+  digest: sha256:24a0c4b4a4c0eb97a1aabb8e29f18e917d05abfe1b7a7c07857230879ce7d3d3
+```
+
+### Usage via plugin
+#### Arguments
+
+> ImageTag   [image.Image]
+>
+> FieldSpecs \[\][config.FieldSpec]
+
+#### Example
+> ```
+> apiVersion: builtin
+> kind: ImageTagTransformer
+> metadata:
+>   name: not-important-to-example
+> imageTag:
+>   name: nginx
+>   newTag: v2
+> ```
+
+
+
+## _LabelTransformer_
+### Usage via `kustomization.yaml`
+
+#### field name: `commonLabels`
+
+Adds labels to all resources and selectors
+
+```
+commonLabels:
+  someName: someValue
+  owner: alice
+  app: bingo
+```
+
+### Usage via plugin
+#### Arguments
+
+> Labels  map\[string\]string
+>
+> FieldSpecs \[\][config.FieldSpec]
+
+#### Example
+> ```
+> apiVersion: builtin
+> kind: LabelTransformer
+> metadata:
+>   name: not-important-to-example
+> labels:
+>   app: myApp
+>   env: production
+> fieldSpecs:
+> - path: metadata/labels
+>   create: true
+> ```
+
+
+
+
+
+
+
+
+## _NamespaceTransformer_
+### Usage via `kustomization.yaml`
+
+#### field name: `namespace`
+
+Adds namespace to all resources
+
+```
+namespace: my-namespace
+```
+
+### Usage via plugin
+#### Arguments
+
+> [types.ObjectMeta]
+>
+> FieldSpecs \[\][config.FieldSpec]
+
+#### Example
+> ```
+> apiVersion: builtin
+>  kind: NamespaceTransformer
+>  metadata:
+>    name: not-important-to-example
+>    namespace: test
+>  fieldSpecs:
+>  - path: metadata/namespace
+>    create: true
+>  - path: subjects
+>    kind: RoleBinding
+>    group: rbac.authorization.k8s.io
+>  - path: subjects
+>    kind: ClusterRoleBinding
+>    group: rbac.authorization.k8s.io
+> ```
+
+
+
+
+
+## _PatchesJson6902_
+### Usage via `kustomization.yaml`
+
+#### field name: `patchesJson6902`
+
+Each entry in this list should resolve to
+a kubernetes object and a JSON patch that will be applied
+to the object.
+The JSON patch is documented at https://tools.ietf.org/html/rfc6902
+
+target field points to a kubernetes object within the same kustomization
+by the object's group, version, kind, name and namespace.
+path field is a relative file path of a JSON patch file.
+The content in this patch file can be either in JSON format as
+
+```
+ [
+   {"op": "add", "path": "/some/new/path", "value": "value"},
+   {"op": "replace", "path": "/some/existing/path", "value": "new value"}
+ ]
+ ```
+
+or in YAML format as
+
+```
+- op: add
+  path: /some/new/path
+  value: value
+- op: replace
+  path: /some/existing/path
+  value: new value
+```
+
+```
+patchesJson6902:
+- target:
+    version: v1
+    kind: Deployment
+    name: my-deployment
+  path: add_init_container.yaml
+- target:
+    version: v1
+    kind: Service
+    name: my-service
+  path: add_service_annotation.yaml
+```
+
+The patch content can be an inline string as well:
+
+```
+patchesJson6902:
+- target:
+    version: v1
+    kind: Deployment
+    name: my-deployment
+  patch: |-
+    - op: add
+      path: /some/new/path
+      value: value
+    - op: replace
+      path: /some/existing/path
+      value: "new value"
+```
+
+### Usage via plugin
+#### Arguments
+> Target [types.PatchTarget]
+>
+> Path   string
+>
+> JsonOp string
+
+#### Example
+> ```
+> apiVersion: builtin
+> kind: PatchJson6902Transformer
+> metadata:
+>   name: not-important-to-example
+> target:
+>   group: apps
+>   version: v1
+>   kind: Deployment
+>   name: my-deploy
+> path: jsonpatch.json
+> ```
+
+
+## _PatchesStrategicMerge_
+### Usage via `kustomization.yaml`
+
+#### field name: `patchesStrategicMerge`
+
+Each entry in this list should be either a relative
+file path or an inline content
+resolving to a partial or complete resource
+definition.
+
+The names in these (possibly partial) resource
+files must match names already loaded via the
+`resources` field.  These entries are used to
+_patch_ (modify) the known resources.
+
+Small patches that do one thing are best, e.g. modify
+a memory request/limit, change an env var in a
+ConfigMap, etc.  Small patches are easy to review and
+easy to mix together in overlays.
+
+```
+patchesStrategicMerge:
+- service_port_8888.yaml
+- deployment_increase_replicas.yaml
+- deployment_increase_memory.yaml
+```
+
+The patch content can be a inline string as well.
+```
+patchesStrategicMerge:
+- |-
+  apiVersion: apps/v1
+  kind: Deployment
+  metadata:
+    name: nginx
+  spec:
+    template:
+      spec:
+        containers:
+          - name: nginx
+            image: nignx:latest
+```
+
+Note that kustomize does not support more than one patch
+for the same object that contain a _delete_ directive. To remove
+several fields / slice elements from an object create a single
+patch that performs all the needed deletions.
+
+### Usage via plugin
+
+#### Arguments
+
+> Paths \[\][types.PatchStrategicMerge]
+>
+> Patches string
+
+
+#### Example
+> ```
+> apiVersion: builtin
+> kind: PatchStrategicMergeTransformer
+> metadata:
+>   name: not-important-to-example
+> paths:
+> - patch.yaml
+> ```
+
+
+## _PatchTransformer_
+### Usage via `kustomization.yaml`
+
+#### field name: `patches`
+
+Each entry in this list should resolve to an Patch
+object, which includes a patch and a target selector. 
+The patch can be either a strategic merge patch or a
+JSON patch. it can be either a patch file or an inline
+string. The target selects
+resources by group, version, kind, name, namespace,
+labelSelector and annotationSelector. A resource
+which matches all the specified fields is selected
+to apply the patch.
+
+```
+patches:
+- path: patch.yaml
+  target:
+    group: apps
+    version: v1
+    kind: Deployment
+    name: deploy.*
+    labelSelector: "env=dev"
+    annotationSelector: "zone=west"
+- patch: |-
+    - op: replace
+      path: /some/existing/path
+      value: new value
+  target:
+    kind: MyKind
+    labelSelector: "env=dev"        
+```
+
+The `name` and `namespace` fields of the patch target selector are
+automatically anchored regular expressions. This means that the value `myapp`
+is equivalent to `^myapp$`. 
+
+### Usage via plugin
+#### Arguments
+
+> Path string
+>
+> Patch string
+>
+> Target \*[types.Selector] 
+
+#### Example
+> ```
+> apiVersion: builtin
+> kind: PatchTransformer
+> metadata:
+>   name: not-important-to-example
+> patch: '[{"op": "replace", "path": "/spec/template/spec/containers/0/image", "value": "nginx:latest"}]'
+> target:
+>   name: .*Deploy
+>   kind: Deployment
+> ```
+
+
+
+
+## _PrefixSuffixTransformer_
+### Usage via `kustomization.yaml`
+
+#### field names: `namePrefix`, `nameSuffix`
+
+Prepends or postfixes the value to the names
+of all resources.
+
+E.g. a deployment named `wordpress` could
+become `alices-wordpress` or  `wordpress-v2`
+or `alices-wordpress-v2`. 
+
+```
+namePrefix: alices-
+nameSuffix: -v2
+```
+
+The suffix is appended before the content hash if
+the resource type is ConfigMap or Secret.
+
+### Usage via plugin
+#### Arguments
+
+> Prefix     string
+>
+> Suffix     string
+>
+> FieldSpecs \[\][config.FieldSpec]
+
+#### Example
+> ```
+> apiVersion: builtin
+> kind: PrefixSuffixTransformer
+> metadata:
+>   name: not-important-to-example
+> prefix: baked-
+> suffix: -pie
+> fieldSpecs:
+>   - path: metadata/name
+> ```
+
+
+
+## _ReplicaCountTransformer_
+### Usage via `kustomization.yaml`
+
+#### field name: `replicas`
+
+Replicas modified the number of replicas for a resource.
+
+E.g. Given this kubernetes Deployment fragment:
+
+```
+kind: Deployment
+metadata:
+  name: deployment-name
+spec:
+  replicas: 3
+```
+
+one can change the number of replicas to 5
+by adding the following to your kustomization:
+
+```
+replicas:
+- name: deployment-name
+  count: 5
+```
+
+This field accepts a list, so many resources can
+be modified at the same time.
+
+As this declaration does not take in a `kind:` nor a `group:`
+it will match any `group` and `kind` that has a matching name and
+that is one of:
+- `Deployment`
+- `ReplicationController`
+- `ReplicaSet`
+- `StatefulSet`
+
+For more complex use cases, revert to using a patch.
+
+### Usage via plugin
+
+#### Arguments
+
+> Replica [types.Replica]
+>
+> FieldSpecs \[\][config.FieldSpec]
+
+#### Example
+> ```
+> apiVersion: builtin
+> kind: ReplicaCountTransformer
+> metadata:
+>   name: not-important-to-example
+> replica:
+>   name: myapp
+>   count: 23
+> fieldSpecs:
+> - path: spec/replicas
+>   create: true
+>   kind: Deployment
+> - path: spec/replicas
+>   create: true
+>   kind: ReplicationController
+> ```
+
+
+
+## _SecretGenerator_
+
+### Usage via `kustomization.yaml`
+
+#### field name: `secretGenerator`
+
+Each entry in the argument list
+results in the creation of
+one Secret resource
+(it's a generator of n secrets).
+
+```
+secretGenerator:
+- name: app-tls
+  files:
+  - secret/tls.cert
+  - secret/tls.key
+  type: "kubernetes.io/tls"
+- name: app-tls-namespaced
+  # you can define a namespace to generate
+  # a secret in, defaults to: "default"
+  namespace: apps
+  files:
+  - tls.crt=catsecret/tls.cert
+  - tls.key=secret/tls.key
+  type: "kubernetes.io/tls"
+- name: env_file_secret
+  envs:
+  - env.txt
+  type: Opaque
+```
+
+### Usage via plugin
+
+#### Arguments
+
+> [types.ObjectMeta]
+>
+> [types.GeneratorOptions]
+>
+> [types.SecretArgs]
+
+#### Example
+
+> ```
+> apiVersion: builtin
+> kind: SecretGenerator
+> metadata:
+>   name: my-secret
+>   namespace: whatever
+> behavior: merge
+> envs:
+> - a.env
+> - b.env
+> files:
+> - obscure=longsecret.txt
+> literals:
+> - FRUIT=apple
+> - VEGETABLE=carrot
+> ```

docs/plugins/goPluginGuidedExample.md

@@ -1,10 +1,14 @@
 # Go Plugin Guided Example for Linux
 
-This is a (no reading allowed!) 60 second copy/paste guided
-example.  Full plugin docs [here](README.md).
-
 [SopsEncodedSecrets repository]: https://github.com/monopole/sopsencodedsecrets
 [Go plugin]: https://golang.org/pkg/plugin
+[Go plugin caveats]: goPluginCaveats.md
+
+This is a (no reading allowed!) 60 second copy/paste guided
+example.
+
+Full plugin docs [here](README.md).
+Be sure to read the [Go plugin caveats].
 
 This demo uses a Go plugin, `SopsEncodedSecrets`,
 that lives in the [sopsencodedsecrets repository].
@@ -17,23 +21,32 @@ current setup.
 
 #### requirements
 
- * linux, git, curl, Go 1.12
- * Google cloud (gcloud) install
- * a Google account (will use Google kms -
-   volunteers needed to convert to a GPG example).
+* linux, git, curl, Go 1.12
+
+For encryption
+
+* gpg
+
+Or
+
+* Google cloud (gcloud) install
+* a Google account with KMS permission
 
 ## Make a place to work
 
-```
+```shell
+# Keeping these separate to avoid cluttering the DEMO dir.
 DEMO=$(mktemp -d)
+tmpGoPath=$(mktemp -d)
 ```
 
 ## Install kustomize
 
-Need v3.0.0 for what follows:
+Need v3.0.0 for what follows, and you must _compile_
+it (not download the binary from the release page):
 
-```
-GOBIN=$DEMO/bin go get sigs.k8s.io/kustomize/v3/cmd/kustomize@v3.0.0-pre
+```shell
+GOPATH=$tmpGoPath go install sigs.k8s.io/kustomize/v3/cmd/kustomize
 ```
 
 ## Make a home for plugins
@@ -54,8 +67,8 @@ The kustomize program reads the config file
 kustomization file), then locates the Go plugin's
 object code at the following location:
 
-> ```
-> $XGD_CONFIG_HOME/kustomize/plugin/$apiVersion/$lKind/$kind.so
+> ```shell
+> $XDG_CONFIG_HOME/kustomize/plugin/$apiVersion/$lKind/$kind.so
 > ```
 
 where `lKind` holds the lowercased kind.  The
@@ -74,11 +87,11 @@ left to plugins to find their own config.
 This demo will house the plugin it uses at the
 ephemeral directory
 
-```
+```shell
 PLUGIN_ROOT=$DEMO/kustomize/plugin
 ```
 
-and ephemerally set `XGD_CONFIG_HOME` on a command
+and ephemerally set `XDG_CONFIG_HOME` on a command
 line below.
 
 ### What apiVersion and kind?
@@ -97,10 +110,10 @@ to a plugin.
 This demo uses a plugin called _SopsEncodedSecrets_,
 and it lives in the [SopsEncodedSecrets repository].
 
-Somewhat arbitrarily, we'll chose to install 
+Somewhat arbitrarily, we'll chose to install
 this plugin with
 
-```
+```shell
 apiVersion=mygenerators
 kind=SopsEncodedSecrets
 ```
@@ -111,7 +124,7 @@ By convention, the ultimate home of the plugin
 code and supplemental data, tests, documentation,
 etc. is the lowercase form of its kind.
 
-```
+```shell
 lKind=$(echo $kind | awk '{print tolower($0)}')
 ```
 
@@ -121,7 +134,7 @@ In this case, the repo name matches the lowercase
 kind already, so we just clone the repo and get
 the proper directory name automatically:
 
-```
+```shell
 mkdir -p $PLUGIN_ROOT/${apiVersion}
 cd $PLUGIN_ROOT/${apiVersion}
 git clone git@github.com:monopole/sopsencodedsecrets.git
@@ -129,7 +142,7 @@ git clone git@github.com:monopole/sopsencodedsecrets.git
 
 Remember this directory:
 
-```
+```shell
 MY_PLUGIN_DIR=$PLUGIN_ROOT/${apiVersion}/${lKind}
 ```
 
@@ -138,16 +151,16 @@ MY_PLUGIN_DIR=$PLUGIN_ROOT/${apiVersion}/${lKind}
 Plugins may come with their own tests.
 This one does, and it hopefully passes:
 
-```
+```shell
 cd $MY_PLUGIN_DIR
 go test SopsEncodedSecrets_test.go
 ```
 
 Build the object code for use by kustomize:
 
-```
+```shell
 cd $MY_PLUGIN_DIR
-go build -buildmode plugin -o ${kind}.so ${kind}.go
+GOPATH=$tmpGoPath go build -buildmode plugin -o ${kind}.so ${kind}.go
 ```
 
 This step may succeed, but kustomize might
@@ -163,7 +176,7 @@ On load failure
    version of Go (_go1.12_) on the same `$GOOS`
    (_linux_) and `$GOARCH` (_amd64_) used to build
    the kustomize being [used in this demo].
-   
+
  * change the plugin's dependencies in its `go.mod`
    to match the versions used by kustomize (check
    kustomize's `go.mod` used in its tagged commit).
@@ -180,11 +193,11 @@ reusable instead of bizarrely woven throughout the
 code as a individual special cases.
 
 ## Create a kustomization
-​
+
 Make a kustomization directory to
 hold all your config:
 
-```
+```shell
 MYAPP=$DEMO/myapp
 mkdir -p $MYAPP
 ```
@@ -194,12 +207,13 @@ Make a config file for the SopsEncodedSecrets plugin.
 Its `apiVersion` and `kind` allow the plugin to be
 found:
 
-```
+```shell
 cat <<EOF >$MYAPP/secGenerator.yaml
 apiVersion: ${apiVersion}
 kind: ${kind}
 metadata:
-  name: forbiddenValues
+  name: mySecretGenerator
+name: forbiddenValues
 namespace: production
 file: myEncryptedData.yaml
 keys:
@@ -214,7 +228,7 @@ This plugin expects to find more data in
 Make a kustomization file referencing the plugin
 config:
 
-```
+```shell
 cat <<EOF >$MYAPP/kustomization.yaml
 commonLabels:
   app: hello
@@ -223,31 +237,46 @@ generators:
 EOF
 ```
 
-Now for the hard part.  Generate the real encrypted data.
+Now generate the real encrypted data.
 
+### Assure you have an encryption tool installed
 
-### Assure you have a Google Cloud sops key ring.
+We're going to use [sops](https://github.com/mozilla/sops) to encode a file. Choose either GPG or Google Cloud KMS as the secret provider to continue.
 
-We're going to use [sops](https://github.com/mozilla/sops) to encode a file.
+#### GPG
 
 Try this:
 
+```shell
+gpg --list-keys
 ```
+
+If it returns a list, presumably you've already created keys. If not, try import test keys from sops for dev.
+
+```shell
+curl https://raw.githubusercontent.com/mozilla/sops/master/pgp/sops_functional_tests_key.asc | gpg --import
+SOPS_PGP_FP="1022470DE3F0BC54BC6AB62DE05550BC07FB1A0A"
+```
+
+#### Google Cloude KMS
+
+Try this:
+
+```shell
 gcloud kms keys list --location global --keyring sops
 ```
 
-If it succeeds, presumably you've already
-created keys and placed them in a keyring called `sops`.
-If not, do this:
+If it succeeds, presumably you've already created keys and placed them in a keyring called sops. If not, do this:
 
-```
+```shell
 gcloud kms keyrings create sops --location global
 gcloud kms keys create sops-key --location global \
     --keyring sops --purpose encryption
 ```
 
 Extract your keyLocation for use below:
-```
+
+```shell
 keyLocation=$(\
     gcloud kms keys list --location global --keyring sops |\
     grep GOOGLE | cut -d " " -f1)
@@ -256,42 +285,73 @@ echo $keyLocation
 
 ### Install `sops`
 
-```
-GOBIN=$DEMO/bin go install go.mozilla.org/sops/cmd/sops
+```shell
+GOPATH=$tmpGoPath go install go.mozilla.org/sops/cmd/sops
 ```
 
-### Create data encrypted with your Google Cloud key
+### Create data encrypted with your private key
 
 Create raw data to encrypt:
-```
+
+```shell
 cat <<EOF >$MYAPP/myClearData.yaml
 VEGETABLE: carrot
 ROCKET: saturn-v
 FRUIT: apple
 CAR: dymaxion
 EOF
-
 ```
 
 Encrypt the data into file the plugin wants to read:
 
+With PGP
+
+```shell
+$tmpGoPath/bin/sops --encrypt \
+  --pgp $SOPS_PGP_FP \
+  $MYAPP/myClearData.yaml >$MYAPP/myEncryptedData.yaml
 ```
-$DEMO/bin/sops --encrypt \
+
+Or GCP KMS
+
+```shell
+$tmpGoPath/bin/sops --encrypt \
   --gcp-kms $keyLocation \
   $MYAPP/myClearData.yaml >$MYAPP/myEncryptedData.yaml
 ```
 
-
 Review the files
-```
+
+```shell
 tree $DEMO
 ```
 
+This should look something like:
+
+> ```shell
+> /tmp/tmp.0kIE9VclPt
+> ├── kustomize
+> │   └── plugin
+> │       └── mygenerators
+> │           └── sopsencodedsecrets
+> │               ├── go.mod
+> │               ├── go.sum
+> │               ├── LICENSE
+> │               ├── README.md
+> │               ├── SopsEncodedSecrets.go
+> │               ├── SopsEncodedSecrets.so
+> │               └── SopsEncodedSecrets_test.go
+> └── myapp
+>     ├── kustomization.yaml
+>     ├── myClearData.yaml
+>     ├── myEncryptedData.yaml
+>     └── secGenerator.yaml
+> ```
 
 ## Build your app, using the plugin:
 
-```
-XDG_CONFIG_HOME=$DEMO $DEMO/bin/kustomize build --enable_alpha_plugins $MYAPP
+```shell
+XDG_CONFIG_HOME=$DEMO $tmpGoPath/bin/kustomize build --enable_alpha_plugins $MYAPP
 ```
 
 This should emit a kubernetes secret, with
@@ -299,10 +359,9 @@ encrypted data for the names `ROCKET` and `CAR`.
 
 Above, if you had set
 
-> ```
+> ```shell
 > PLUGIN_ROOT=$HOME/.config/kustomize/plugin
 > ```
 
 there would be no need to use `XDG_CONFIG_HOME` in the
 _kustomize_ command above.
-

docs/v3.0.0.md

@@ -1,18 +1,15 @@
 # kustomize 3.0.0
 
 This release is basically [v2.1.0](v2.1.0.md),
-with some post-v2.1.0 bugs fixed and a `v3` in Go
-package paths.
+with many post-v2.1.0 bugs fixed (in about 150
+commits) and a `v3` in Go package paths.
 
 [plugin]: https://github.com/kubernetes-sigs/kustomize/tree/master/docs/plugins
 
 The major version increment to `v3` puts a new
 floor on a stable API for [plugin] developers
 (both _Go_ plugin developers and _exec_ plugin
-developers who happen to use Go), to carry them
-through the coming series of minor releases and
-patches.
-
+developers who happen to use Go).
 
 ### Why so soon after v2.1.0?
 

docs/v3.1.0.md

@@ -0,0 +1,127 @@
+# kustomize 3.1.0
+
+
+## Extended patches
+Since this version, Kustomize allows applying one patch to multiple resources. This works for both Strategic Merge Patch and JSON Patch. Take a look at [patch multiple objects](../examples/patchMultipleObjects.md).
+
+## Improved Resource Matching
+
+Multiple improvements have been made to allow the user to leverage "namespace"
+instead/or with "name suffix/prefix" to segregate resources.
+
+### Patch resolution improvement
+
+The following example demonstrates how using the namespace field in the patch definition,
+will let the user define two different patches against two different Deployment having the
+same "deploy1" name but in different namespaces in the same Kustomize context/folder.
+Unless the `namespace:` field has been specified in the kustomization.yaml, no namespace
+value will be handled as Kubernetes `default` namespace.
+
+```yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: deploy1
+  namespace: main
+spec:
+  template:
+    spec:
+      containers:
+      - name: nginx
+        env:
+        - name: ANOTHERENV
+          value: TESTVALUE
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: deploy1
+  namespace: production
+spec:
+  template:
+    spec:
+      containers:
+      - name: main
+        env:
+        - name: ANOTHERENV
+          value: PRODVALUE
+```
+
+
+### Variable resolution improvement
+
+It is possible to add namespace field to the variable declaration. In the following example,
+two `Service` objects with the same `elasticsearch` name have been declared.
+Specifying the namespace in the objRef of the corresponding varriables, allows Kustomize to
+resovlve thoses variables.
+If the namespace is not specified, Kustomize will handle it has a "wildcard" value.
+
+Extract of kustomization.yaml:
+
+```yaml
+vars:
+- name: elasticsearch-test-protocol
+  objref:
+    kind: Service
+    name: elasticsearch
+    namespace: test
+    apiVersion: v1
+  fieldref:
+    fieldpath: spec.ports[0].protocol
+- name: elasticsearch-dev-protocol
+  objref:
+    kind: Service
+    name: elasticsearch
+    namespace: dev
+    apiVersion: v1
+  fieldref:
+    fieldpath: spec.ports[0].protocol
+
+```
+
+### Simultaneous change of names and namespaces
+
+Kustomize is now able to deal with simultaneous changes of name and namespace.
+Special attention has been paid the handling of: 
+- ClusterRoleBinding/RoleBinding "subjects" field,
+- ValidatingWebhookConfiguration "webhooks" field.
+
+The user should be able to use a kustomization.yaml as shown in the example bellow
+even if ClusterRoleBind,RoleBinding and ValidatingWebookConfiguration are part of the
+resources he needs to declare.
+
+Extract of kustomization.yaml:
+
+```yaml
+namePrefix: pfx-
+nameSuffix: -sfx
+namespace: testnamespace
+
+resources:
+...
+```
+
+### Resource and Kustomize Context matching.
+
+Kustomize is now able to support more aggregation patterns.
+
+If for instance, the top level of kustomization.yaml, is simply
+combining sub-components, (as in the following example), Kustomize has improved
+resource matching capabilities. This removes some of the constraints which were
+present on the utilization of prefix/suffix and namespace transformers in the
+individual components.
+
+```yaml
+resources:
+- ../component1
+- ../component2
+- ../component3
+```
+
+## Other improvements
+
+- Image transformation has been improved. This allows the user to update the sha256 of
+  an image with another sha256.
+- Multiple default transformer configuration entries have been added, removing the need for the
+  user to add them as part of the `configurations:` section of the kustomization.yaml.
+- `kustomize` help command has been tidied up.

docs/v3.2.0.md

@@ -0,0 +1,29 @@
+# kustomize 3.2.0
+
+
+## Inline Patch
+Since this version, Kustomize allows inline patches in all three of `patchesStrategicMerge`, `patchesJson6902` and `patches`. Take a look at [inline patch](../examples/inlinePatch.md).
+
+## New Subcommand
+
+Since this version, one can create a kustomization.yaml file in a directory through a `create` subcommand.
+
+Create a new overlay from the base ../base
+```
+kustomize create --resources ../base
+```
+
+Create a new kustomization detecing resources in the current directory
+```
+kustomize create --autodetect
+```
+
+Once can also add all resources in the current directory recursively by
+
+```
+kustomize create --autodetect --recursive
+```
+
+### New Example Generator
+A new example generator of using go-getter to download resources is added. Take a look at [go-getter generator](../examples/goGetterGeneratorPlugin.md).
+

docs/versioningPolicy.md

@@ -232,11 +232,11 @@ moment forward.
 [beta-level rules]: https://github.com/kubernetes/community/blob/master/contributors/devel/api_changes.md#alpha-beta-and-stable-versions
 [changes]: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api_changes.md
 [adapt]: https://github.com/kubernetes-sigs/kustomize/blob/master/pkg/types/kustomization.go#L166
-[special]: https://github.com/kubernetes/community/blob/master/contributors/devel/api-conventions.md#resources
+[special]: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#resources
 [k8s API]: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md
 [conventions]: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md
 [release process]: ../releasing/README.md
 [kustomization]: glossary.md#kustomization
-[`kind`]: https://github.com/kubernetes/community/blob/master/contributors/devel/api-conventions.md#types-kinds
+[`kind`]: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#types-kinds
 [`apiVersion`]: https://kubernetes.io/docs/concepts/overview/kubernetes-api/#api-versioning
 [semantic versioning]: https://semver.org

docs/zh/INSTALL.md

@@ -33,7 +33,7 @@ chmod u+x kustomize
 
 使用 [Go] v1.10.1 或更高版本安装（如果可以访问 [golang.org]）：
 
-<!-- @installkustomize @test -->
+<!-- @installkustomize @testAgainstLatestRelease -->
 ```
 go install sigs.k8s.io/kustomize/v3/cmd/kustomize
 ```

docs/zh/README.md

@@ -6,7 +6,7 @@
 
  * [示例](../../examples) - 各种使用流程和概念的详细演示。
 
- * [术语表](../glossary.md) - 用于消除术语歧义。
+ * [术语表](glossary.md) - 用于消除术语歧义。
 
  * [Kustomize 字段](fields.md) - 介绍 [kustomization](../glossary.md#kustomization) 文件中各字段的含义。
 
@@ -19,6 +19,8 @@
 
 ## 发行说明
 
+ * [3.1](../v3.1.0.md) - 2019年7月下旬，扩展 patches 和改进的资源匹配。
+
  * [3.0](../v3.0.0.md) - 2019年6月下旬，插件开发者发布。
 
  * [2.1](../v2.1.0.md) - 2019年6月18日

docs/zh/glossary.md

@@ -0,0 +1,308 @@
+# 词汇表
+
+[CRD spec]: https://kubernetes.io/docs/tasks/access-kubernetes-api/custom-resources/custom-resource-definitions/
+[CRD]: #custom-resource-definition
+[DAM]: #声明式应用程序管理
+[Declarative Application Management]: https://github.com/kubernetes/community/blob/master/contributors/design-proposals/architecture/declarative-application-management.md
+[JSON]: https://www.json.org/
+[JSONPatch]: https://tools.ietf.org/html/rfc6902
+[JSONMergePatch]: https://tools.ietf.org/html/rfc7386
+[Resource]: #resource
+[YAML]: http://www.yaml.org/start.html
+[application]: #application
+[apply]: #apply
+[apt]: https://en.wikipedia.org/wiki/APT_(Debian)
+[base]: #base
+[bases]: #base
+[bespoke]: #bespoke-configuration
+[gitops]: #gitops
+[k8s]: #kubernetes
+[kubernetes]: #kubernetes
+[kustomize]: #kustomize
+[kustomization]: #kustomization
+[kustomizations]: #kustomization
+[off-the-shelf]: #off-the-shelf-configuration
+[overlay]: #overlay
+[overlays]: #overlay
+[patch]: #patch
+[patches]: #patch
+[patchJson6902]: #patchjson6902
+[patchExampleJson6902]: https://github.com/kubernetes-sigs/kustomize/blob/master/examples/jsonpatch.md
+[patchesJson6902]: #patchjson6902
+[proposal]: https://github.com/kubernetes/community/pull/1629
+[rebase]: https://git-scm.com/docs/git-rebase
+[资源]: #资源
+[resources]: #resource
+[root]: #kustomization-root
+[rpm]: https://en.wikipedia.org/wiki/Rpm_(software)
+[strategic-merge]: https://git.k8s.io/community/contributors/devel/sig-api-machinery/strategic-merge-patch.md
+[target]: #target
+[transformer]: #transformer
+[variant]: #variant
+[variants]: #variant
+[workflow]: workflows.md
+
+## 应用
+
+**应用**是为某种目的关联起来的一组 Kubernetes 资源，例如一个前有负载均衡器，后有数据库的 Web 服务器。用标签、命名和元数据将[资源]组织起来，可以进行**添加**或**删除**等操作。
+
+有提案（[Declarative Application Management]）描述了一种称为应用的新的 Kubernetes 资源。更加正式的描述了这一思路，并在应用程序级别提供了运维和仪表盘的支持。
+
+[Kustomize] 对 Kubernetes 资源进行配置，其中描述的应用程序资源只是另一种普通的资源。
+
+## Apply
+
+**Apply** 这个动词在 Kubernetes 的上下文中，指的是一个 Kubernetes 命令以及能够对集群施加影响的进程内 [API 端点](https://goo.gl/UbCRuf)。
+
+用户可以将对集群的运行要求用一组完整的资源列表的形式进行表达，通过 **apply** 命令进行提交。
+
+集群把新提交的资源和之前提交的状态以及当前的实际状态进行合并，形成新的状态。这就是 Kubernetes 的状态管理过程。
+
+## Base
+
+**Base** 指的是会被其它 [Kustomization] 引用的 [Kustomization]。
+
+包括 [Overlay] 在内的任何 Kustomization，都可以作为其它 Kustomization 的 Base。
+
+Base 对引用自己的 Overlay 并无感知。
+
+Base 和 [Overlay] 可以作为 Git 仓库中的唯一内容，用于简单的 [GitOps] 管理。对仓库的变更可以触发构建、测试以及部署过程。
+
+## 定制配置
+
+**定制**配置是由组织为满足自身需要，在内部创建和管理的 [Kustomization] 和[资源]。
+
+和**定制配置**关联的 [Workflow] 比关联到通用配置的 [Workflow] 要简单一些，原因是通用配置是共享的，需要周期性的跟踪他人作出的变更。
+
+## Custom resource definition
+
+可以通过定制 CRD ([CRD spec]) 的方式对 Kubernetes API 进行扩展。
+
+CRD 定义的[资源]是一种全新的资源，可以和 ConfigMap、Deployment 之类的原生资源以相同的方式来使用。
+
+Kustomize 能够生成自定义资源，但是要完成这个目标，必须给出对应的 CRD，这样才能正确的对这种结构进行处理。
+
+## 声明式应用程序管理
+
+Kustomize 鼓励对声明式应用程序管理（[Declarative Application Management]）的支持，这种方式是一系列 Kubernetes 集群管理的最佳实践。Kustomize 应该可以：
+
+- 适用于任何配置，例如自有配置、共享配置、无状态、有状态等。
+- 支持通用配置，以及创建变体（例如开发、预发布、生产）。
+- 开放使用原生 Kubernetes API，而不是隐藏它们。
+- 不会给版本控制系统和集成的评审和审计工作造成困难。
+- 用 Unix 的风格和其它工具进行协作。
+- 避免使用模板、领域特定的语言等额外的学习内容。
+
+## 生成器
+
+生成器生成的资源，可以直接使用，也可以输出给转换器（[Transformer]）。
+
+## GitOps
+
+一种 DevOps 或者 CICD 流程，这种流程以 Git 作为唯一的事实，并且在这种事实发生变化时采取措施（例如构建、测试和部署）。
+
+## Kustomization
+
+**Kustomization** 这个词可以指 `kustomization.yaml` 这个文件，更常见的情况是一个包含了 `kustomization.yaml` 及其所有直接引用文件的相对路径（所有不需要 URL 的本地数据）。
+
+也就是说，如果在 [Kustomize] 的上下文中说到 **Kustomization**，可能是以下的情况之一：
+
+- 一个叫做 `kustomization.yaml` 的文件。
+- 一个压缩包（包含 YAML 文件以及它的引用文件）。
+- 一个 Git 压缩包。
+- 一个 Git 仓库的 URL。
+
+一个 Kustomization 文件包含的[字段](fields.md)，分为四个类别：
+
+- `resources`：待定制的现存[资源]，示例字段：`resources`、`crds`。
+- `generator`：将要创建的**新**资源，示例字段：`configMapGenerator`（传统）、`secretGenerator`（传统）、`generators`（v2.1）
+- `transformer`：对前面提到的新旧资源进行**处理**的方式。示例字段：`namePrefix`、`nameSuffix`、`images`、`commonLabels`、`patchesJson6902` 等。在 v2.1 中还有更多的 `transformer`。
+- `meta`：会对上面几种字段产生影响。示例字段：`vars`、`namespace`、`apiVersion`、`kind` 等。
+
+## Kustomization root
+
+直接包含 `kustomization.yaml` 文件的目录。
+
+处理 Kustomization 文件时，可能访问到该目录以内或以外的文件。
+
+像 YAML 资源这样的数据文件，或者用于生成 ConfigMap 或 Secret 的包含 `name=value` 的文本文件，或者用于补丁转换的补丁文件，必须**在这个目录的内部**，需要显式的使用**相对路径**来表达。
+
+v2.1 中有一个特殊选项 `--load_restrictions none` 能够放宽这个限制，从而让不同的 Kustomization 可以共享补丁文件。
+
+可以用 URL、绝对路径或者相对路径引用其它的 Kustomization（包含 `kustomization.yaml` 文件的其它目录）。
+
+如果 `A` Kustomization 依赖 `B` Kustomization，那么：
+
+- `B` 不能包含 `A`。
+- `B` 不能依赖 `A`，间接依赖也不可以。
+
+`A` 可以包含 `B`，但是这样的话，最简单的方式可能是让 `A` 直接依赖 `B` 的资源，并去除 `B` 的 `kustomization.yaml` 文件（就是把 `B` 合并到 `A`）。
+
+通常情况下，`B` 和 `A` 处于同级目录，或者 `B` 放在一个完全独立的 Git 仓库里，可以从任意的 Kustomization 进行引用。
+
+常见布局大致如下：
+
+> ```
+> ├── base
+> │   ├── deployment.yaml
+> │   ├── kustomization.yaml
+> │   └── service.yaml
+> └── overlays
+>     ├── dev
+>     │   ├── kustomization.yaml
+>     │   └── patch.yaml
+>     ├── prod
+>     │   ├── kustomization.yaml
+>     │   └── patch.yaml
+>     └── staging
+>         ├── kustomization.yaml
+>         └── patch.yaml
+> ```
+
+`dev`、`prod` 以及 `staging` 是否依赖于 `base`，要根据 `kustomization.yaml` 具体判断。
+
+## Kubernetes
+
+[Kubernetes](https://kubernetes.io) 是一个开源软件，为容器化应用提供了自动部署、伸缩和管理的能力。
+
+它经常会被简写为 `k8s`。
+
+## Kubernetes 风格的对象
+
+[必要字段]: https://kubernetes.io/docs/concepts/overview/working-with-objects/kubernetes-objects/#required-fields
+
+用 YAML 或者 JSON 文件表达一个对象，其中包含一些[必要字段]。`kind` 字段用于标识对象类型，`metadata/name` 字段用于区分实例，`apiVersion` 表示的是版本（如果有多个版本的话）。
+
+## Kustomize
+
+`kustomize` 是一个面向 Kubernetes 的命令行工具，用一种无模板、结构化的的方式为为声明式配置提供定制支持。
+
+`面向 Kubernetes` 的意思是 Kustomize 对 API 资源、Kubernetes 概念（例如名称、标签、命名空间等）、以及资源补丁是有支持的。
+
+Kustomize 是 [DAM] 的一个实现。
+
+## 通用配置
+
+通用配置是一种用于共享的 Kustomization 以及资源。
+
+例如创建一个这样的 Github 仓库：
+
+> ```
+> github.com/username/someapp/
+>   kustomization.yaml
+>   deployment.yaml
+>   configmap.yaml
+>   README.md
+> ```
+
+其他人可以 `fork` 这个仓库，并把它们的 Fork `clone` 到本地进行定制。
+
+用户可以用这个克隆回来的版本作为 [Base]，在此基础上定制 [Overlay] 来满足自身需求。
+
+## Overlay
+
+`Overlay` 是一个 依赖于其它 Kustomization 的 Kustomization。
+
+Overlay 引用（通过文件路径、URI 或者别的什么方式）的 [Kustomization] 被称为 [Base]。
+
+Overlay 无法脱离 Base 独立生效。
+
+Overlay 可以作为其它 Overlay 的 Base。
+
+通常 Overlay 都是不止一个的，因为实际情况中就需要为单一 Base 创建不同的[变体]，例如 `development`、`QA`、`production` 等。
+
+总的说来，这些变体使用的资源基本是一致的，只有一些简单的差异，例如 Deployment 的实例数量、特定 Pod 的 CPU 资源、ConfigMap 中的数据源定义等。
+
+可以这样把配置提交到集群：
+
+> ```
+>  kustomize build someapp/overlays/staging |\
+>      kubectl apply -f -
+>
+>  kustomize build someapp/overlays/production |\
+>      kubectl apply -f -
+> ```
+
+对 Base 的使用是隐性的——Overlay 的依赖是指向 Base 的。
+
+请参考 [root]。
+
+## 包
+
+在 Kustomize 中，`包`是没有意义的，Kustomize 并无意成为 [apt]、[rpm] 那样的传统包管理工具。
+
+## Patch
+
+修改资源的通用指令。
+
+有两种功能类似但是实现不同的补丁方式：[strategic merge patch](#patchstrategicmerge) 和 [JSON patch](#patchjson6902)。
+
+## patchStrategicMerge
+
+`patchStrategicMerge` 是 [strategic-merge] 风格的补丁（SMP）。
+
+SMP 看上去像个不完整的 Kubernetes 资源 YAML。SMP 中包含 `TypeMeta` 字段，用于表明这个补丁的目标[资源]的 `group/version/kind/name`，剩余的字段是一个嵌套的结构，用于指定新的字段值，例如镜像标签。
+
+缺省情况下，SMP 会**替换**目标值。如果目标值是一个字符串，这种行为是合适的，但是如果目标值是个列表，可能就不合适了。
+
+可以加入 `directive` 来修改这种行为，，可以接受的 `directive` 包括 `replace`（缺省）、`merge`（不替换列表）、`delete` 等（[相关说明][strategic-merge]）。
+
+注意对自定义资源来说，SMP 会被当作 [json merge patches][JSONMergePatch].
+
+有趣的事实：所有的资源文件都可以当作 SMP 使用，相同 `group/version/kind/name` 资源中的匹配字段会被替换，其它内容则保持不变。
+
+## patchJson6902
+
+`patchJson6902` 引用一个 Kubernetes 资源，并用 [JSONPatch] 指定了修改这一资源的方法。
+
+`patchJson6902` 几乎可以做到所有 `patchStrategicMerge` 的功能，但是语法更加简单，参考[示例][patchExampleJson6902]
+
+## 插件
+
+Kustomize 可以使用的一段代码，但是无需编译到 Kustomize 内部，可以作为 Kustomization 的一部分，生成或转换 Kubernetes 资源。
+
+[插件](../plugins)的细节。
+
+## 资源
+
+在 REST-ful API 的上下文中，资源是 `GET`、`PUT` 或者 `POST` 等 HTTP 操作的目标。Kubernetes 提供了 REST-ful API 界面，用于和客户端进行交互。
+
+在 Kustomization 的上下文中，资源是一个相对于 [root] 的相对路径，指向 [YAML] 或者 [JSON] 文件，描述了一个 Kubernetes API 对象，例如 Deployment 或者 ConfigMap，或者一个 Kustomization、或者一个指向 Kustomization 的 URL。
+
+或者说任何定义了[对象](https://kubernetes.io/docs/concepts/overview/working-with-objects/kubernetes-objects/#required-fields)的格式正确的 YAML 文件，其中包含了 `kind` 和 `metadata/name` 字段，都是资源。
+
+## Root
+
+参看 [kustomization root][root].
+
+## sub-target / sub-application / sub-package
+
+不存在 `sub-xxx`，只有 [Base] 和 [Overlay]。
+
+## Target
+
+`target` 是 `kustomize build` 的参数，例如：
+
+> ```
+>  kustomize build $target
+> ```
+
+`$target` 必须是一个指向 [Kustomization] 的路径或者 URL。
+
+要创建用于进行 [Apply] 操作的资源，`target` 中必须包含或者引用所有相关信息。
+
+[Base] 或者 [Overlay] 都可以作为 `target`。
+
+## Transformer
+
+转换器能够修改资源，或者在 `kustomize build` 的过程中获取资源的信息。
+
+## 变体
+
+在集群中把 [Overlay] 应用到 [Base] 上的产物称为**变体**。
+
+比如 `staging` 和 `production` 两个 Overlay，都修改了同样的 Base，来创建各自的变体。
+
+`staging` 变体包含了一组用来保障测试过程的资源，或者一些想要看到生产环境下一个版本的外部用户。
+
+`production` 变体用于承载生产流量，可能使用大量的副本，分配更多的 CPU 和内存。

examples/README.md

@@ -2,71 +2,71 @@ English | [简体中文](zh/README.md)
 
 # Examples
 
-These examples assume that `kustomize` is on your `$PATH`.
+To run these examples, your `$PATH` must contain `kustomize`.
+See the [installation instructions](../docs/INSTALL.md).
 
-They are covered by [pre-commit](../travis/pre-commit.sh)
-tests, and should work with HEAD
-
-<!-- @installkustomize @test -->
-```
-go get sigs.k8s.io/kustomize/v3/cmd/kustomize
-```
+These examples are [tested](../travis/pre-commit.sh)
+to work with the latest _released_ version of kustomize.
 
 Basic Usage
 
   * [configGenerations](configGeneration.md) -
    Rolling update when ConfigMapGenerator changes.
-   
+
   * [combineConfigs](combineConfigs.md) -
    Mixing configuration data from different owners
    (e.g. devops/SRE and developers).
-  
+
   * [generatorOptions](generatorOptions.md) -
-   Modifying behavior of all ConfigMap and Secret generators. 
+   Modifying behavior of all ConfigMap and Secret generators.
 
   * [vars](wordpress/README.md) - Injecting k8s runtime data into
      container arguments (e.g. to point wordpress to a SQL service) by vars.
- 
+
   * [image names and tags](image.md) - Updating image names and tags without applying a patch.
- 
+
   * [remote target](remoteBuild.md) - Building a kustomization from a github URL
- 
-  * [json patch](jsonpatch.md) - Apply a json patch in a kustomization    
+
+  * [json patch](jsonpatch.md) - Apply a json patch in a kustomization
+
+  * [patch multiple objects](patchMultipleObjects.md) - Apply a patch to multiple objects
 
 Advanced Usage
-  
+
 - generator plugins:
-  
+
    * [last mile helm](chart.md) - Make last mile modifications to
      a helm chart.
-     
-   * [secret generation](secretGeneratorPlugin.md) - Generating secrets from a plugin. 
+
+   * [secret generation](secretGeneratorPlugin.md) - Generating secrets from a plugin.
+
+   * [remote sources](goGetterGeneratorPlugin.md) - Generating from remote sources.
 
 - transformer plugins:
    * [validation transformer](validationTransformer/README.md) -
    validate resources through a transformer
-  
+
 - customize builtin transformer configurations
-  
+
    * [transformer configs](transformerconfigs/README.md) - Customize transformer configurations
-   
+
 
 Multi Variant Examples
 
   * [hello world](helloWorld/README.md) - Deploy multiple
    (differently configured) variants of a simple Hello
    World server.
-  
+
   * [LDAP](ldap/README.md) - Deploy multiple
      (differently configured) variants of a LDAP server.
-     
+
   * [springboot](springboot/README.md) - Create a Spring Boot
    application production configuration from scratch.
 
   * [mySql](mySql/README.md) - Create a MySQL production
    configuration from scratch.
- 
+
   * [breakfast](breakfast.md) - Customize breakfast for
      Alice and Bob.
-     
-  * [multibases](multibases/README.md) - Composing three variants (dev, staging, production) with a common base.  
+
+  * [multibases](multibases/README.md) - Composing three variants (dev, staging, production) with a common base.

examples/breakfast.md

@@ -6,14 +6,14 @@
 
 Define a place to work:
 
-<!-- @makeWorkplace @test -->
+<!-- @makeWorkplace @testAgainstLatestRelease -->
 ```
 DEMO_HOME=$(mktemp -d)
 ```
 
 Make a place to put the base breakfast configuration:
 
-<!-- @baseDir @test -->
+<!-- @baseDir @testAgainstLatestRelease -->
 ```
 mkdir -p $DEMO_HOME/breakfast/base
 ```
@@ -21,7 +21,7 @@ mkdir -p $DEMO_HOME/breakfast/base
 Make a `kustomization` to define what goes into
 breakfast.  This breakfast has coffee and pancakes:
 
-<!-- @baseKustomization @test -->
+<!-- @baseKustomization @testAgainstLatestRelease -->
 ```
 cat <<EOF >$DEMO_HOME/breakfast/base/kustomization.yaml
 resources:
@@ -34,7 +34,7 @@ Here's a _coffee_ type.  Give it a `kind` and `metdata/name` field
 to conform to [kubernetes API object style]; no other
 file or definition is needed:
 
-<!-- @coffee @test -->
+<!-- @coffee @testAgainstLatestRelease -->
 ```
 cat <<EOF >$DEMO_HOME/breakfast/base/coffee.yaml
 kind: Coffee
@@ -50,7 +50,7 @@ The `name` field merely distinguishes this instance of
 coffee from others (if there were any).
 
 Likewise, define _pancakes_:
-<!-- @pancakes @test -->
+<!-- @pancakes @testAgainstLatestRelease -->
 ```
 cat <<EOF >$DEMO_HOME/breakfast/base/pancakes.yaml
 kind: Pancakes
@@ -64,7 +64,7 @@ EOF
 Make a custom [variant] of breakfast for Alice, who
 likes her coffee hot:
 
-<!-- @aliceOverlay @test -->
+<!-- @aliceOverlay @testAgainstLatestRelease -->
 ```
 mkdir -p $DEMO_HOME/breakfast/overlays/alice
 
@@ -87,7 +87,7 @@ EOF
 
 And likewise a [variant] for Bob, who wants _five_ pancakes, with strawberries:
 
-<!-- @bobOverlay @test -->
+<!-- @bobOverlay @testAgainstLatestRelease -->
 ```
 mkdir -p $DEMO_HOME/breakfast/overlays/bob
 
@@ -111,14 +111,14 @@ EOF
 
 One can now generate the configs for Alice’s breakfast:
 
-<!-- @generateAlice @test -->
+<!-- @generateAlice @testAgainstLatestRelease -->
 ```
 kustomize build $DEMO_HOME/breakfast/overlays/alice
 ```
 
 Likewise for Bob:
 
-<!-- @generateBob @test -->
+<!-- @generateBob @testAgainstLatestRelease -->
 ```
 kustomize build $DEMO_HOME/breakfast/overlays/bob
 ```

examples/combineConfigs.md

@@ -128,7 +128,7 @@ defined in the [helloworld] demo.
 
 It will all live in this work directory:
 
-<!-- @makeWorkplace @test -->
+<!-- @makeWorkplace @testAgainstLatestRelease -->
 ```
 DEMO_HOME=$(mktemp -d)
 ```
@@ -139,7 +139,7 @@ DEMO_HOME=$(mktemp -d)
 
 Make a place to put the base configuration:
 
-<!-- @baseDir @test -->
+<!-- @baseDir @testAgainstLatestRelease -->
 ```
 mkdir -p $DEMO_HOME/base
 ```
@@ -150,7 +150,7 @@ environments. Here we're only defining a java
 properties file, and a `kustomization` file that
 references it.
 
-<!-- @baseKustomization @test -->
+<!-- @baseKustomization @testAgainstLatestRelease -->
 ```
 cat <<EOF >$DEMO_HOME/base/common.properties
 color=blue
@@ -171,14 +171,14 @@ EOF
 Make an abbreviation for the parent of the overlay
 directories:
 
-<!-- @overlays @test -->
+<!-- @overlays @testAgainstLatestRelease -->
 ```
 OVERLAYS=$DEMO_HOME/overlays
 ```
 
 Create the files that define the _development_ overlay:
 
-<!-- @developmentFiles @test -->
+<!-- @developmentFiles @testAgainstLatestRelease -->
 ```
 mkdir -p $OVERLAYS/development
 
@@ -206,7 +206,7 @@ EOF
 
 One can now generate the configMaps for development:
 
-<!-- @runDev @test -->
+<!-- @runDev @testAgainstLatestRelease -->
 ```
 kustomize build $OVERLAYS/development
 ```
@@ -260,7 +260,7 @@ deletes unused configMaps.
 Next, create the files for the _production_ overlay:
 
 
-<!-- @productionFiles @test -->
+<!-- @productionFiles @testAgainstLatestRelease -->
 ```
 mkdir -p $OVERLAYS/production
 
@@ -287,7 +287,7 @@ EOF
 
 One can now generate the configMaps for production:
 
-<!-- @runProd @test -->
+<!-- @runProd @testAgainstLatestRelease -->
 ```
 kustomize build $OVERLAYS/production
 ```

examples/configGeneration.md

@@ -26,7 +26,7 @@ In this demo, the same [hello_world](helloWorld/README.md) is used while the Con
 ### Establish base and staging
 
 Establish the base with a configMapGenerator
-<!-- @establishBase @test -->
+<!-- @establishBase @testAgainstLatestRelease -->
 ```
 DEMO_HOME=$(mktemp -d)
 
@@ -53,7 +53,7 @@ EOF
 ```
 
 Establish the staging with a patch applied to the ConfigMap
-<!-- @establishStaging @test -->
+<!-- @establishStaging @testAgainstLatestRelease -->
 ```
 OVERLAYS=$DEMO_HOME/overlays
 mkdir -p $OVERLAYS/staging
@@ -91,7 +91,7 @@ configured with data from a configMap.
 The deployment refers to this map by name:
 
 
-<!-- @showDeployment @test -->
+<!-- @showDeployment @testAgainstLatestRelease -->
 ```
 grep -C 2 configMapKeyRef $BASE/deployment.yaml
 ```
@@ -117,7 +117,7 @@ collected](https://github.com/kubernetes-sigs/kustomize/issues/242).
 
 The _staging_ [variant] here has a configMap [patch]:
 
-<!-- @showMapPatch @test -->
+<!-- @showMapPatch @testAgainstLatestRelease -->
 ```
 cat $OVERLAYS/staging/map.yaml
 ```
@@ -128,7 +128,7 @@ resource spec.
 
 The ConfigMap it modifies is declared from a configMapGenerator.
 
-<!-- @showMapBase @test -->
+<!-- @showMapBase @testAgainstLatestRelease -->
 ```
 grep -C 4 configMapGenerator $BASE/kustomization.yaml
 ```
@@ -141,7 +141,7 @@ _not_ what gets used in the cluster.  By design,
 kustomize modifies names of ConfigMaps declared from ConfigMapGenerator.  To see the names
 ultimately used in the cluster, just run kustomize:
 
-<!-- @grepStagingName @test -->
+<!-- @grepStagingName @testAgainstLatestRelease -->
 ```
 kustomize build $OVERLAYS/staging |\
     grep -B 8 -A 1 staging-the-map
@@ -159,7 +159,7 @@ The suffix to the configMap name is generated from a
 hash of the maps content - in this case the name suffix
 is _k25m8k5k5m_:
 
-<!-- @grepStagingHash @test -->
+<!-- @grepStagingHash @testAgainstLatestRelease -->
 ```
 kustomize build $OVERLAYS/staging | grep k25m8k5k5m
 ```
@@ -167,7 +167,7 @@ kustomize build $OVERLAYS/staging | grep k25m8k5k5m
 Now modify the map patch, to change the greeting
 the server will use:
 
-<!-- @changeMap @test -->
+<!-- @changeMap @testAgainstLatestRelease -->
 ```
 sed -i.bak 's/pineapple/kiwi/' $OVERLAYS/staging/map.yaml
 ```
@@ -181,7 +181,7 @@ kustomize build $OVERLAYS/staging |\
 
 Run kustomize again to see the new configMap names:
 
-<!-- @grepStagingName @test -->
+<!-- @grepStagingName @testAgainstLatestRelease -->
 ```
 kustomize build $OVERLAYS/staging |\
     grep -B 8 -A 1 staging-the-map
@@ -192,7 +192,7 @@ in three new names ending in _cd7kdh48fd_ - one in the
 configMap name itself, and two in the deployment that
 uses the map:
 
-<!-- @countHashes @test -->
+<!-- @countHashes @testAgainstLatestRelease -->
 ```
 test 3 == \
   $(kustomize build $OVERLAYS/staging | grep cd7kdh48fd | wc -l); \

examples/configureBuiltinPlugin.md

@@ -0,0 +1,333 @@
+[builtin operations]: ../docs/plugins/builtins.md
+[builtin plugins]: ../docs/plugins/builtins.md
+[plugins]: ../docs/plugins
+[plugin]: ../docs/plugins
+[fields]: ../docs/fields.md
+[fields in a kustomization file]: ../docs/fields.md
+[TransformerConfig]: ../pkg/transformers/config/transformerconfig.go
+[kustomization]: ../docs/glossary.md#kustomization
+
+# Customizing kustomize
+
+The [fields in a kustomization file] allow the user to
+specify which resource files to use as input, how to
+_generate_ new resources, and how to _transform_ those
+resources - add labels, patch them, etc.
+
+These fields are simple (low argument count) directives.
+For example, the `commonAnnotations` field demands only a
+list of _name:value_ pairs.
+
+If using a field triggers behavior that pleases the user,
+everyone's happy.
+
+If not, the user can ask for new behavior to be implemented
+in kustomize proper (and wait), or the user can write a
+transformer or generator [plugin].  This latter option
+generally means writing code; a Go plugin, a Go binary,
+a C++ binary, a `bash` script - something.
+
+There's a third option.  If one merely wants to tweak
+behavior that already exists in kustomize, one may be able
+to do so by just writing some YAML.
+
+## Configure the builtin plugins
+
+All of kustomize's [builtin operations] are implemented 
+and usable as plugins.
+
+Using the fields is convenient and brief, but necessarily
+specifies only part of the entire plugin specification.  The
+unspecified part is defaulted to what are hopefully
+generally appealing values.
+
+If, instead, one invokes the plugins directly using the
+`transformers` or `generators` field, one can (indeed
+_must_) specify the entire plugin configuration.
+
+## Example: field vs plugin
+
+Define a place to work:
+
+<!-- @makeWorkplace @testAgainstLatestRelease -->
+```
+DEMO_HOME=$(mktemp -d)
+```
+
+### Using the `commonLabels` and `commonAnnotations` fields
+ 
+In this simple example, we'll use just two resources: a deployment and a service.
+
+Define them:
+
+<!-- @makeRes1 @testAgainstLatestRelease -->
+```
+cat <<EOF >$DEMO_HOME/deployment.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: deployment
+spec:
+  replicas: 10
+  template:
+    spec:
+      containers:
+      - name: the-container
+        image: monopole/hello:1
+EOF
+```
+
+<!-- @makeRes2 @testAgainstLatestRelease -->
+```
+cat <<EOF >$DEMO_HOME/service.yaml
+apiVersion: v1
+kind: Service
+metadata:
+  name: service
+spec:
+  type: LoadBalancer
+  ports:
+  - protocol: TCP
+    port: 8666
+    targetPort: 8080
+EOF
+```
+
+Now make a kustomization file that causes them
+to be read and transformed:
+
+<!-- @makeKustomization @testAgainstLatestRelease -->
+```
+cat <<'EOF' >$DEMO_HOME/kustomization.yaml
+namePrefix: hello-
+commonLabels:
+  app: hello
+commonAnnotations:
+  area: "51"
+  greeting: Take me to your leader
+resources:
+- deployment.yaml
+- service.yaml
+EOF
+```
+
+And run kustomize:
+
+<!-- @checkLabel @testAgainstLatestRelease -->
+```
+kustomize build $DEMO_HOME
+```
+
+The output will be something like
+
+> ```
+> apiVersion: v1
+> kind: Service
+> metadata:
+>   annotations:
+>     area: "51"
+>     greeting: Take me to your leader
+>   labels:
+>     app: hello
+>   name: hello-service
+> spec:
+>   ports:
+>   - port: 8666
+>     protocol: TCP
+>     targetPort: 8080
+>   selector:
+>     app: hello
+>   type: LoadBalancer
+> ---
+> apiVersion: apps/v1
+> kind: Deployment
+> metadata:
+>   annotations:
+>     area: "51"
+>     greeting: Take me to your leader
+>   labels:
+>     app: hello
+>   name: hello-deployment
+> spec:
+>   replicas: 10
+>   selector:
+>     matchLabels:
+>       app: hello
+>   template:
+>     metadata:
+>       annotations:
+>         area: "51"
+>         greeting: Take me to your leader
+>       labels:
+>         app: hello
+>     spec:
+>       containers:
+>       - image: monopole/hello:1
+>         name: the-container
+> ```
+
+Let's say we are unhappy with this result.
+
+We only want the annotations
+to be applied down in the pod templates,
+and don't want to see them in the metadata
+for Service or Deployment.
+
+We like that the label _app: hello_ ended up in
+
+ - Service `spec.selector`
+ - Deployment `spec.selector.matchLabels`
+ - Deployment `spec.template.metadata.labels`
+
+as this binds the Service (load balancer) to the pods,
+and the Deployment itself to its own pods -
+but we again don't care to see these labels in
+the metadata for the Service and the Deployment.
+
+
+### Configuring the builtin plugins instead
+
+To fine tune this, invoke the same transformations
+using the plugin approach.
+
+Change the kustomization file:
+
+<!-- @makeKustomization @testAgainstLatestRelease -->
+```
+cat <<'EOF' >$DEMO_HOME/kustomization.yaml
+namePrefix: hello-
+transformers:
+- myAnnotator.yaml
+- myLabeller.yaml
+resources:
+- deployment.yaml
+- service.yaml
+EOF
+```
+
+Then make the two plugin configuration files
+(`myAnnotator.yaml`, `myLabeller.yaml`)
+referred to by the `transformers` field above.
+For details about the fields to specify, see
+the documentation for the [builtin plugins].
+
+<!-- @makeAnnotatorPluginConfig @testAgainstLatestRelease -->
+```
+cat <<EOF >$DEMO_HOME/myAnnotator.yaml
+apiVersion: builtin
+kind: AnnotationsTransformer
+metadata:
+  name: notImportantHere
+annotations:
+  area: 51
+  greeting: take me to your leader
+fieldSpecs:
+- kind: Deployment
+  path: spec/template/metadata/annotations
+  create: true
+EOF
+```
+
+<!-- @makeLabelPluginConfig @testAgainstLatestRelease -->
+```
+cat <<EOF >$DEMO_HOME/myLabeller.yaml
+apiVersion: builtin
+kind: LabelTransformer
+metadata:
+  name: notImportantHere
+labels:
+  app: hello
+fieldSpecs:
+- kind: Service
+  path: spec/selector
+  create: true
+- kind: Deployment
+  path: spec/selector/matchLabels
+  create: true
+- kind: Deployment
+  path: spec/template/metadata/labels
+  create: true
+EOF
+```
+
+Finally, run kustomize again:
+
+<!-- @checkLabel @testAgainstLatestRelease -->
+```
+kustomize build $DEMO_HOME
+```
+
+The output should resemble the following,
+with fewer labels and annotations.
+
+> ```
+> apiVersion: v1
+> kind: Service
+> metadata:
+>   name: hello-service
+> spec:
+>   ports:
+>   - port: 8666
+>     protocol: TCP
+>     targetPort: 8080
+>   selector:
+>     app: hello
+>   type: LoadBalancer
+> ---
+> apiVersion: apps/v1
+> kind: Deployment
+> metadata:
+>   name: hello-deployment
+> spec:
+>   replicas: 10
+>   selector:
+>     matchLabels:
+>       app: hello
+>   template:
+>     metadata:
+>       annotations:
+>         area: "51"
+>         greeting: take me to your leader
+>       labels:
+>         app: hello
+>     spec:
+>       containers:
+>       - image: monopole/hello:1
+>         name: the-container
+> ```
+
+
+## The old way to do this
+
+The original (and still functional) way to customize
+kustomize is to specify a `configurations` field in the
+kustomization file.
+
+This field, normally omitted because it overrides hardcoded
+data, accepts a list of file path arguments.  The files, in
+turn, specify which fields in which k8s objects should be
+affected by particular builtin transformations.  It's a
+global configuration cutting across transformations, and
+doesn't effect generators at all.
+
+At `build` time, the configuration files are unmarshalled
+into one instance of [TransformerConfig].  This
+object, _plus_ the field values for `namePrefix`, etc.  are
+fed into the transformation code to build the output.
+
+The best way to write these custom configuration files is to
+generate the files from the hardcoded values built into
+kustomize via these commands:
+
+> ```
+> mkdir /tmp/junk
+> kustomize config save -d /tmp/junk
+> ```
+
+One can then edit those file or files, and specify the
+resulting edited files in a `configurations:`
+field in a kustomization file used in a `build`.
+
+Using plugins _completely ignores_ both hard coded
+tranformer configuration, and any configuration loaded by
+the `configuration` field.

examples/generatorOptions.md

@@ -13,7 +13,7 @@ DEMO_HOME=$(mktemp -d)
 
 Create a kustomization and add a ConfigMap generator to it.
 
-<!-- @createCMGenerator @test -->
+<!-- @createCMGenerator @testAgainstLatestRelease -->
 ```
 cat > $DEMO_HOME/kustomization.yaml << EOF
 configMapGenerator:
@@ -25,7 +25,7 @@ EOF
 ```
 
 Add following generatorOptions
-<!-- @addGeneratorOptions @test -->
+<!-- @addGeneratorOptions @testAgainstLatestRelease -->
 ```
 cat >> $DEMO_HOME/kustomization.yaml << EOF
 generatorOptions:
@@ -39,7 +39,7 @@ EOF
 Run `kustomize build` and make sure that the generated ConfigMap
  
  - doesn't have name suffix
-    <!-- @verify @test -->
+    <!-- @verify @testAgainstLatestRelease -->
     ```
     test 1 == \
     $(kustomize build $DEMO_HOME | grep "name: my-configmap$" | wc -l); \

examples/goGetterGeneratorPlugin.md

@@ -0,0 +1,132 @@
+# Remote Sources
+
+Kustomize supports building a [remote target], but the URLs are limited to common [Git repository specs].
+
+To extend the supported format, Kustomize has a [plugin] system that allows one to integrate third-party tools such as [hashicorp/go-getter] to "download things from a string URL suing a variety of protocols", extract the content and generated resources as part of kustomize build.
+
+[remote target]: https://github.com/kubernetes-sigs/kustomize/blob/master/examples/remoteBuild.md
+[Git repository specs]: https://github.com/kubernetes-sigs/kustomize/blob/master/pkg/git/repospec_test.go
+[plugin]: ../docs/plugins
+[hashicorp/go-getter]: https://github.com/hashicorp/go-getter
+
+## Make a place to work
+
+<!-- @makeWorkplace @test -->
+```sh
+DEMO_HOME=$(mktemp -d)
+mkdir -p $DEMO_HOME/base
+```
+
+## Use a remote kustomize layer
+
+Define a kustomization representing your _local_ variant (aka environment).
+
+This could involve any number of kustomizations (see other examples), but in this case just add the name prefix `my-` to all resources:
+
+<!-- @writeKustLocal @test -->
+```sh
+cat <<'EOF' >$DEMO_HOME/kustomization.yaml
+namePrefix:  my-
+resources:
+- base/
+EOF
+```
+
+It refer a remote base defined as below:
+
+<!-- @writeKustLocal @test -->
+```sh
+cat <<'EOF' >$DEMO_HOME/base/kustomization.yaml
+generators:
+- goGetter.yaml
+EOF
+```
+
+The base refers to a generator configuration file called `goGetter.yaml`.
+
+This file lets one specify the source URL, and other things like sub path in the package, defaulting to base directory, and command to run under the path, defaulting to `kustomize build`.
+
+Create the config file `goGetter.yaml`, specifying the arbitrarily chosen name _example_:
+
+<!-- @writeGeneratorConfig @test -->
+```sh
+cat <<'EOF' >$DEMO_HOME/base/goGetter.yaml
+apiVersion: someteam.example.com/v1
+kind: GoGetter
+metadata:
+  name: example
+url: github.com/kustless/kustomize-examples.git
+EOF
+```
+
+Because this particular YAML file is listed in the `generators:` stanza of a kustomization file, it is treated as the binding between a generator plugin - identified by the _apiVersion_ and _kind_ fields - and other fields that configure the plugin.
+
+Download the plugin to your `DEMO_HOME` and make it executable:
+
+<!-- @installPlugin @test -->
+```sh
+plugin=plugin/someteam.example.com/v1/gogetter/GoGetter
+curl -s --create-dirs -o \
+"$DEMO_HOME/kustomize/$plugin" \
+"https://raw.githubusercontent.com/\
+kubernetes-sigs/kustomize/master/$plugin"
+
+chmod a+x $DEMO_HOME/kustomize/$plugin
+```
+
+Define a helper function to run kustomize with the correct environment and flags for plugins:
+
+<!-- @defineKustomizeIt @test -->
+```sh
+function kustomizeIt {
+  XDG_CONFIG_HOME=$DEMO_HOME \
+  kustomize build --enable_alpha_plugins \
+    $DEMO_HOME/$1
+}
+```
+
+Finally, build the local variant.  Notice that all
+resource  names now have the `my-` prefix:
+
+<!-- @doLocal @test -->
+```sh
+clear
+kustomizeIt
+```
+
+Compare local variant to remote base:
+
+<!-- @doCompare @test-->
+```sh
+diff <(kustomizeIt) <(kustomizeIt base) | more
+
+...
+<   name: my-remote-cm
+---
+>   name: remote-cm
+```
+
+To see the unmodified but extracted sources, run kustomize on the base.  Every invocation here is re-downloading and re-building the package.
+
+<!-- @showBase @test -->
+```sh
+kustomizeIt base
+```
+
+## Use non-kustomize remote sources
+
+Sometimes the remote sources does not include `kustomization.yaml`. To use that in the plugin, set command to override the default build.
+
+<!-- @setCommand @test -->
+```sh
+echo "command: cat resources.yaml" >>$DEMO_HOME/base/goGetter.yaml
+```
+
+Finally, built it
+
+<!-- @finalLocal @test -->
+```sh
+kustomizeIt
+```
+
+and observe the output includes raw `resources.yaml` instead of building result of remote `kustomization.yaml`.

examples/helloWorld/README.md

@@ -22,7 +22,7 @@ Steps:
 
 First define a place to work:
 
-<!-- @makeWorkplace @test -->
+<!-- @makeWorkplace @testAgainstLatestRelease -->
 ```
 DEMO_HOME=$(mktemp -d)
 ```
@@ -44,7 +44,7 @@ To keep this document shorter, the base resources are
 off in a supplemental data directory rather than
 declared here as HERE documents.  Download them:
 
-<!-- @downloadBase @test -->
+<!-- @downloadBase @testAgainstLatestRelease -->
 ```
 BASE=$DEMO_HOME/base
 mkdir -p $BASE
@@ -57,7 +57,7 @@ curl -s -o "$BASE/#1.yaml" "https://raw.githubusercontent.com\
 
 Look at the directory:
 
-<!-- @runTree @test -->
+<!-- @runTree @testAgainstLatestRelease -->
 ```
 tree $DEMO_HOME
 ```
@@ -78,7 +78,7 @@ One could immediately apply these resources to a
 cluster:
 
 > ```
-> kubectl apply -f $DEMO_HOME/base
+> kubectl apply -k $DEMO_HOME/base
 > ```
 
 to instantiate the _hello_ service.  `kubectl`
@@ -88,7 +88,7 @@ would only recognize the resource files.
 
 The `base` directory has a [kustomization] file:
 
-<!-- @showKustomization @test -->
+<!-- @showKustomization @testAgainstLatestRelease -->
 ```
 more $BASE/kustomization.yaml
 ```
@@ -96,7 +96,7 @@ more $BASE/kustomization.yaml
 Optionally, run `kustomize` on the base to emit
 customized resources to `stdout`:
 
-<!-- @buildBase @test -->
+<!-- @buildBase @testAgainstLatestRelease -->
 ```
 kustomize build $BASE
 ```
@@ -106,14 +106,14 @@ kustomize build $BASE
 A first customization step could be to change the _app
 label_ applied to all resources:
 
-<!-- @addLabel @test -->
+<!-- @addLabel @testAgainstLatestRelease -->
 ```
 sed -i.bak 's/app: hello/app: my-hello/' \
     $BASE/kustomization.yaml
 ```
 
 See the effect:
-<!-- @checkLabel @test -->
+<!-- @checkLabel @testAgainstLatestRelease -->
 ```
 kustomize build $BASE | grep -C 3 app:
 ```
@@ -127,7 +127,7 @@ Create a _staging_ and _production_ [overlay]:
  * Web server greetings from these cluster
    [variants] will differ from each other.
 
-<!-- @overlayDirectories @test -->
+<!-- @overlayDirectories @testAgainstLatestRelease -->
 ```
 OVERLAYS=$DEMO_HOME/overlays
 mkdir -p $OVERLAYS/staging
@@ -139,7 +139,7 @@ mkdir -p $OVERLAYS/production
 In the `staging` directory, make a kustomization
 defining a new name prefix, and some different labels.
 
-<!-- @makeStagingKustomization @test -->
+<!-- @makeStagingKustomization @testAgainstLatestRelease -->
 ```
 cat <<'EOF' >$OVERLAYS/staging/kustomization.yaml
 namePrefix: staging-
@@ -162,7 +162,7 @@ greeting from _Good Morning!_ to _Have a pineapple!_
 
 Also, enable the _risky_ flag.
 
-<!-- @stagingMap @test -->
+<!-- @stagingMap @testAgainstLatestRelease -->
 ```
 cat <<EOF >$OVERLAYS/staging/map.yaml
 apiVersion: v1
@@ -180,7 +180,7 @@ EOF
 In the production directory, make a kustomization
 with a different name prefix and labels.
 
-<!-- @makeProductionKustomization @test -->
+<!-- @makeProductionKustomization @testAgainstLatestRelease -->
 ```
 cat <<EOF >$OVERLAYS/production/kustomization.yaml
 namePrefix: production-
@@ -202,7 +202,7 @@ EOF
 Make a production patch that increases the replica
 count (because production takes more traffic).
 
-<!-- @productionDeployment @test -->
+<!-- @productionDeployment @testAgainstLatestRelease -->
 ```
 cat <<EOF >$OVERLAYS/production/deployment.yaml
 apiVersion: apps/v1
@@ -228,7 +228,7 @@ EOF
 
 Review the directory structure and differences:
 
-<!-- @listFiles @test -->
+<!-- @listFiles @testAgainstLatestRelease -->
 ```
 tree $DEMO_HOME
 ```
@@ -288,12 +288,12 @@ something like
 
 The individual resource sets are:
 
-<!-- @buildStaging @test -->
+<!-- @buildStaging @testAgainstLatestRelease -->
 ```
 kustomize build $OVERLAYS/staging
 ```
 
-<!-- @buildProduction @test -->
+<!-- @buildProduction @testAgainstLatestRelease -->
 ```
 kustomize build $OVERLAYS/production
 ```

examples/image.md

@@ -3,14 +3,14 @@
 
 Define a place to work:
 
-<!-- @makeWorkplace @test -->
+<!-- @makeWorkplace @testAgainstLatestRelease -->
 ```
 DEMO_HOME=$(mktemp -d)
 ```
 
 Make a `kustomization` containing a pod resource
 
-<!-- @createKustomization @test -->
+<!-- @createKustomization @testAgainstLatestRelease -->
 ```
 cat <<EOF >$DEMO_HOME/kustomization.yaml
 resources:
@@ -20,7 +20,7 @@ EOF
 
 Declare the pod resource
 
-<!-- @createDeployment @test -->
+<!-- @createDeployment @testAgainstLatestRelease -->
 ```
 cat <<EOF >$DEMO_HOME/pod.yaml
 apiVersion: v1
@@ -46,7 +46,7 @@ The image `busybox` and tag `1.29.0` can be changed by adding `images` in `kusto
 
 
 Add `images`:
-<!-- @addImages @test -->
+<!-- @addImages @testAgainstLatestRelease -->
 ```
 cd $DEMO_HOME
 kustomize edit set image busybox=alpine:3.6
@@ -61,14 +61,14 @@ The following `images` will be added to `kustomization.yaml`:
 > ```
 
 Now build this `kustomization`
-<!-- @kustomizeBuild @test -->
+<!-- @kustomizeBuild @testAgainstLatestRelease -->
 ```
 kustomize build $DEMO_HOME
 ```
 
 Confirm that this replaces _both_ busybox images and tags for `alpine:3.6`:
 
-<!-- @confirmImages @test -->
+<!-- @confirmImages @testAgainstLatestRelease -->
 ```
 test 2 = \
   $(kustomize build $DEMO_HOME | grep alpine:3.6 | wc -l); \

examples/inlinePatch.md

@@ -0,0 +1,265 @@
+[Strategic Merge Patch]: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-api-machinery/strategic-merge-patch.md
+[JSON Patch]: https://tools.ietf.org/html/rfc6902
+
+# Demo: Inline Patch
+
+A kustomization file supports patching in three ways:
+- patchesStrategicMerge: A list of patch files where each file is parsed as a [Stragetic Merge Patch].
+- patchesJSON6902: A list of patches and associated targetes, where each file is parsed as a [JSON Patch] and can only be applied to one target resource.
+- patches: A list of patches and their associated targets. The patch can be applied to multiple objects. It auto detects whether the patch is a [Strategic Merge Patch] or [JSON Patch].
+
+Since 3.2.0, all three support inline patch, where the patch content is put inside the kustomization file as a single string. With this feature, no separate patch files need to be created.
+
+Make a base kustomization containing a Deployment resource.
+<!-- @createKustomization @test -->
+```
+DEMO_HOME=$(mktemp -d)
+
+BASE=$DEMO_HOME/base
+mkdir $BASE
+
+cat <<EOF >$BASE/kustomization.yaml
+resources:
+- deployments.yaml
+EOF
+
+cat <<EOF >$BASE/deployments.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: deploy
+spec:
+  template:
+    metadata:
+      labels:
+        foo: bar
+    spec:
+      containers:
+        - name: nginx
+          image: nginx
+          args:
+          - one
+          - two
+EOF
+```
+
+
+## Inline Patch for PatchesStrategicMerge
+
+Create an overlay and add an inline patch in `patchesStrategicMerge` field to the kustomization file
+to change the image from `nginx` to `nginx:latest`.
+
+<!-- @addSMPatch @test -->
+```
+SMP_OVERLAY=$DEMO_HOME/smp
+mkdir $SMP_OVERLAY
+cat <<EOF >$SMP_OVERLAY/kustomization.yaml
+resources:
+- ../base
+
+patchesStrategicMerge:
+- |-
+  apiVersion: apps/v1
+  kind: Deployment
+  metadata:
+    name: deploy
+  spec:
+    template:
+      spec:
+        containers:
+        - name: nginx
+          image: nginx:latest
+          
+EOF
+```
+
+Running `kustomize build $SMP_OVERLAY`, in the output confirm that image is updated successfully.
+
+<!-- @confirmSMPatch @test -->
+```
+test 1 == \
+  $(kustomize build $SMP_OVERLAY | grep "image: nginx:latest" | wc -l); \
+  echo $?
+```
+
+The output is
+```yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: deploy
+spec:
+  template:
+    metadata:
+      labels:
+        foo: bar
+    spec:
+      containers:
+        - name: nginx
+          image: nginx:latest
+          args:
+          - one
+          - two
+```
+
+`$patch: delete` and `$patch: replace` also work in the inline patch. Change the inline patch to delete the container `nginx`.
+
+<!-- @addDeleteSMPatch @test -->
+```
+cat <<EOF >$SMP_OVERLAY/kustomization.yaml
+resources:
+- ../base
+
+patchesStrategicMerge:
+- |-
+  apiVersion: apps/v1
+  kind: Deployment
+  metadata:
+    name: deploy
+  spec:
+    template:
+      spec:
+        containers:
+        - name: nginx
+          $patch: delete
+          
+EOF
+```
+Running `kustomize build $SMP_OVERLAY`, in the output confirm that the `nginx` container has been deleted.
+
+<!-- @confirmDeleteSMPatch @test -->
+```
+test 0 == \
+  $(kustomize build $SMP_OVERLAY | grep "image: nginx" | wc -l); \
+  echo $?
+```
+
+The output is
+```
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: deploy
+spec:
+  template:
+    metadata:
+      labels:
+        foo: bar
+    spec:
+      containers: []
+```
+
+## Inline Patch for PatchesJson6902
+
+Create an overlay and add an inline patch in `patchesJSON6902` field to the kustomization file
+to change the image from `nginx` to `nginx:latest`.
+
+<!-- @addJSONPatch @test -->
+```
+JSON_OVERLAY=$DEMO_HOME/json
+mkdir $JSON_OVERLAY
+cat <<EOF >$JSON_OVERLAY/kustomization.yaml
+resources:
+- ../base
+
+patchesJSON6902:
+- target:
+    group: apps
+    version: v1
+    kind: Deployment
+    name: deploy
+  patch: |-
+    - op: replace
+      path: /spec/template/spec/containers/0/image
+      value: nginx:latest
+EOF
+```
+
+Running `kustomize build $JSON_OVERLAY`, in the output confirm that image is updated successfully.
+
+<!-- @confirmJSONPatch @test -->
+```
+test 1 == \
+  $(kustomize build $JSON_OVERLAY | grep "image: nginx:latest" | wc -l); \
+  echo $?
+```
+
+The output is
+```yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: deploy
+spec:
+  template:
+    metadata:
+      labels:
+        foo: bar
+    spec:
+      containers:
+        - name: nginx
+          image: nginx:latest
+          args:
+          - one
+          - two
+```
+
+## Inline Patch for Patches
+
+Create an overlay and add an inline patch in `patches` field to the kustomization file
+to change the image from `nginx` to `nginx:latest`.
+
+<!-- @addPatch @test -->
+```
+PATCH_OVERLAY=$DEMO_HOME/patch
+mkdir $PATCH_OVERLAY
+cat <<EOF > $PATCH_OVERLAY/kustomization.yaml
+resources:
+- ../base
+
+patches:
+- target:
+    kind: Deployment
+    name: deploy
+  patch: |-
+    apiVersion: apps/v1
+    kind: Deployment
+    metadata:
+      name: deploy
+    spec:
+      template:
+        spec:
+          containers:
+          - name: nginx
+            image: nginx:latest
+EOF
+```
+
+Running `kustomize build $PATCH_OVERLAY`, in the output confirm that image is updated successfully.
+
+<!-- @confirmPatch @test -->
+```
+test 1 == \
+  $(kustomize build $PATCH_OVERLAY | grep "image: nginx:latest" | wc -l); \
+  echo $?
+```
+
+The output is
+```yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: deploy
+spec:
+  template:
+    metadata:
+      labels:
+        foo: bar
+    spec:
+      containers:
+        - name: nginx
+          image: nginx:latest
+          args:
+          - one
+          - two
+```

examples/integration_tests.sh

@@ -28,7 +28,7 @@
 # before running it.
 #
 # At time of writing, its 'call point' was in
-# https://github.com/kubernetes/test-infra/blob/master/jobs/config.json
+# https://github.com/kubernetes/test-infra/blob/master/config/jobs/kubernetes-sigs/kustomize/kustomize-config.yaml
 
 function exitWith {
   local msg=$1
@@ -53,7 +53,7 @@ function setUpEnv {
     exitWith "Script must be run from $expectedRepo"
   fi
 
-  GO111MODULE=on go install . || \
+  GO111MODULE=on go install ./cmd/kustomize || \
     { exitWith "Failed to install kustomize."; }
 
   PATH=$GOPATH/bin:$PATH

examples/jsonpatch.md

@@ -6,7 +6,7 @@ The example below modifies an `Ingress` object with such a patch.
 
 Make a `kustomization` containing an ingress resource.
 
-<!-- @createIngress @test -->
+<!-- @createIngress @testAgainstLatestRelease -->
 ```
 DEMO_HOME=$(mktemp -d)
 
@@ -16,7 +16,7 @@ resources:
 EOF
 
 cat <<EOF >$DEMO_HOME/ingress.yaml
-apiVersion: extensions/v1beta1
+apiVersion: networking.k8s.io/v1beta1
 kind: Ingress
 metadata:
   name: my-ingress
@@ -36,7 +36,7 @@ Declare a JSON patch file to update two fields of the Ingress object:
 - change host from `foo.bar.com` to `foo.bar.io`
 - change servicePort from `80` to `8080`
 
-<!-- @addJsonPatch @test -->
+<!-- @addJsonPatch @testAgainstLatestRelease -->
 ```
 cat <<EOF >$DEMO_HOME/ingress_patch.json
 [
@@ -48,7 +48,7 @@ EOF
 
 You can also write the patch in YAML format. This example also shows the "add" operation:
 
-<!-- @addYamlPatch @test -->
+<!-- @addYamlPatch @testAgainstLatestRelease -->
 ```
 cat <<EOF >$DEMO_HOME/ingress_patch.yaml
 - op: replace
@@ -67,12 +67,12 @@ EOF
 
 Apply the patch by adding _patchesJson6902_ field in kustomization.yaml
 
-<!-- @applyJsonPatch @test -->
+<!-- @applyJsonPatch @testAgainstLatestRelease -->
 ```
 cat <<EOF >>$DEMO_HOME/kustomization.yaml
 patchesJson6902:
 - target:
-    group: extensions
+    group: networking.k8s.io
     version: v1beta1
     kind: Ingress
     name: my-ingress
@@ -81,14 +81,14 @@ EOF
 ```
 
 Running `kustomize build $DEMO_HOME`, in the output confirm that host has been updated correctly.
-<!-- @confirmHost @test -->
+<!-- @confirmHost @testAgainstLatestRelease -->
 ```
 test 1 == \
   $(kustomize build $DEMO_HOME | grep "host: foo.bar.io" | wc -l); \
   echo $?
 ```
 Running `kustomize build $DEMO_HOME`, in the output confirm that the servicePort has been updated correctly.
-<!-- @confirmServicePort @test -->
+<!-- @confirmServicePort @testAgainstLatestRelease -->
 ```
 test 1 == \
   $(kustomize build $DEMO_HOME | grep "servicePort: 8080" | wc -l); \
@@ -97,12 +97,12 @@ test 1 == \
 
 If the patch is YAML-formatted, it will be parsed correctly:
 
-<!-- @applyYamlPatch @test -->
+<!-- @applyYamlPatch @testAgainstLatestRelease -->
 ```
 cat <<EOF >>$DEMO_HOME/kustomization.yaml
 patchesJson6902:
 - target:
-    group: extensions
+    group: networking.k8s.io
     version: v1beta1
     kind: Ingress
     name: my-ingress
@@ -110,7 +110,7 @@ patchesJson6902:
 EOF
 ```
 
-<!-- @confirmYamlPatch @test -->
+<!-- @confirmYamlPatch @testAgainstLatestRelease -->
 ```
 test 1 == \
   $(kustomize build $DEMO_HOME | grep "path: /test" | wc -l); \

examples/ldap/README.md

@@ -18,7 +18,7 @@ Steps:
 
 First define a place to work:
 
-<!-- @makeWorkplace @test -->
+<!-- @makeWorkplace @testAgainstLatestRelease -->
 ```
 DEMO_HOME=$(mktemp -d)
 ```
@@ -38,7 +38,7 @@ To keep this document shorter, the base resources are
 off in a supplemental data directory rather than
 declared here as HERE documents.  Download them:
 
-<!-- @downloadBase @test -->
+<!-- @downloadBase @testAgainstLatestRelease -->
 ```
 BASE=$DEMO_HOME/base
 mkdir -p $BASE
@@ -53,7 +53,7 @@ curl -s -o "$BASE/#1" "$CONTENT/base\
 
 Look at the directory:
 
-<!-- @runTree @test -->
+<!-- @runTree @testAgainstLatestRelease -->
 ```
 tree $DEMO_HOME
 ```
@@ -84,7 +84,7 @@ would only recognize the resource files.
 
 The `base` directory has a [kustomization] file:
 
-<!-- @showKustomization @test -->
+<!-- @showKustomization @testAgainstLatestRelease -->
 ```
 more $BASE/kustomization.yaml
 ```
@@ -92,7 +92,7 @@ more $BASE/kustomization.yaml
 Optionally, run `kustomize` on the base to emit
 customized resources to `stdout`:
 
-<!-- @buildBase @test -->
+<!-- @buildBase @testAgainstLatestRelease -->
 ```
 kustomize build $BASE
 ```
@@ -101,14 +101,14 @@ kustomize build $BASE
 
 A first customization step could be to set the name prefix to all resources:
 
-<!-- @namePrefix @test -->
+<!-- @namePrefix @testAgainstLatestRelease -->
 ```
 cd $BASE
 kustomize edit set nameprefix "my-"
 ```
 
 See the effect:
-<!-- @checkNameprefix @test -->
+<!-- @checkNameprefix @testAgainstLatestRelease -->
 ```
 kustomize build $BASE | grep -C 3 "my-"
 ```
@@ -121,7 +121,7 @@ Create a _staging_ and _production_ [overlay]:
  * _Production_ has a higher replica count and a persistent disk.
  * [variants] will differ from each other.
 
-<!-- @overlayDirectories @test -->
+<!-- @overlayDirectories @testAgainstLatestRelease -->
 ```
 OVERLAYS=$DEMO_HOME/overlays
 mkdir -p $OVERLAYS/staging
@@ -132,7 +132,7 @@ mkdir -p $OVERLAYS/production
 
 Download the staging customization and patch.
 
-<!-- @downloadStagingKustomization @test -->
+<!-- @downloadStagingKustomization @testAgainstLatestRelease -->
 ```
 curl -s -o "$OVERLAYS/staging/#1" "$CONTENT/overlays/staging\
 /{config.env,deployment.yaml,kustomization.yaml}"
@@ -159,7 +159,7 @@ as well as 2 replica
 #### Production Kustomization
 
 Download the production customization and patch.
-<!-- @downloadProductionKustomization @test -->
+<!-- @downloadProductionKustomization @testAgainstLatestRelease -->
 ```
 curl -s -o "$OVERLAYS/production/#1" "$CONTENT/overlays/production\
 /{deployment.yaml,kustomization.yaml}"
@@ -196,7 +196,7 @@ The production customization adds 6 replica as well as a consistent disk.
 
 Review the directory structure and differences:
 
-<!-- @listFiles @test -->
+<!-- @listFiles @testAgainstLatestRelease -->
 ```
 tree $DEMO_HOME
 ```
@@ -258,12 +258,12 @@ The difference output should look something like
 
 The individual resource sets are:
 
-<!-- @buildStaging @test -->
+<!-- @buildStaging @testAgainstLatestRelease -->
 ```
 kustomize build $OVERLAYS/staging
 ```
 
-<!-- @buildProduction @test -->
+<!-- @buildProduction @testAgainstLatestRelease -->
 ```
 kustomize build $OVERLAYS/production
 ```

examples/multibases/README.md

@@ -20,13 +20,13 @@ that is just a single pod.
 
 Define a place to work:
 
-<!-- @makeWorkplace @test -->
+<!-- @makeWorkplace @testAgainstLatestRelease -->
 ```
 DEMO_HOME=$(mktemp -d)
 ```
 
 Define a common base:
-<!-- @makeBase @test -->
+<!-- @makeBase @testAgainstLatestRelease -->
 ```
 BASE=$DEMO_HOME/base
 mkdir $BASE
@@ -51,7 +51,7 @@ EOF
 ```
 
 Define a dev variant overlaying base:
-<!-- @makeDev @test -->
+<!-- @makeDev @testAgainstLatestRelease -->
 ```
 DEV=$DEMO_HOME/dev
 mkdir $DEV
@@ -64,7 +64,7 @@ EOF
 ```
 
 Define a staging variant overlaying base:
-<!-- @makeStaging @test -->
+<!-- @makeStaging @testAgainstLatestRelease -->
 ```
 STAG=$DEMO_HOME/staging
 mkdir $STAG
@@ -77,7 +77,7 @@ EOF
 ```
 
 Define a production variant overlaying base:
-<!-- @makeProd @test -->
+<!-- @makeProd @testAgainstLatestRelease -->
 ```
 PROD=$DEMO_HOME/production
 mkdir $PROD
@@ -90,7 +90,7 @@ EOF
 ```
 
 Then define a _Kustomization_ composing three variants together:
-<!-- @makeTopLayer @test -->
+<!-- @makeTopLayer @testAgainstLatestRelease -->
 ```
 cat <<EOF >$DEMO_HOME/kustomization.yaml
 resources:
@@ -119,7 +119,7 @@ Now the workspace has following directories
 
 Confirm that the `kustomize build` output contains three pod objects from dev, staging and production variants.
 
-<!-- @confirmVariants @test -->
+<!-- @confirmVariants @testAgainstLatestRelease -->
 ```
 test 1 == \
   $(kustomize build $DEMO_HOME | grep cluster-a-dev-myapp-pod | wc -l); \

examples/multibases/multi-namespace.md

@@ -8,13 +8,13 @@ following demonstrates this using a base that's just one pod.
 
 Define a place to work:
 
-<!-- @makeWorkplace @test -->
+<!-- @makeWorkplace @testAgainstLatestRelease -->
 ```
 DEMO_HOME=$(mktemp -d)
 ```
 
 Define a common base:
-<!-- @makeBase @test -->
+<!-- @makeBase @testAgainstLatestRelease -->
 ```
 BASE=$DEMO_HOME/base
 mkdir $BASE
@@ -39,7 +39,7 @@ EOF
 ```
 
 Define a variant in namespace-a overlaying base:
-<!-- @makeNamespaceA @test -->
+<!-- @makeNamespaceA @testAgainstLatestRelease -->
 ```
 NSA=$DEMO_HOME/namespace-a
 mkdir $NSA
@@ -60,7 +60,7 @@ EOF
 ```
 
 Define a variant in namespace-b overlaying base:
-<!-- @makeNamespaceB @test -->
+<!-- @makeNamespaceB @testAgainstLatestRelease -->
 ```
 NSB=$DEMO_HOME/namespace-b
 mkdir $NSB
@@ -81,7 +81,7 @@ EOF
 ```
 
 Then define a _Kustomization_ composing two variants together:
-<!-- @makeTopLayer @test -->
+<!-- @makeTopLayer @testAgainstLatestRelease -->
 ```
 cat <<EOF >$DEMO_HOME/kustomization.yaml
 resources:
@@ -107,7 +107,7 @@ Now the workspace has following directories
 
 Confirm that the `kustomize build` output contains two pod objects from namespace-a and namespace-b.
 
-<!-- @confirmVariants @test -->
+<!-- @confirmVariants @testAgainstLatestRelease -->
 ```
 test 2 == \
   $(kustomize build $DEMO_HOME| grep -B 4 "namespace: namespace-[ab]" | grep "name: myapp-pod" | wc -l); \

examples/mySql/README.md

@@ -11,7 +11,7 @@ In the production environment we want:
 - MySQL to use persistent disk for storing data.
 
 First make a place to work:
-<!-- @makeDemoHome @test -->
+<!-- @makeDemoHome @testAgainstLatestRelease -->
 ```
 DEMO_HOME=$(mktemp -d)
 ```
@@ -25,7 +25,7 @@ as HERE documents.
 
 Download them:
 
-<!-- @downloadResources @test -->
+<!-- @downloadResources @testAgainstLatestRelease -->
 ```
 curl -s  -o "$DEMO_HOME/#1.yaml" "https://raw.githubusercontent.com\
 /kubernetes-sigs/kustomize\
@@ -40,14 +40,14 @@ a file called `kustomization.yaml`.
 
 Start this file:
 
-<!-- @kustomizeYaml @test -->
+<!-- @kustomizeYaml @testAgainstLatestRelease -->
 ```
 touch $DEMO_HOME/kustomization.yaml
 ```
 
 ### Add the resources
 
-<!-- @addResources @test -->
+<!-- @addResources @testAgainstLatestRelease -->
 ```
 cd $DEMO_HOME
 
@@ -73,7 +73,7 @@ Arrange for the MySQL resources to begin with prefix
 _prod-_ (since they are meant for the _production_
 environment):
 
-<!-- @customizeLabel @test -->
+<!-- @customizeLabel @testAgainstLatestRelease -->
 ```
 cd $DEMO_HOME
 
@@ -91,7 +91,7 @@ cat kustomization.yaml
 This `namePrefix` directive adds _prod-_ to all
 resource names.
 
-<!-- @genNamePrefixConfig @test -->
+<!-- @genNamePrefixConfig @testAgainstLatestRelease -->
 ```
 kustomize build $DEMO_HOME
 ```
@@ -134,7 +134,7 @@ selector.
 `kustomize` does not have `edit set label` command to add
 a label, but one can always edit `kustomization.yaml` directly:
 
-<!-- @customizeLabels @test -->
+<!-- @customizeLabels @testAgainstLatestRelease -->
 ```
 sed -i.bak 's/app: helloworld/app: prod/' \
     $DEMO_HOME/kustomization.yaml
@@ -153,7 +153,7 @@ environment. So we want to use Persistent Disk in
 production. kustomize lets you apply `patchesStrategicMerge` to the
 resources.
 
-<!-- @createPatchFile @test -->
+<!-- @createPatchFile @testAgainstLatestRelease -->
 ```
 cat <<'EOF' > $DEMO_HOME/persistent-disk.yaml
 apiVersion: apps/v1beta2 # for versions before 1.9.0 use apps/v1beta2
@@ -173,7 +173,7 @@ EOF
 
 Add the patch file to `kustomization.yaml`:
 
-<!-- @specifyPatch @test -->
+<!-- @specifyPatch @testAgainstLatestRelease -->
 ```
 cat <<'EOF' >> $DEMO_HOME/kustomization.yaml
 patchesStrategicMerge:
@@ -199,7 +199,7 @@ The output of the following command can now be applied
 to the cluster (i.e. piped to `kubectl apply`) to
 create the production environment.
 
-<!-- @finalInflation @test -->
+<!-- @finalInflation @testAgainstLatestRelease -->
 ```
 kustomize build $DEMO_HOME  # | kubectl apply -f -
 ```

examples/patchMultipleObjects.md

@@ -0,0 +1,188 @@
+[Strategic Merge Patch]: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-api-machinery/strategic-merge-patch.md
+[JSON patches]: https://tools.ietf.org/html/rfc6902
+[label selector]: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#label-selectors
+
+
+# Demo: applying a patch to multiple resources
+
+A kustomization file supports customizing resources via both
+[Strategic Merge Patch] and [JSON patches]. Now one patch can be
+applied to multiple resources.
+
+This can be done by specifying a patch and a target selector as follows:
+```
+patches:
+- path: <PatchFile>
+  target:
+    group: <Group>
+    version: <Version>
+    kind: <Kind>
+    name: <Name>
+    namespace: <Namespace>
+    labelSelector: <LabelSelector>
+    annotationSelector: <AnnotationSelector>
+```
+Both `labelSelector` and `annotationSelector` should follow the convention in [label selector].
+Kustomize selects the targets which match all the fields in `target` to apply the patch.
+
+The example below shows how to inject a sidecar container for all deployment resources.
+
+Make a `kustomization` containing a Deployment resource.
+
+<!-- @createDeployment @testAgainstLatestRelease -->
+```
+DEMO_HOME=$(mktemp -d)
+
+cat <<EOF >$DEMO_HOME/kustomization.yaml
+resources:
+- deployments.yaml
+EOF
+
+cat <<EOF >$DEMO_HOME/deployments.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: deploy1
+spec:
+  template:
+    metadata:
+      labels:
+        old-label: old-value
+    spec:
+      containers:
+        - name: nginx
+          image: nginx
+          args:
+          - one
+          - two
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: deploy2
+spec:
+  template:
+    metadata:
+      labels:
+        key: value
+    spec:
+      containers:
+        - name: busybox
+          image: busybox
+EOF
+```
+
+Declare a Strategic Merge Patch file to inject a sidecar container:
+
+<!-- @addPatch @testAgainstLatestRelease -->
+```
+cat <<EOF >$DEMO_HOME/patch.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: not-important
+spec:
+  template:
+    spec:
+      containers:
+        - name: istio-proxy
+          image: docker.io/istio/proxyv2
+          args:
+          - proxy
+          - sidecar
+EOF
+```
+
+Apply the patch by adding _patches_ field in kustomization.yaml
+
+<!-- @applyPatch @testAgainstLatestRelease -->
+```
+cat <<EOF >>$DEMO_HOME/kustomization.yaml
+patches:
+- path: patch.yaml
+  target:
+    kind: Deployment
+EOF
+```
+
+Running `kustomize build $DEMO_HOME`, in the output confirm that both Deployment resources are patched correctly.
+
+<!-- @confirmPatch @testAgainstLatestRelease -->
+```
+test 2 == \
+  $(kustomize build $DEMO_HOME | grep "image: docker.io/istio/proxyv2" | wc -l); \
+  echo $?
+```
+
+The output is as follows:
+
+```yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: deploy1
+spec:
+  template:
+    metadata:
+      labels:
+        old-label: old-value
+    spec:
+      containers:
+      - args:
+        - proxy
+        - sidecar
+        image: docker.io/istio/proxyv2
+        name: istio-proxy
+      - args:
+        - one
+        - two
+        image: nginx
+        name: nginx
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: deploy2
+spec:
+  template:
+    metadata:
+      labels:
+        key: value
+    spec:
+      containers:
+      - args:
+        - proxy
+        - sidecar
+        image: docker.io/istio/proxyv2
+        name: istio-proxy
+      - image: busybox
+        name: busybox
+```
+
+## Target selector
+- Select resources with name matching `name*`
+  ```yaml
+  target:
+    name: name*
+  ```
+- Select all Deployment resources
+  ```yaml
+  target:
+    kind: Deployment
+  ```
+- Select resources matching label `app=hello`
+  ```yaml
+  target:
+    labelSelector: app=hello
+  ```
+- Select resources matching annotation `app=hello`
+  ```yaml
+  target:
+    annotationSelector: app=hello
+  ```
+- Select all Deployment resources matching label `app=hello`
+  ```yaml
+  target:
+    kind: Deployment
+    labelSelector: app=hello
+  ```

examples/remoteBuild.md

@@ -11,7 +11,7 @@ To try this immediately, run a build against the kustomization
 in the [multibases](multibases/README.md) example.  There's
 one pod in the output:
 
-<!-- @remoteOverlayBuild @test -->
+<!-- @remoteOverlayBuild @testAgainstLatestRelease -->
 
 ```
 target="github.com/kubernetes-sigs/kustomize//examples/multibases/dev/?ref=v1.0.6"
@@ -24,7 +24,7 @@ Run against the overlay in that example to get three pods
 (the overlay combines the dev, staging and prod bases for
 someone who wants to send them all at the same time):
 
-<!-- @remoteBuild @test -->
+<!-- @remoteBuild @testAgainstLatestRelease -->
 ```
 target="https://github.com/kubernetes-sigs/kustomize//examples/multibases?ref=v1.0.6"
 test 3 == \
@@ -34,7 +34,7 @@ test 3 == \
 
 A base can be a URL:
 
-<!-- @createOverlay @test -->
+<!-- @createOverlay @testAgainstLatestRelease -->
 ```
 DEMO_HOME=$(mktemp -d)
 
@@ -48,7 +48,7 @@ EOF
 Build this to confirm that all three pods from the base
 have the `remote-` prefix.
 
-<!-- @remoteBases @test -->
+<!-- @remoteBases @testAgainstLatestRelease -->
 ```
 test 3 == \
   $(kustomize build $DEMO_HOME | grep remote-.*-myapp-pod | wc -l); \

examples/secretGeneratorPlugin.md

@@ -25,7 +25,7 @@ etc.
 
 ## Make a place to work
 
-<!-- @establishBase @test -->
+<!-- @establishBase @testAgainstLatestRelease -->
 ```
 DEMO_HOME=$(mktemp -d)
 ```
@@ -43,7 +43,7 @@ Here's an example combining all three methods:
 
 Make an env file with some short secrets:
 
-<!-- @makeEnvFile @test -->
+<!-- @makeEnvFile @testAgainstLatestRelease -->
 ```
 cat <<'EOF' >$DEMO_HOME/foo.env
 ROUTER_PASSWORD=admin
@@ -53,7 +53,7 @@ EOF
 
 Make a text file with a long secret:
 
-<!-- @makeLongSecretFile @test -->
+<!-- @makeLongSecretFile @testAgainstLatestRelease -->
 ```
 cat <<'EOF' >$DEMO_HOME/longsecret.txt
 Lorem ipsum dolor sit amet,
@@ -67,7 +67,7 @@ And make a kustomization file referring to the
 above and _additionally_ defining some literal KV
 pairs in line:
 
-<!-- @makeKustomization1 @test -->
+<!-- @makeKustomization1 @testAgainstLatestRelease -->
 ```
 cat <<'EOF' >$DEMO_HOME/kustomization.yaml
 secretGenerator:
@@ -84,7 +84,7 @@ EOF
 
 Now generate the Secret:
 
-<!-- @build1 @test -->
+<!-- @build1 @testAgainstLatestRelease -->
 ```
 result=$(kustomize build $DEMO_HOME)
 echo "$result"
@@ -141,7 +141,7 @@ from a database.
 
 Download it
 
-<!-- @copyPlugin @test -->
+<!-- @copyPlugin @testAgainstLatestRelease -->
 ```
 repo=https://raw.githubusercontent.com/kubernetes-sigs/kustomize
 pPath=plugin/someteam.example.com/v1/secretsfromdatabase
@@ -165,7 +165,7 @@ go build -buildmode plugin \
 
 Create a configuration file for it:
 
-<!-- @makeConfiguration @test -->
+<!-- @makeConfiguration @testAgainstLatestRelease -->
 ```
 cat <<'EOF' >$DEMO_HOME/secretFromDb.yaml
 apiVersion: someteam.example.com/v1
@@ -183,7 +183,7 @@ EOF
 Create a new kustomization file
 referencing this plugin:
 
-<!-- @makeKustomization2 @test -->
+<!-- @makeKustomization2 @testAgainstLatestRelease -->
 ```
 cat <<'EOF' >$DEMO_HOME/kustomization.yaml
 generators:

examples/springboot/README.md

@@ -13,7 +13,7 @@ In the production environment we want to customize the following:
 - health check and readiness check.
 
 First make a place to work:
-<!-- @makeDemoHome @test -->
+<!-- @makeDemoHome @testAgainstLatestRelease -->
 ```
 DEMO_HOME=$(mktemp -d)
 ```
@@ -27,7 +27,7 @@ as HERE documents.
 
 Download them:
 
-<!-- @downloadResources @test -->
+<!-- @downloadResources @testAgainstLatestRelease -->
 ```
 CONTENT="https://raw.githubusercontent.com\
 /kubernetes-sigs/kustomize\
@@ -44,14 +44,14 @@ a file called `kustomization.yaml`.
 
 Start this file:
 
-<!-- @kustomizeYaml @test -->
+<!-- @kustomizeYaml @testAgainstLatestRelease -->
 ```
 touch $DEMO_HOME/kustomization.yaml
 ```
 
 ### Add the resources
 
-<!-- @addResources @test -->
+<!-- @addResources @testAgainstLatestRelease -->
 ```
 cd $DEMO_HOME
 
@@ -71,7 +71,7 @@ cat kustomization.yaml
 
 ### Add configMap generator
 
-<!-- @addConfigMap @test -->
+<!-- @addConfigMap @testAgainstLatestRelease -->
 ```
 echo "app.name=Kustomize Demo" >$DEMO_HOME/application.properties
 
@@ -102,7 +102,7 @@ For Spring Boot application, we can set an active profile through the environmen
 the application will pick up an extra `application-<profile>.properties` file. With this, we can customize the configMap in two
 steps. Add an environment variable through the patch and add a file to the configMap.
 
-<!-- @customizeConfigMap @test -->
+<!-- @customizeConfigMap @testAgainstLatestRelease -->
 ```
 cat <<EOF >$DEMO_HOME/patch.yaml
 apiVersion: apps/v1beta2
@@ -149,7 +149,7 @@ Arrange for the resources to begin with prefix
 _prod-_ (since they are meant for the _production_
 environment):
 
-<!-- @customizeLabel @test -->
+<!-- @customizeLabel @testAgainstLatestRelease -->
 ```
 cd $DEMO_HOME
 kustomize edit set nameprefix 'prod-'
@@ -165,7 +165,7 @@ This `namePrefix` directive adds _prod-_ to all
 resource names, as can be seen by building the
 resources:
 
-<!-- @build1 @test -->
+<!-- @build1 @testAgainstLatestRelease -->
 ```
 kustomize build $DEMO_HOME | grep prod-
 ```
@@ -180,7 +180,7 @@ selector.
 add a label, but one can always edit
 `kustomization.yaml` directly:
 
-<!-- @customizeLabels @test -->
+<!-- @customizeLabels @testAgainstLatestRelease -->
 ```
 cat <<EOF >>$DEMO_HOME/kustomization.yaml
 commonLabels:
@@ -191,7 +191,7 @@ EOF
 Confirm that the resources now all have names prefixed
 by `prod-` and the label tuple `env:prod`:
 
-<!-- @build2 @test -->
+<!-- @build2 @testAgainstLatestRelease -->
 ```
 kustomize build $DEMO_HOME | grep -C 3 env
 ```
@@ -205,7 +205,7 @@ set JVM options accordingly.
 
 Download the patch `memorylimit_patch.yaml`. It contains the memory limits setup.
 
-<!-- @downloadPatch @test -->
+<!-- @downloadPatch @testAgainstLatestRelease -->
 ```
 curl -s  -o "$DEMO_HOME/#1.yaml" \
   "$CONTENT/overlays/production/{memorylimit_patch}.yaml"
@@ -243,7 +243,7 @@ has end points such as `/actuator/health` for this. We can customize the k8s dep
 
 Download the patch `healthcheck_patch.yaml`. It contains the liveness probes and readyness probes.
 
-<!-- @downloadPatch @test -->
+<!-- @downloadPatch @testAgainstLatestRelease -->
 ```
 curl -s  -o "$DEMO_HOME/#1.yaml" \
   "$CONTENT/overlays/production/{healthcheck_patch}.yaml"
@@ -281,7 +281,7 @@ The output contains
 
 Add these patches to the kustomization:
 
-<!-- @addPatch @test -->
+<!-- @addPatch @testAgainstLatestRelease -->
 ```
 cd $DEMO_HOME
 kustomize edit add patch memorylimit_patch.yaml
@@ -301,7 +301,7 @@ The output of the following command can now be applied
 to the cluster (i.e. piped to `kubectl apply`) to
 create the production environment.
 
-<!-- @finalBuild @test -->
+<!-- @finalBuild @testAgainstLatestRelease -->
 ```
 kustomize build $DEMO_HOME  # | kubectl apply -f -
 ```

examples/transformerconfigs/README.md

@@ -175,4 +175,4 @@ metadata:
   annotations:
     foo.k8s.io/bar: baz
 ```
-Kustomize supports escaping special characters in path, e.g `matadata/annotations/foo.k8s.io\/bar`
+Kustomize supports escaping special characters in path, e.g `metadata/annotations/foo.k8s.io\/bar`

examples/transformerconfigs/crd/README.md

@@ -3,7 +3,7 @@
 This tutorial shows how to add transformer configurations to support a custom resource.
 
 Create a workspace by
-<!-- @createws @test -->
+<!-- @createws @testAgainstLatestRelease -->
 ```
 DEMO_HOME=$(mktemp -d)
 ```
@@ -17,7 +17,7 @@ Consider a CRD of kind `MyKind` with fields
 - `.spec.selectors` as the label selectors
 
 Add the following file to configure the transformers for the above fields
-<!-- @addConfig @test -->
+<!-- @addConfig @testAgainstLatestRelease -->
 ```
 mkdir $DEMO_HOME/kustomizeconfig
 cat > $DEMO_HOME/kustomizeconfig/mykind.yaml << EOF
@@ -51,7 +51,7 @@ EOF
 Create a file with some resources that
 includes an instance of `MyKind`:
 
-<!-- @createResource @test -->
+<!-- @createResource @testAgainstLatestRelease -->
 ```
 cat > $DEMO_HOME/resources.yaml << EOF
 apiVersion: v1
@@ -88,7 +88,7 @@ EOF
 
 Create a kustomization referring to it:
 
-<!-- @createKustomization @test -->
+<!-- @createKustomization @testAgainstLatestRelease -->
 ```
 cat > $DEMO_HOME/kustomization.yaml << EOF
 resources:
@@ -112,7 +112,7 @@ EOF
 
 Use the customized transformer configurations by specifying them
 in the kustomization file:
-<!-- @addTransformerConfigs @test -->
+<!-- @addTransformerConfigs @testAgainstLatestRelease -->
 ```
 cat >> $DEMO_HOME/kustomization.yaml << EOF
 configurations:
@@ -122,7 +122,7 @@ EOF
 
 Run `kustomize build` and verify that the namereference is correctly resolved.
 
-<!-- @build @test -->
+<!-- @build @testAgainstLatestRelease -->
 ```
 test 2 == \
 $(kustomize build $DEMO_HOME | grep -A 2 ".*Ref" | grep "test-" | wc -l); \
@@ -131,7 +131,7 @@ echo $?
 
 Run `kustomize build` and verify that the vars correctly resolved.
 
-<!-- @verify @test -->
+<!-- @verify @testAgainstLatestRelease -->
 ```
 test 0 == \
 $(kustomize build $DEMO_HOME | grep "BEE_ACTION" | wc -l); \

examples/transformerconfigs/images/README.md

@@ -3,7 +3,7 @@
 This tutorial shows how to modify images in resources, and create a custom images transformer configuration.
 
 Create a workspace by
-<!-- @createws @test -->
+<!-- @createws @testAgainstLatestRelease -->
 ```
 DEMO_HOME=$(mktemp -d)
 ```
@@ -15,7 +15,7 @@ Consider a Custom Resource Definition(CRD) of kind `MyKind` with field
 
 Add the following file to configure the images transformer for the CRD:
 
-<!-- @addConfig @test -->
+<!-- @addConfig @testAgainstLatestRelease -->
 ```
 mkdir $DEMO_HOME/kustomizeconfig
 cat > $DEMO_HOME/kustomizeconfig/mykind.yaml << EOF
@@ -30,7 +30,7 @@ EOF
 
 Create a file with some resources that includes an instance of `MyKind`:
 
-<!-- @createResource @test -->
+<!-- @createResource @testAgainstLatestRelease -->
 ```
 cat > $DEMO_HOME/resources.yaml << EOF
 
@@ -66,7 +66,7 @@ EOF
 
 Create a kustomization.yaml referring to it:
 
-<!-- @createKustomization @test -->
+<!-- @createKustomization @testAgainstLatestRelease -->
 ```
 cat > $DEMO_HOME/kustomization.yaml << EOF
 resources:
@@ -90,7 +90,7 @@ EOF
 
 Use the customized transformer configurations by specifying them
 in the kustomization file:
-<!-- @addTransformerConfigs @test -->
+<!-- @addTransformerConfigs @testAgainstLatestRelease -->
 ```
 cat >> $DEMO_HOME/kustomization.yaml << EOF
 configurations:
@@ -100,27 +100,27 @@ EOF
 
 Run `kustomize build` and verify that the images have been updated.
 
-<!-- @build @test -->
+<!-- @build @testAgainstLatestRelease -->
 ```
 test 1 == \
 $(kustomize build $DEMO_HOME | grep -A 2 ".*image" | grep "new-crd-image:new-v1-tag" | wc -l); \
 echo $?
 ```
 
-<!-- @build @test -->
+<!-- @build @testAgainstLatestRelease -->
 ```
 test 1 == \
 $(kustomize build $DEMO_HOME | grep -A 2 ".*image" | grep "new-app-1:MYNEWTAG-1" | wc -l); \
 echo $?
 ```
 
-<!-- @build @test -->
+<!-- @build @testAgainstLatestRelease -->
 ```
 test 1 == \
 $(kustomize build $DEMO_HOME | grep -A 2 ".*image" | grep "my-docker2@sha" | wc -l); \
 echo $?
 ```
-<!-- @build @test -->
+<!-- @build @testAgainstLatestRelease -->
 ```
 test 1 == \
 $(kustomize build $DEMO_HOME | grep -A 2 ".*image" | grep "prod-mysql:v3" | wc -l); \

examples/validationTransformer/README.md

@@ -24,7 +24,7 @@ loaded by Kustomize.
 
 Make a place to work:
 
-<!-- @makeWorkplace @test -->
+<!-- @makeWorkplace @testAgainstLatestRelease -->
 ```
 DEMO_HOME=$(mktemp -d)
 mkdir -p $DEMO_HOME/valid
@@ -38,7 +38,7 @@ mkdir -p $PLUGINDIR
 Download the [kubeval] binary depending on the operating system
 and add it to $PATH.
 
-<!-- @downloadKubeval @test -->
+<!-- @downloadKubeval @testAgainstLatestRelease -->
 ```
 OS=`uname | sed -e 's/Linux/linux/' -e 's/Darwin/darwin/'`
 wget https://github.com/instrumenta/kubeval/releases/download/0.9.2/kubeval-${OS}-amd64.tar.gz
@@ -60,7 +60,7 @@ A transformer plugin for the validation can be written as a
 bash script, which execute the [kubeval] binary and return proper
 output and exit code.
 
-<!-- @writePlugin @test -->
+<!-- @writePlugin @testAgainstLatestRelease -->
 ```
 cat <<'EOF' > $PLUGINDIR/Validator
 #!/bin/bash
@@ -95,7 +95,7 @@ chmod +x $PLUGINDIR/Validator
 Define a kustomization containing a valid ConfigMap
 and the transformer plugin.
 
-<!-- @writeKustomization @test -->
+<!-- @writeKustomization @testAgainstLatestRelease -->
 ```
 cat <<'EOF' >$DEMO_HOME/valid/configmap.yaml
 apiVersion: v1
@@ -125,7 +125,7 @@ EOF
 Define a kustomization containing an invalid ConfigMap
 and the transformer plugin.
 
-<!-- @writeKustomization @test -->
+<!-- @writeKustomization @testAgainstLatestRelease -->
 ```
 cat <<'EOF' >$DEMO_HOME/invalid/configmap.yaml
 apiVersion: v1
@@ -175,7 +175,7 @@ The directory structure is as the following:
 Define a helper function to run kustomize with the
 correct environment and flags for plugins:
 
-<!-- @defineKustomizeBd @test -->
+<!-- @defineKustomizeBd @testAgainstLatestRelease -->
 ```
 function kustomizeBd {
   XDG_CONFIG_HOME=$DEMO_HOME \
@@ -187,7 +187,7 @@ function kustomizeBd {
 
 Build the valid variant
 
-<!-- @buildValid @test -->
+<!-- @buildValid @testAgainstLatestRelease -->
 ```
 kustomizeBd valid
 ```
@@ -215,7 +215,7 @@ data: Invalid type. Expected: object, given: array
 
 ## cleanup
 
-<!-- @cleanup @test -->
+<!-- @cleanup @testAgainstLatestRelease -->
 ```shell
 rm -rf $DEMO_HOME
 ```

examples/wordpress/README.md

@@ -8,7 +8,7 @@ To run WordPress, it's necessary to
 - access the service name of MySQL database from WordPress container
 
 First make a place to work:
-<!-- @makeDemoHome @test -->
+<!-- @makeDemoHome @testAgainstLatestRelease -->
 ```
 DEMO_HOME=$(mktemp -d)
 MYSQL_HOME=$DEMO_HOME/mysql
@@ -21,7 +21,7 @@ mkdir -p $WORDPRESS_HOME
 
 Download the resources and `kustomization.yaml` for WordPress.
 
-<!-- @downloadResources @test -->
+<!-- @downloadResources @testAgainstLatestRelease -->
 ```
 CONTENT="https://raw.githubusercontent.com\
 /kubernetes-sigs/kustomize\
@@ -33,7 +33,7 @@ curl -s -o "$WORDPRESS_HOME/#1.yaml" \
 
 Download the resources and `kustomization.yaml` for MySQL.
 
-<!-- @downloadResources @test -->
+<!-- @downloadResources @testAgainstLatestRelease -->
 ```
 CONTENT="https://raw.githubusercontent.com\
 /kubernetes-sigs/kustomize\
@@ -48,7 +48,7 @@ curl -s -o "$MYSQL_HOME/#1.yaml" \
 Create a new kustomization with two bases,
 `wordpress` and `mysql`:
 
-<!-- @createKustomization @test -->
+<!-- @createKustomization @testAgainstLatestRelease -->
 ```
 cat <<EOF >$DEMO_HOME/kustomization.yaml
 resources:
@@ -65,7 +65,7 @@ In the new kustomization, apply a patch for wordpress deployment. The patch does
 - Add an initial container to show the mysql service name
 - Add environment variable that allow wordpress to find the mysql database
 
-<!-- @downloadPatch @test -->
+<!-- @downloadPatch @testAgainstLatestRelease -->
 ```
 CONTENT="https://raw.githubusercontent.com\
 /kubernetes-sigs/kustomize\
@@ -105,7 +105,7 @@ $(WORDPRESS_SERVICE) and $(MYSQL_SERVICE).
 
 ### Bind the Variables to k8s Object Fields
 
-<!-- @addVarRef @test -->
+<!-- @addVarRef @testAgainstLatestRelease -->
 ```
 cat <<EOF >>$DEMO_HOME/kustomization.yaml
 vars:
@@ -128,7 +128,7 @@ EOF
 ### Substitution
 Confirm the variable substitution:
 
-<!-- @kustomizeBuild @test -->
+<!-- @kustomizeBuild @testAgainstLatestRelease -->
 ```
 kustomize build $DEMO_HOME
 ```

examples/zh/README.md

@@ -14,18 +14,20 @@ go get sigs.k8s.io/kustomize/v3/cmd/kustomize
 
   * [configGenerations](configGeneration.md) - 当 ConfigMapGenerator 修改时进行滚动更新。
 
-  * [combineConfigs](../combineConfigs.md) - 融合来自不同用户的配置数据（例如来自 devops/SRE 和 developers）。
+  * [combineConfigs](combineConfigs.md) - 融合来自不同用户的配置数据（例如来自 devops/SRE 和 developers）。
 
-  * [generatorOptions](../generatorOptions.md) -修改所有 ConfigMapGenerator 和 SecretGenerator 的行为。
+  * [generatorOptions](generatorOptions.md) -修改所有 ConfigMapGenerator 和 SecretGenerator 的行为。
 
-  * [vars](../wordpress/README.md) - 通过 vars 将一个资源的数据注入另一个资源的容器参数 （例如，为 wordpress 指定 SQL 服务）。
+  * [vars](vars.md) - 通过 vars 将一个资源的数据注入另一个资源的容器参数 （例如，为 wordpress 指定 SQL 服务）。
 
-  * [image names and tags](../image.md) - 在不使用 patch 的情况下更新镜像名称和标签。
+  * [image names and tags](image.md) - 在不使用 patch 的情况下更新镜像名称和标签。
 
-  * [remote target](../remoteBuild.md) - 通过 github URL 来构建 kustomization 。
+  * [remote target](remoteBuild.md) - 通过 github URL 来构建 kustomization 。
 
   * [json patch](jsonpatch.md) -在 kustomization 中应用 json patch 。
 
+  * [patch multiple objects](patchMultipleObjects.md) - 通过一个patch来修改多个资源。
+
 高级用法
 
   - generator 插件:
@@ -34,6 +36,10 @@ go get sigs.k8s.io/kustomize/v3/cmd/kustomize
 
     * [secret generation](../secretGeneratorPlugin.md) - 生成 Secret。
 
+  - transformer 插件:
+
+   * [validation transformer](../validationTransformer/README.md) - 通过 transformer 验证资源。
+
   - 定制内建 transformer 配置
 
     * [transformer configs](../transformerconfigs/README.md) - 自定义 transformer 配置。

examples/zh/combineConfigs.md

@@ -0,0 +1,230 @@
+[overlay]: ../docs/glossary.md#overlay
+[target]: ../docs/glossary.md#target
+
+# 示例：devops和开发配合管理配置数据
+
+场景：在生产环境中有一个基于 Java 由多个内部团队（注册、结账和搜索等）共同开发的商店服务。
+
+这个服务在不同的环境中运行：_development_、 _testing_、 _staging_ 和 _production_，从 Java 的 properties 文件中读取配置。
+
+为每个环境维护一个大的 properties 文件是很困难的。这个文件需要频繁的修改，并且这些修改都需要由 devops 工程师来进行，因为：
+
+  1. 这个文件包含 devops 工程师需要知道，而开发人员不必知道的值
+  2. 比如生产环境的 properties 包含敏感数据，比如生产数据库的登录凭据。
+
+## Property sharding
+
+通过一些研究，我们注意到属性可以分为不同的类别。
+
+### Property sharding
+
+例如：国际化数据、物理常量，外部服务位置等静态数据。
+
+_这些无论哪个环境，都一样的配置。_
+
+这些都只需要一组配置。将这组配置放在一个文件中：
+
+ * `common.properties`
+
+### Plumbing properties
+
+例如：静态资源（HTML、CSS、JavaScript）的位置，产品和用户的数据表，负载均衡的端口，日志收集等。
+
+_这些属性的不同，恰恰是环境的不同之处。_
+
+DevOps 或 SRE 工程师需要完全控制生产环境中的这些配置；测试需要调整数据库来支持测试；而开发则希望尝试开发中遇到的各种不同的情景。
+
+将这些值放入
+
+ * `development/plumbing.properties`
+ * `staging/plumbing.properties`
+ * `production/plumbing.properties`
+
+
+### Secret properties
+
+例如：用户表的位置、数据库凭证、解密密钥等。
+
+_这些需要 devops 工程师控制，其他人没有访问权限。_
+
+将这些值放入
+
+ * `development/secret.properties`
+ * `staging/secret.properties`
+ * `production/secret.properties`
+
+[kubernetes secret]: https://kubernetes.io/docs/tasks/inject-data-application/distribute-credentials-secure/
+
+例如使用 unix 文件权限和模式来限制访问控制，或者使用更好的方法-使用专门用于存储密码的服务，并且使用 kustomize 中的 `secretGenerator` 字段在 Kubernetes 中创建 secret 来存储密码。
+
+<!--
+secretGenerator:
+- name: app-tls
+  files:
+    tls.crt=tls.cert
+    tls.key=tls.key
+  type: "kubernetes.io/tls"
+EOF
+-->
+
+## 混合管理方法
+
+基于相同的 base 创建 _n_ 个 overlays 来创建 _n_ 个集群环境的方法。
+
+在本例的其余部分，我们将使用 _n==2_，这里只使用 _development_ 和 _production_ ，可以使用相同的方法来增加更多的环境。
+
+运行 `kustomize build` 基于 [overlay] 的 [target] 来创建集群环境。
+
+[helloworld]: helloWorld.md
+
+以下示例将执行此操作，但将侧重于 configMap 构建，而不用担心如何将 configMaps 关联到 Deployment（[helloworld] 示例中介绍的）。
+
+所有文件（包括共享 property 文件）都将在目录树中创建，目录中包含 base 和 overlay 文件的目录，这些都与 [helloworld] 中演示的一致。
+
+它将全部存在于此工作目录中：
+
+<!-- @makeWorkplace @test -->
+```bash
+DEMO_HOME=$(mktemp -d)
+```
+
+### 创建 base
+
+<!-- kubectl create configmap BOB --dry-run -o yaml --from-file db. -->
+
+创建放置 base 配置的路径：
+
+<!-- @baseDir @test -->
+```bash
+mkdir -p $DEMO_HOME/base
+```
+
+向 base 中的插入数据，base 中应该包含所有环境共有的资源，这里我们只定义一个 java properties 文件，以及一个引用他们的 `kustomization` 文件。
+
+<!-- @baseKustomization @test -->
+```bash
+cat <<EOF >$DEMO_HOME/base/common.properties
+color=blue
+height=10m
+EOF
+
+cat <<EOF >$DEMO_HOME/base/kustomization.yaml
+configMapGenerator:
+- name: my-configmap
+  files:
+  - common.properties
+EOF
+```
+
+### 创建并使用 overlay 用于 _开发_
+
+创建一个 overlays 目录：
+
+<!-- @overlays @test -->
+```bash
+OVERLAYS=$DEMO_HOME/overlays
+```
+
+创建 _development_ overlay：
+
+<!-- @developmentFiles @test -->
+```bash
+mkdir -p $OVERLAYS/development
+
+cat <<EOF >$OVERLAYS/development/plumbing.properties
+port=30000
+EOF
+
+cat <<EOF >$OVERLAYS/development/secret.properties
+dbpassword=mothersMaidenName
+EOF
+
+cat <<EOF >$OVERLAYS/development/kustomization.yaml
+resources:
+- ../../base
+namePrefix: dev-
+nameSuffix: -v1
+configMapGenerator:
+- name: my-configmap
+  behavior: merge
+  files:
+  - plumbing.properties
+  - secret.properties
+EOF
+```
+
+现在可以生成开发使用的 configMaps ：
+
+<!-- @runDev @test -->
+```bash
+kustomize build $OVERLAYS/development
+```
+
+#### 检查 ConfigMap 名称
+
+可以在输出中看到生成的 `ConfigMap` 名称。
+
+名称应该是这样的：`dev-my-configmap-v1-2gccmccgd5`：
+
+ * `"dev-"` 来自 `namePrefix` 字段
+ * `"my-configmap"` 来自 `configMapGenerator/name` 字段
+ * `"-v1"` 来自 `nameSuffix` 字段
+ * `"-2gccmccgd5"` 为哈希值，是 `kustomize` 根据 configMap 的内容计算的
+
+哈希后缀很关键，如果 configMap 内容发生变化， configMap 的名称也会发生变化，以及从 `kustomize` 出现在 YAML 输出中的对该名称的所有引用。
+
+名称更改意味着如果使用类似命令将此 YAML 应用于群集，则 Deployment 将执行滚动更新重启以获取新数据。
+
+> ```bash
+> kustomize build $OVERLAYS/development | kubectl apply -f -
+> ```
+
+Deployment 无法自动检测 ConfigMap 是否发生改变。
+
+如果更改 configMap 的数据, 而不更改其名称以及对该名称的所有引用, 则必须重新启动Deployment中的那些Pods以获取更改。
+
+最佳的做法就是将 configMap 视为不变的。
+
+不去编辑 configMap ，而是使用 _新_ 的名称的 _新_ configMap，并在 Deployment 中引用新的 configMap 。而 `kustomize` 使用 `configMapGenerator` 指令和相关的命名控件使这很容易。
+
+### 创建并且使用 overlay 用于 _生产_
+
+接下来创建 _production_ overlay 的文件：
+
+<!-- @productionFiles @test -->
+```bash
+mkdir -p $OVERLAYS/production
+
+cat <<EOF >$OVERLAYS/production/plumbing.properties
+port=8080
+EOF
+
+cat <<EOF >$OVERLAYS/production/secret.properties
+dbpassword=thisShouldProbablyBeInASecretInstead
+EOF
+
+cat <<EOF >$OVERLAYS/production/kustomization.yaml
+resources:
+- ../../base
+namePrefix: prod-
+configMapGenerator:
+- name: my-configmap
+  behavior: merge
+  files:
+  - plumbing.properties
+  - secret.properties
+EOF
+```
+
+现在可以生成用于生产的 configMap：
+
+<!-- @runProd @test -->
+```bash
+kustomize build $OVERLAYS/production
+```
+
+可以直接在 CI/CD 流程中执行如下命令，将应用部署到集群：
+
+> ```bash
+> kustomize build $OVERLAYS/production | kubectl apply -f -
+> ```

examples/zh/configGeneration.md

@@ -30,7 +30,7 @@ kustomize 提供了两种添加 ConfigMap 的方法：
 ### 建立 base 和 staging
 
 使用 configMapGenerator 建立 base
-<!-- @establishBase @test -->
+<!-- @establishBase @testAgainstLatestRelease -->
 ```
 DEMO_HOME=$(mktemp -d)
 
@@ -57,7 +57,7 @@ EOF
 ```
 
 通过应用 ConfigMap patch 的方式建立 staging
-<!-- @establishStaging @test -->
+<!-- @establishStaging @testAgainstLatestRelease -->
 ```
 OVERLAYS=$DEMO_HOME/overlays
 mkdir -p $OVERLAYS/staging
@@ -93,7 +93,7 @@ EOF
 
 deployment 按照名称引用此 ConfigMap ：
 
-<!-- @showDeployment @test -->
+<!-- @showDeployment @testAgainstLatestRelease -->
 ```
 grep -C 2 configMapKeyRef $BASE/deployment.yaml
 ```
@@ -111,7 +111,7 @@ grep -C 2 configMapKeyRef $BASE/deployment.yaml
 
 _staging_ 的 [variant] 包含一个 configMap 的 [patch]：
 
-<!-- @showMapPatch @test -->
+<!-- @showMapPatch @testAgainstLatestRelease -->
 ```
 cat $OVERLAYS/staging/map.yaml
 ```
@@ -120,7 +120,7 @@ cat $OVERLAYS/staging/map.yaml
 
 在 ConfigMapGenerator 中声明 ConfigMap 的修改。
 
-<!-- @showMapBase @test -->
+<!-- @showMapBase @testAgainstLatestRelease -->
 ```
 grep -C 4 configMapGenerator $BASE/kustomization.yaml
 ```
@@ -129,7 +129,7 @@ grep -C 4 configMapGenerator $BASE/kustomization.yaml
 
 但是，文件中指定的名称值不是群集中使用的名称值。根据设计，kustomize 修改从 ConfigMapGenerator 声明的 ConfigMaps 的名称。要查看最终在群集中使用的名称，只需运行 kustomize：
 
-<!-- @grepStagingName @test -->
+<!-- @grepStagingName @testAgainstLatestRelease -->
 ```
 kustomize build $OVERLAYS/staging |\
     grep -B 8 -A 1 staging-the-map
@@ -141,14 +141,14 @@ kustomize build $OVERLAYS/staging |\
 
 configMap 名称的后缀是由 map 内容的哈希生成的 - 在这种情况下，名称后缀是 _k25m8k5k5m_ ：
 
-<!-- @grepStagingHash @test -->
+<!-- @grepStagingHash @testAgainstLatestRelease -->
 ```
 kustomize build $OVERLAYS/staging | grep k25m8k5k5m
 ```
 
 现在修改 map patch ，更改该服务将使用的问候消息：
 
-<!-- @changeMap @test -->
+<!-- @changeMap @testAgainstLatestRelease -->
 ```
 sed -i.bak 's/pineapple/kiwi/' $OVERLAYS/staging/map.yaml
 ```
@@ -162,7 +162,7 @@ kustomize build $OVERLAYS/staging |\
 
 再次运行 kustomize 查看新的 configMap 名称：
 
-<!-- @grepStagingName @test -->
+<!-- @grepStagingName @testAgainstLatestRelease -->
 ```
 kustomize build $OVERLAYS/staging |\
     grep -B 8 -A 1 staging-the-map
@@ -170,7 +170,7 @@ kustomize build $OVERLAYS/staging |\
 
 确认 configMap 内容的更改将会生成以 _cd7kdh48fd_ 结尾的三个新名称 - 一个在 configMap 的名称中，另两个在使用 ConfigMap 的 deployment 中：
 
-<!-- @countHashes @test -->
+<!-- @countHashes @testAgainstLatestRelease -->
 ```
 test 3 == \
   $(kustomize build $OVERLAYS/staging | grep cd7kdh48fd | wc -l); \

examples/zh/generatorOptions.md

@@ -0,0 +1,60 @@
+# Generator Options
+
+Kustomize 提供了修改 ConfigMapGenerator 和 SecretGenerator 行为的选项，这些选项包括：
+ 
+ - 不再将基于内容生成的哈希后缀添加到资源名称后
+ - 为生成的资源添加 labels
+ - 为生成的资源添加 annotations
+ 
+这个示例将展示如何运用这些选项，首先创建一个工作空间：
+```bash
+DEMO_HOME=$(mktemp -d)
+```
+
+创建 kustomization 并且为其添加一个 ConfigMapGenerator
+
+<!-- @createCMGenerator @test -->
+```bash
+cat > $DEMO_HOME/kustomization.yaml << EOF
+configMapGenerator:
+- name: my-configmap
+  literals:	
+  - foo=bar
+  - baz=qux
+EOF
+```
+
+添加如下 generatorOptions
+<!-- @addGeneratorOptions @test -->
+```bash
+cat >> $DEMO_HOME/kustomization.yaml << EOF
+generatorOptions:
+ disableNameSuffixHash: true
+ labels:
+   kustomize.generated.resource: somevalue
+ annotations:
+   annotations.only.for.generated: othervalue
+EOF
+```
+运行 `kustomize build` 并且确定生成的 ConfigMap 。
+ 
+ - 确定没有名称后缀
+    <!-- @verify @test -->
+    ```
+    test 1 == \
+    $(kustomize build $DEMO_HOME | grep "name: my-configmap$" | wc -l); \
+    echo $?
+    ```
+ - 确定 label `kustomize.generated.resource: somevalue`
+     ```
+     test 1 == \
+     $(kustomize build $DEMO_HOME | grep -A 1 "labels" | grep "kustomize.generated.resource" | wc -l); \
+     echo $?
+     ```
+ - 确定 annotation `annotations.only.for.generated: othervalue`
+      ```
+      test 1 == \
+      $(kustomize build $DEMO_HOME | grep -A 1 "annotations" | grep "annotations.only.for.generated" | wc -l); \
+      echo $?
+      ```
+      

examples/zh/helloWorld.md

@@ -21,7 +21,7 @@
 
 首先创建一个工作空间：
 
-<!-- @makeWorkplace @test -->
+<!-- @makeWorkplace @testAgainstLatestRelease -->
 ```
 DEMO_HOME=$(mktemp -d)
 ```
@@ -38,7 +38,7 @@ DEMO_HOME=$(mktemp -d)
 
 为了使本文档保持简洁，base 的资源位于补充目录中，并不在此处，请按照下面的方法下载它们：
 
-<!-- @downloadBase @test -->
+<!-- @downloadBase @testAgainstLatestRelease -->
 ```
 BASE=$DEMO_HOME/base
 mkdir -p $BASE
@@ -51,7 +51,7 @@ curl -s -o "$BASE/#1.yaml" "https://raw.githubusercontent.com\
 
 观察该目录：
 
-<!-- @runTree @test -->
+<!-- @runTree @testAgainstLatestRelease -->
 ```
 tree $DEMO_HOME
 ```
@@ -80,14 +80,14 @@ tree $DEMO_HOME
 
 `base` 目录中包含一个 [kustomization] 文件：
 
-<!-- @showKustomization @test -->
+<!-- @showKustomization @testAgainstLatestRelease -->
 ```
 more $BASE/kustomization.yaml
 ```
 
 （可选）在 base 目录上运行 `kustomize` 将定制过的 resources 打印到标准输出：
 
-<!-- @buildBase @test -->
+<!-- @buildBase @testAgainstLatestRelease -->
 ```
 kustomize build $BASE
 ```
@@ -96,14 +96,14 @@ kustomize build $BASE
 
 定制 _app label_ 并应用于所有的 resources ：
 
-<!-- @addLabel @test -->
+<!-- @addLabel @testAgainstLatestRelease -->
 ```
 sed -i.bak 's/app: hello/app: my-hello/' \
     $BASE/kustomization.yaml
 ```
 
 查看效果：
-<!-- @checkLabel @test -->
+<!-- @checkLabel @testAgainstLatestRelease -->
 ```
 kustomize build $BASE | grep -C 3 app:
 ```
@@ -116,7 +116,7 @@ kustomize build $BASE | grep -C 3 app:
  * _Production_ 包含更多的副本数。
  * 来自这些集群 [variants] 的问候消息将与来自其他集群的不同。
 
-<!-- @overlayDirectories @test -->
+<!-- @overlayDirectories @testAgainstLatestRelease -->
 ```
 OVERLAYS=$DEMO_HOME/overlays
 mkdir -p $OVERLAYS/staging
@@ -127,7 +127,7 @@ mkdir -p $OVERLAYS/production
 
 在 `staging` 目录中创建一个 kustomization 文件，用来定义一个新的名称前缀和一些不同的 labels 。
 
-<!-- @makeStagingKustomization @test -->
+<!-- @makeStagingKustomization @testAgainstLatestRelease -->
 ```
 cat <<'EOF' >$OVERLAYS/staging/kustomization.yaml
 namePrefix: staging-
@@ -149,7 +149,7 @@ EOF
 
 同时，将 _risky_ 标记设置为 true 。
 
-<!-- @stagingMap @test -->
+<!-- @stagingMap @testAgainstLatestRelease -->
 ```
 cat <<EOF >$OVERLAYS/staging/map.yaml
 apiVersion: v1
@@ -166,7 +166,7 @@ EOF
 
 在 `production` 目录中创建一个 kustomization 文件，用来定义一个新的名称前缀和 labels 。
 
-<!-- @makeProductionKustomization @test -->
+<!-- @makeProductionKustomization @testAgainstLatestRelease -->
 ```
 cat <<EOF >$OVERLAYS/production/kustomization.yaml
 namePrefix: production-
@@ -187,7 +187,7 @@ EOF
 
 因为生产环境需要处理更多的流量，新建一个 production patch 来增加副本数。
 
-<!-- @productionDeployment @test -->
+<!-- @productionDeployment @testAgainstLatestRelease -->
 ```
 cat <<EOF >$OVERLAYS/production/deployment.yaml
 apiVersion: apps/v1
@@ -210,7 +210,7 @@ EOF
 
 查看目录结构和差异：
 
-<!-- @listFiles @test -->
+<!-- @listFiles @testAgainstLatestRelease -->
 ```
 tree $DEMO_HOME
 ```
@@ -268,12 +268,12 @@ diff \
 
 输出不同 _overlys_ 的配置：
 
-<!-- @buildStaging @test -->
+<!-- @buildStaging @testAgainstLatestRelease -->
 ```
 kustomize build $OVERLAYS/staging
 ```
 
-<!-- @buildProduction @test -->
+<!-- @buildProduction @testAgainstLatestRelease -->
 ```
 kustomize build $OVERLAYS/production
 ```

examples/zh/image.md

@@ -0,0 +1,74 @@
+# 示例: 改变镜像名称和标签
+
+首先构建一个工作空间：
+
+<!-- @makeWorkplace @testAgainstLatestRelease -->
+```bash
+DEMO_HOME=$(mktemp -d)
+```
+
+创建包含pod资源的 `kustomization`
+
+<!-- @testAgainstLatestRelease to @test -->
+```bash
+cat <<EOF >$DEMO_HOME/kustomization.yaml
+resources:
+- pod.yaml
+EOF
+```
+
+创建 pod 资源pod.yaml
+
+<!-- @createDeployment @test -->
+```bash
+cat <<EOF >$DEMO_HOME/pod.yaml
+apiVersion: v1
+kind: Pod
+metadata:
+  name: myapp-pod
+  labels:
+    app: myapp
+spec:
+  containers:
+  - name: myapp-container
+    image: busybox:1.29.0
+    command: ['sh', '-c', 'echo The app is running! && sleep 3600']
+  initContainers:
+  - name: init-mydb
+    image: busybox:1.29.0
+    command: ['sh', '-c', 'until nslookup mydb; do echo waiting for mydb; sleep 2; done;']
+EOF
+```
+
+`myapp-pod` 包含一个init容器和一个普通容器，两者都使用 `busybox：1.29.0` 镜像。
+
+在 `kustomization.yaml` 中添加 `images` 字段来更改镜像 `busybox` 和标签 `1.29.0` 。
+
+- 通过 `kustomize` 添加 `images`：
+    <!-- @addImages @test -->
+    ```bash
+    cd $DEMO_HOME
+    kustomize edit set image busybox=alpine:3.6
+    ```
+
+- 将`images`字段将被添加到`kustomization.yaml`：
+    > ```yaml
+    > images:
+    > - name: busybox
+    >   newName: alpine
+    >   newTag: 3.6
+    > ```
+
+构建 `kustomization`
+<!-- @kustomizeBuild @testAgainstLatestRelease -->
+```bash
+kustomize build $DEMO_HOME
+```
+
+确认`busybox`镜像和标签是否被替换为`alpine：3.6`：
+<!-- @confirmImages @testAgainstLatestRelease -->
+```
+test 2 = \
+  $(kustomize build $DEMO_HOME | grep alpine:3.6 | wc -l); \
+  echo $?
+```

examples/zh/jsonpatch.md

@@ -1,4 +1,4 @@
-# 例子: 应用 json patch（json补丁）
+# 示例: 应用 json patch（json补丁）
 
 kustomization文件支持通过[JSON patches](https://tools.ietf.org/html/rfc6902)来修改已有的资源.
 
@@ -6,7 +6,7 @@ kustomization文件支持通过[JSON patches](https://tools.ietf.org/html/rfc690
 
 首先，创建一个包含`ingress`的`kustomization`文件.
 
-<!-- @createIngress @test -->
+<!-- @createIngress @testAgainstLatestRelease -->
 ```bash
 DEMO_HOME=$(mktemp -d)
 
@@ -16,7 +16,7 @@ resources:
 EOF
 
 cat <<EOF >$DEMO_HOME/ingress.yaml
-apiVersion: extensions/v1beta1
+apiVersion: networking.k8s.io/v1beta1
 kind: Ingress
 metadata:
   name: my-ingress
@@ -36,7 +36,7 @@ EOF
 - 把 host 从 `foo.bar.com` 改为 `foo.bar.io`
 - 把 servicePort 从 `80` 改为 `8080`
 
-<!-- @addJsonPatch @test -->
+<!-- @addJsonPatch @testAgainstLatestRelease -->
 ```bash
 cat <<EOF >$DEMO_HOME/ingress_patch.json
 [
@@ -48,7 +48,7 @@ EOF
 
 JSON patch 也可以写成 YAML 的格式.该例子顺便展示了“添加”操作：
 
-<!-- @addYamlPatch @test -->
+<!-- @addYamlPatch @testAgainstLatestRelease -->
 ```bash
 cat <<EOF >$DEMO_HOME/ingress_patch.yaml
 - op: replace
@@ -67,12 +67,12 @@ EOF
 
 在kustomization.yaml文件中增加 _patchesJson6902_ 字段，以应用该补丁
 
-<!-- @applyJsonPatch @test -->
+<!-- @applyJsonPatch @testAgainstLatestRelease -->
 ```bash
 cat <<EOF >>$DEMO_HOME/kustomization.yaml
 patchesJson6902:
 - target:
-    group: extensions
+    group: networking.k8s.io
     version: v1beta1
     kind: Ingress
     name: my-ingress
@@ -82,7 +82,7 @@ EOF
 
 运行 `kustomize build $DEMO_HOME`, 在输出那里确认 host 已经被正确更新.
 
-<!-- @confirmHost @test -->
+<!-- @confirmHost @testAgainstLatestRelease -->
 ```bash
 test 1 == \
   $(kustomize build $DEMO_HOME | grep "host: foo.bar.io" | wc -l); \
@@ -91,7 +91,7 @@ test 1 == \
 
 运行 `kustomize build $DEMO_HOME`, 在输出那里确认 servicePort 已经被正确更新.
 
-<!-- @confirmServicePort @test -->
+<!-- @confirmServicePort @testAgainstLatestRelease -->
 
 ```bash
 test 1 == \
@@ -101,12 +101,12 @@ test 1 == \
 
 如果 patch 是YAML格式的，就能正确解析:
 
-<!-- @applyYamlPatch @test -->
+<!-- @applyYamlPatch @testAgainstLatestRelease -->
 ```bash
 cat <<EOF >>$DEMO_HOME/kustomization.yaml
 patchesJson6902:
 - target:
-    group: extensions
+    group: networking.k8s.io
     version: v1beta1
     kind: Ingress
     name: my-ingress
@@ -116,9 +116,9 @@ EOF
 
 运行 `kustomize build $DEMO_HOME`, 在输出那里确认有 `/test` 这个路径.
 
-<!-- @confirmYamlPatch @test -->
+<!-- @confirmYamlPatch @testAgainstLatestRelease -->
 ```bash
 test 1 == \
   $(kustomize build $DEMO_HOME | grep "path: /test" | wc -l); \
   echo $?
-```
+```

examples/zh/patchMultipleObjects.md

@@ -0,0 +1,186 @@
+[Strategic Merge Patch]: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-api-machinery/strategic-merge-patch.md
+[JSON patches]: https://tools.ietf.org/html/rfc6902
+[label selector]: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#label-selectors
+
+
+# 示例：通过一个 patch 来修改多个资源
+
+kustomization.yaml 支持通过 [Strategic Merge Patch] 和 [JSON patch] 来自定义资源。自 3.1.0 起，一个 patch 可以修改多个资源。
+
+这可以通过指定 patch 和它所修改的 target 来完成，如下所示：
+```yaml
+patches:
+- path: <PatchFile>
+  target:
+    group: <Group>
+    version: <Version>
+    kind: <Kind>
+    name: <Name>
+    namespace: <Namespace>
+    labelSelector: <LabelSelector>
+    annotationSelector: <AnnotationSelector>
+```
+`labelSelector` 和 `annotationSelector` 都应遵循 [label selector] 中的约定。Kustomize 选择匹配`target`中所有字段的目标来应用 patch 。
+
+下面的示例展示了如何为所有部署资源注入 sidecar 容器。
+
+创建一个包含 Deployment 资源的 `kustomization` 。
+
+<!-- @createDeployment @test -->
+```bash
+DEMO_HOME=$(mktemp -d)
+
+cat <<EOF >$DEMO_HOME/kustomization.yaml
+resources:
+- deployments.yaml
+EOF
+
+cat <<EOF >$DEMO_HOME/deployments.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: deploy1
+spec:
+  template:
+    metadata:
+      labels:
+        old-label: old-value
+    spec:
+      containers:
+        - name: nginx
+          image: nginx
+          args:
+          - one
+          - two
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: deploy2
+spec:
+  template:
+    metadata:
+      labels:
+        key: value
+    spec:
+      containers:
+        - name: busybox
+          image: busybox
+EOF
+```
+
+声明 [Strategic Merge Patch] 文件以注入 sidecar 容器：
+
+<!-- @addPatch @test -->
+```bash
+cat <<EOF >$DEMO_HOME/patch.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: not-important
+spec:
+  template:
+    spec:
+      containers:
+        - name: istio-proxy
+          image: docker.io/istio/proxyv2
+          args:
+          - proxy
+          - sidecar
+EOF
+```
+
+在 kustomization.yaml 中添加 _patches_ 字段
+
+<!-- @applyPatch @testAgainstLatestRelease -->
+```bash
+cat <<EOF >>$DEMO_HOME/kustomization.yaml
+patches:
+- path: patch.yaml
+  target:
+    kind: Deployment
+EOF
+```
+
+运行 `kustomize build $DEMO_HOME`，可以在输出中确认两个 Deployment 资源都已正确应用。
+
+<!-- @confirmPatch @test -->
+```bash
+test 2 == \
+  $(kustomize build $DEMO_HOME | grep "image: docker.io/istio/proxyv2" | wc -l); \
+  echo $?
+```
+
+输出如下：
+
+```yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: deploy1
+spec:
+  template:
+    metadata:
+      labels:
+        old-label: old-value
+    spec:
+      containers:
+      - args:
+        - proxy
+        - sidecar
+        image: docker.io/istio/proxyv2
+        name: istio-proxy
+      - args:
+        - one
+        - two
+        image: nginx
+        name: nginx
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: deploy2
+spec:
+  template:
+    metadata:
+      labels:
+        key: value
+    spec:
+      containers:
+      - args:
+        - proxy
+        - sidecar
+        image: docker.io/istio/proxyv2
+        name: istio-proxy
+      - image: busybox
+        name: busybox
+```
+
+## Target 选择
+
+- 选择名称与 `name*` 匹配的资源
+  ```yaml
+  target:
+    name: name*
+  ```
+- 选择所有 Deployment 资源
+  ```yaml
+  target:
+    kind: Deployment
+  ```
+- 选择 label 与 `app=hello` 匹配的资源
+  ```yaml
+  target:
+    labelSelector: app=hello
+  ```
+- 选择 annotation 与 `app=hello` 匹配的资源
+  ```yaml
+  target:
+    annotationSelector: app=hello
+  ```
+- 选择所有 label 与 `app=hello` 匹配的 Deployment 资源
+  ```yaml
+  target:
+    kind: Deployment
+    labelSelector: app=hello
+  ```

examples/zh/remoteBuild.md

@@ -0,0 +1,68 @@
+# remote targets
+
+`kustomize build` 可以将 URL 作为参数传入并运行.
+
+运行效果与如下操作相同:
+
+如果想要要立即尝试此操作，可以按照 [multibases](../multibases/README.md) 示例运行 kustomization 运行构建。然后查看输出中的pod：
+
+<!-- @remoteOverlayBuild @test -->
+
+```bash
+target="github.com/kubernetes-sigs/kustomize//examples/multibases/dev/?ref=v1.0.6"
+test 1 == \
+  $(kustomize build $target | grep dev-myapp-pod | wc -l); \
+  echo $?
+```
+
+在该示例中运行 overlay 将获得三个 pod（在此 overlay 结合了dev、staging 和 prod 的 bases，以便同时将它们全部发送给所有人）：
+
+<!-- @remoteBuild @test -->
+```bash
+target="https://github.com/kubernetes-sigs/kustomize//examples/multibases?ref=v1.0.6"
+test 3 == \
+  $(kustomize build $target | grep cluster-a-.*-myapp-pod | wc -l); \
+  echo $?
+```
+
+将 URL 作为 base ：
+
+<!-- @createOverlay @test -->
+```bash
+DEMO_HOME=$(mktemp -d)
+
+cat <<EOF >$DEMO_HOME/kustomization.yaml
+resources:
+- github.com/kubernetes-sigs/kustomize//examples/multibases?ref=v1.0.6
+namePrefix: remote-
+EOF
+```
+
+构建该 base 以确定所有的三个 pod 都有 `remote-` 前缀。
+
+<!-- @remoteBases @testAgainstLatestRelease -->
+```bash
+test 3 == \
+  $(kustomize build $DEMO_HOME | grep remote-.*-myapp-pod | wc -l); \
+  echo $?
+```
+
+## URL format
+
+URL 需要遵循 [hashicorp/go-getter URL 格式](https://github.com/hashicorp/go-getter#url-format) 。下面是一些遵循此约定的 Github repos 示例url。
+
+- kustomization.yaml 在根目录
+
+  `github.com/Liujingfang1/mysql`
+- kustomization.yaml 在 test 分支的根目录
+
+  `github.com/Liujingfang1/mysql?ref=test`
+- kustomization.yaml 在 v1.0.6 版本的子目录
+
+  `github.com/kubernetes-sigs/kustomize//examples/multibases?ref=v1.0.6`
+- kustomization.yaml repoUrl2 分支的子目录
+
+  `github.com/Liujingfang1/kustomize//examples/helloWorld?ref=repoUrl2`
+- kustomization.yaml commit `7050a45134e9848fca214ad7e7007e96e5042c03` 的子目录
+
+  `github.com/Liujingfang1/kustomize//examples/helloWorld?ref=7050a45134e9848fca214ad7e7007e96e5042c03`

examples/zh/vars.md

@@ -0,0 +1,148 @@
+# 示例: 将 k8s runtime 数据注入容器
+
+本教程将会介绍如何声明变量以及如何在容器中的命令使用变量。要注意的是，变量的查找和替换并不适用于任意字段，默认仅适用于容器的env，args和command。
+
+运行WordPress，以下是必须的：
+
+- WordPress 连接 MySQL 数据库
+- MySQL 服务可以被 WordPress 容器访问
+
+首先构建一个工作空间：
+<!-- @makeDemoHome @test -->
+```bash
+DEMO_HOME=$(mktemp -d)
+MYSQL_HOME=$DEMO_HOME/mysql
+mkdir -p $MYSQL_HOME
+WORDPRESS_HOME=$DEMO_HOME/wordpress
+mkdir -p $WORDPRESS_HOME
+```
+
+### 下载 resources
+
+下载 WordPress 的 resources 和 `kustomization.yaml` 。
+
+<!-- @downloadResources @test -->
+```bash
+CONTENT="https://raw.githubusercontent.com\
+/kubernetes-sigs/kustomize\
+/master/examples/wordpress/wordpress"
+
+curl -s -o "$WORDPRESS_HOME/#1.yaml" \
+  "$CONTENT/{deployment,service,kustomization}.yaml"
+```
+
+下载 MySQL 的 resources 和 `kustomization.yaml` 。
+
+<!-- @downloadResources @test -->
+```bash
+CONTENT="https://raw.githubusercontent.com\
+/kubernetes-sigs/kustomize\
+/master/examples/wordpress/mysql"
+
+curl -s -o "$MYSQL_HOME/#1.yaml" \
+  "$CONTENT/{deployment,service,secret,kustomization}.yaml"
+```
+
+### 创建 kustomization.yaml
+
+基于 `wordpress` 和 `mysql` 的两个 bases 创建一个新的 `kustomization.yaml` ：
+
+<!-- @createKustomization @test -->
+```bash
+cat <<EOF >$DEMO_HOME/kustomization.yaml
+resources:
+- wordpress
+- mysql
+namePrefix: demo-
+patchesStrategicMerge:
+- patch.yaml
+EOF
+```
+
+### 下载 WordPress 的 patchs
+
+在新的 kustomization 中应用 WordPress Deployment 的 patch ，该 patch 包含：
+- 添加初始容器来显示mysql的服务名称
+- 添加允许 wordpress 查找到 mysql 数据库的环境变量
+
+<!-- @downloadPatch @test -->
+```bash
+CONTENT="https://raw.githubusercontent.com\
+/kubernetes-sigs/kustomize\
+/master/examples/wordpress"
+
+curl -s -o "$DEMO_HOME/#1.yaml" \
+  "$CONTENT/{patch}.yaml"
+```
+该 patch 内容如下：
+> ```yaml
+> apiVersion: apps/v1beta2
+> kind: Deployment
+> metadata:
+>   name: wordpress
+> spec:
+>   template:
+>     spec:
+>       initContainers:
+>       - name: init-command
+>         image: debian
+>         command:
+>         - "echo $(WORDPRESS_SERVICE)"
+>         - "echo $(MYSQL_SERVICE)"
+>       containers:
+>       - name: wordpress
+>         env:
+>         - name: WORDPRESS_DB_HOST
+>           value: $(MYSQL_SERVICE)
+>         - name: WORDPRESS_DB_PASSWORD
+>           valueFrom:
+>             secretKeyRef:
+>               name: mysql-pass
+>               key: password
+> ```
+初始化容器的命令需要依赖于k8s资源对象字段的信息，由占位符变量 $(WORDPRESS_SERVICE) 和 $(MYSQL_SERVICE) 表示。
+
+### 将变量绑定到k8s对象字段
+
+<!-- @addVarRef @test -->
+```bash
+cat <<EOF >>$DEMO_HOME/kustomization.yaml
+vars:
+  - name: WORDPRESS_SERVICE
+    objref:
+      kind: Service
+      name: wordpress
+      apiVersion: v1
+    fieldref:
+      fieldpath: metadata.name
+  - name: MYSQL_SERVICE
+    objref:
+      kind: Service
+      name: mysql
+      apiVersion: v1
+EOF
+```
+`WORDPRESS_SERVICE` 来自 `wordpress` 服务的 `metadata.name` 字段。如果不指定 `fieldref` ，则使用默认的 `metadata.name` 。因此 `MYSQL_SERVICE` 来自 `mysql` 服务的 `metadata.name` 字段。
+
+### 替换
+
+运行命令查看替换结果：
+
+<!-- @kustomizeBuild @test -->
+```bash
+kustomize build $DEMO_HOME
+```
+
+预期的输出为：
+
+> ```yaml
+> (truncated)
+> ...
+>      initContainers:
+>      - command:
+>        - echo demo-wordpress
+>        - echo demo-mysql
+>        image: debian
+>        name: init-command
+>
+> ```

go.mod

@@ -6,10 +6,16 @@ require (
 	github.com/emicklei/go-restful v2.9.6+incompatible // indirect
 	github.com/evanphx/json-patch v4.5.0+incompatible
 	github.com/go-openapi/spec v0.19.2
+	github.com/gogo/protobuf v1.2.1 // indirect
 	github.com/golang/protobuf v1.3.1 // indirect
+	github.com/google/gofuzz v1.0.0 // indirect
 	github.com/googleapis/gnostic v0.3.0 // indirect
 	github.com/inconshreveable/mousetrap v1.0.0 // indirect
+	github.com/json-iterator/go v1.1.6 // indirect
 	github.com/mailru/easyjson v0.0.0-20190620125010-da37f6c1e481 // indirect
+	github.com/modern-go/reflect2 v1.0.1 // indirect
+	github.com/onsi/ginkgo v1.8.0 // indirect
+	github.com/onsi/gomega v1.5.0 // indirect
 	github.com/pkg/errors v0.8.1
 	github.com/spf13/cobra v0.0.2
 	github.com/spf13/pflag v1.0.3
@@ -17,10 +23,10 @@ require (
 	golang.org/x/sys v0.0.0-20190621203818-d432491b9138 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/yaml.v2 v2.2.2
-	k8s.io/api v0.0.0-20190809220925-3ab596449d6f
-	k8s.io/apimachinery v0.0.0-20190809020650-423f5d784010
-	k8s.io/client-go v0.0.0-20190812221009-4f902818859a
+	k8s.io/api v0.0.0-20190313235455-40a48860b5ab
+	k8s.io/apimachinery v0.0.0-20190313205120-d7deff9243b1
+	k8s.io/client-go v11.0.0+incompatible
 	k8s.io/klog v0.3.3 // indirect
-	k8s.io/kube-openapi v0.0.0-20190709113604-33be087ad058
+	k8s.io/kube-openapi v0.0.0-20190603182131-db7b694dc208
 	sigs.k8s.io/yaml v1.1.0
 )

go.sum

@@ -1,5 +1,3 @@
-cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
-github.com/Azure/go-autorest v11.1.2+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24=
 github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ=
 github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
 github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI=
@@ -12,13 +10,9 @@ github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/dgrijalva/jwt-go v0.0.0-20160705203006-01aeca54ebda/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
-github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM=
-github.com/elazarl/goproxy v0.0.0-20170405201442-c4fc26588b6e/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
 github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
 github.com/emicklei/go-restful v2.9.6+incompatible h1:tfrHha8zJ01ywiOEC1miGY8st1/igzWB8OmvPgoYX7w=
 github.com/emicklei/go-restful v2.9.6+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
-github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
 github.com/evanphx/json-patch v4.5.0+incompatible h1:ouOWdg56aJriqS0huScTkVXPC5IcNrDCXZ6OoTAWu7M=
 github.com/evanphx/json-patch v4.5.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
 github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
@@ -38,40 +32,26 @@ github.com/go-openapi/swag v0.19.2 h1:jvO6bCMBEilGwMfHhrd61zIID4oIFdwb76V17SM88d
 github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
 github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE=
 github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
-github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d h1:3PaI8p3seN09VjbTYC/QWlUZdZ1qS1zGjy7LH2Wt07I=
-github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
-github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/protobuf v0.0.0-20161109072736-4bd1920723d7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg=
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/google/btree v0.0.0-20160524151835-7d79101e329e/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
-github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/gofuzz v0.0.0-20161122191042-44d81051d367/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI=
 github.com/google/gofuzz v1.0.0 h1:A8PeW59pxE9IoFRqBp37U+mSNaQoZ46F1f0f863XSXw=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
-github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/googleapis/gnostic v0.0.0-20170426233943-68f4ded48ba9/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY=
-github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY=
 github.com/googleapis/gnostic v0.3.0 h1:CcQijm0XKekKjP/YCz28LXVSpgguuB+nCxaSjCe09y0=
 github.com/googleapis/gnostic v0.3.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY=
-github.com/gophercloud/gophercloud v0.1.0/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEoIEcSTewFxm1c5g8=
-github.com/gregjones/httpcache v0.0.0-20170728041850-787624de3eb7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
-github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
-github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
 github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/json-iterator/go v0.0.0-20180612202835-f2b4162afba3 h1:/UewZcckqhvnnS0C6r3Sher2hSEbVmM6Ogpcjen08+Y=
 github.com/json-iterator/go v0.0.0-20180612202835-f2b4162afba3/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
 github.com/json-iterator/go v1.1.6 h1:MrUvLMLTMxbqFJ9kzlvat/rYZqZnW3u4wkLzWTaFwKs=
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
-github.com/json-iterator/go v1.1.7 h1:KfgG9LzI+pYjr4xvmz/5H4FXjokeP+rlHLhv3iH62Fo=
-github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
-github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
@@ -83,16 +63,13 @@ github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN
 github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mailru/easyjson v0.0.0-20190620125010-da37f6c1e481 h1:IaSjLMT6WvkoZZjspGxy3rdaTEmWLoRm49WbtVUi9sA=
 github.com/mailru/easyjson v0.0.0-20190620125010-da37f6c1e481/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
-github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/reflect2 v0.0.0-20180320133207-05fbef0ca5da h1:ZQGIPjr1iTtUPXZFk8WShqb5G+Qg65VHFLtSvmHh+Mw=
 github.com/modern-go/reflect2 v0.0.0-20180320133207-05fbef0ca5da/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
-github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI=
 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
-github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw=
 github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.8.0 h1:VkHVNpR4iVnU8XQR6DBm8BqYjN7CRzw+xKUbVVbbW9w=
@@ -100,13 +77,11 @@ github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+W
 github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
 github.com/onsi/gomega v1.5.0 h1:izbySO9zDPmjJ8rDjLvkA2zJHIo+HkYXHnf7eN7SSyo=
 github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
-github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
 github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
 github.com/spf13/cobra v0.0.2 h1:NfkwRbgViGoyjBKsLI0QMDcuMnhM+SBg3T0cGfpvKDE=
 github.com/spf13/cobra v0.0.2/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
 github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
@@ -117,27 +92,21 @@ github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoH
 github.com/stretchr/testify v0.0.0-20151208002404-e3a8ff8ce365/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
-golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e h1:o3PsSEY8E4eXWkXrIP9YJALUkVZqzHJT5DOasTyn8Vs=
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190209173611-3b5209105503/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a h1:1BGLXjeY4akVXGgbC9HugT3Jv3hCI0z56oJR5vAMgBU=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -149,22 +118,17 @@ golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
-golang.org/x/time v0.0.0-20161028155119-f51c12702a4d/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20181011042414-1f849cf54d09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59 h1:QjA/9ArTfVTLfEhClDCG7SGrZkZixxWpwNCDiwJfh88=
 golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
-google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
-google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
-gopkg.in/inf.v0 v0.9.0/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
 gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
 gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
@@ -175,27 +139,16 @@ gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 k8s.io/api v0.0.0-20190313235455-40a48860b5ab h1:DG9A67baNpoeweOy2spF1OWHhnVY5KR7/Ek/+U1lVZc=
 k8s.io/api v0.0.0-20190313235455-40a48860b5ab/go.mod h1:iuAfoD4hCxJ8Onx9kaTIt30j7jUFS00AXQi6QMi99vA=
-k8s.io/api v0.0.0-20190809220925-3ab596449d6f h1:Ica1a7bw1U7AKaTDYgumijQa95BxzapaG9F7g8h2bjs=
-k8s.io/api v0.0.0-20190809220925-3ab596449d6f/go.mod h1:3Iy+myeAORNCLgjd/Xu9ebwN7Vh59Bw0vh9jhoX+V58=
 k8s.io/apimachinery v0.0.0-20190313205120-d7deff9243b1 h1:IS7K02iBkQXpCeieSiyJjGoLSdVOv2DbPaWHJ+ZtgKg=
 k8s.io/apimachinery v0.0.0-20190313205120-d7deff9243b1/go.mod h1:ccL7Eh7zubPUSh9A3USN90/OzHNSVN6zxzde07TDCL0=
-k8s.io/apimachinery v0.0.0-20190809020650-423f5d784010 h1:pyoq062NftC1y/OcnbSvgolyZDJ8y4fmUPWMkdA6gfU=
-k8s.io/apimachinery v0.0.0-20190809020650-423f5d784010/go.mod h1:Waf/xTS2FGRrgXCkO5FP3XxTOWh0qLf2QhL1qFZZ/R8=
-k8s.io/client-go v0.0.0-20190812221009-4f902818859a h1:Q6MWhZhFJehn507QasOhePysGew/kq+cCeei+M9L1t8=
-k8s.io/client-go v0.0.0-20190812221009-4f902818859a/go.mod h1:2rTYlEIRnJeJ+Y2va8HpkQBKyWjf+Uo3A8KVGcf+eMA=
 k8s.io/client-go v11.0.0+incompatible h1:LBbX2+lOwY9flffWlJM7f1Ct8V2SRNiMRDFeiwnJo9o=
 k8s.io/client-go v11.0.0+incompatible/go.mod h1:7vJpHMYJwNQCWgzmNV+VYUl1zCObLyodBc8nIyt8L5s=
 k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
 k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk=
-k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk=
-k8s.io/klog v0.3.1/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk=
 k8s.io/klog v0.3.3 h1:niceAagH1tzskmaie/icWd7ci1wbG7Bf2c6YGcQv+3c=
 k8s.io/klog v0.3.3/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk=
 k8s.io/kube-openapi v0.0.0-20190603182131-db7b694dc208 h1:5sW+fEHvlJI3Ngolx30CmubFulwH28DhKjGf70Xmtco=
 k8s.io/kube-openapi v0.0.0-20190603182131-db7b694dc208/go.mod h1:nfDlWeOsu3pUf4yWGL+ERqohP4YsZcBJXWMK+gkzOA4=
-k8s.io/kube-openapi v0.0.0-20190709113604-33be087ad058 h1:di3XCwddOR9cWBNpfgXaskhh6cgJuwcK54rvtwUaC10=
-k8s.io/kube-openapi v0.0.0-20190709113604-33be087ad058/go.mod h1:nfDlWeOsu3pUf4yWGL+ERqohP4YsZcBJXWMK+gkzOA4=
-k8s.io/utils v0.0.0-20190801114015-581e00157fb1/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew=
 sigs.k8s.io/structured-merge-diff v0.0.0-20190525122527-15d366b2352e/go.mod h1:wWxsB5ozmmv/SG7nM11ayaAW51xMvak/t1r0CSlcokI=
 sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs=
 sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=

internal/tools/README.md

@@ -0,0 +1,428 @@
+## What is this?
+### In short
+Be the GoDoc.org of k8s configuration files.
+
+### More explicitly
+Support k8s document indexing from open-source configurations in order to make
+it easy for people to learn to use a new feature, explore k8s configs in a
+central hub, and see some metrics about kustomize use.
+
+We want people to be able to support three main classes of queries:
+
+1. Structured document queries: how should I use the following fields
+   - Grace periods: `spec:template:spec:terminationGracePeriod`?
+   - Kustomize inline patch: `patches:patch`?
+
+2. Key value queries: how should I use this more specific use case of a
+   structure configuration.
+   - HorizontalPodAutoScalers: `kind=HorizontalPodAutoScaler`?
+   - Patches on StatefulSets: `patches:target:kind=StatefulSet`?
+
+3. Full text search: search the comments and the document text from any
+   type of k8s config file.
+
+## Road map
+There is a lot that can be added in order to improve the state of this
+application. Some more details along with general thoughts and comments can be
+found in the Roadmap.md file in this directory. This README contains only
+what can be considered as mostly complete and iterable parts of this project.
+
+## Running this project
+Everything is configured using kubernetes, so it should be easy for people to
+spin this up on any k8s cluster. Everything should just work (TM).
+
+The config files live in the `config` directory.
+
+```
+config
+├── base
+│   └── kustomization.yaml
+├── crawler
+│   ├── base
+│   │   ├── github_api_secret.txt
+│   │   └── kustomization.yaml
+│   ├── cronjob
+│   │   ├── cronjob.yaml
+│   │   └── kustomization.yaml
+│   └── job
+│       ├── job.yaml
+│       └── kustomization.yaml
+├── elastic
+│   └── ...
+├── redis
+│   ├── document_keystore
+│   │   ├── kustomization.yaml
+│   │   ├── redis.yaml
+│   │   └── service.yaml
+│   └── http_cache
+│       ├── kustomization.yaml
+│       ├── redis.yaml
+│       └── service.yaml
+├── webapp
+│   ├── backend
+│   │   ├── deployment.yaml
+│   │   ├── kustomization.yaml
+│   │   └── service.yaml
+│   └── frontend
+│       ├── deployment.yaml
+│       ├── kustomization.yaml
+│       └── service.yaml
+└── schema_files
+    └── kustomization_index
+        ├── es_index_mappings.json
+        └── es_index_settings.json
+```
+
+To get everything up and running you have to:
+
+1. Get some instance of elasticsearch working... and configure the
+   configmapGenerator in `config/base` to point to the right endpoint(s). The
+   configurations that need this value to be populated are the following:
+    - `config/crawler/cronjob` to run periodic crawls.
+    - `config/crawler/job` to run crawls on demand.
+    - `config/webapp/backend` to run the search server.
+
+2. Configure the elasticsearch indices:
+```
+kustomize build config/schema_files/kustomization_index | kubectl apply -f -
+```
+This will run a `curl` command that reads json data from a ConfigMap. This will
+setup the schema.  If you want to make more complex modifications to the
+schema, you should refer to the elastic docs to figure out whether the mapping
+can be added to the current index, or whether you will need to copy the
+existing index into a different one with the appropriate mappings. Modifications
+can be made by using the elasticsearch go library and writing a simple program,
+or it can be made with any http command to the appropriate server endpoint from
+within the cluster. Unfortunately I did not have the time to write a few helper
+tools for this. Feel free to contact me if you need help with modifying
+elasticsearch configs, I'm by no means an expert, but I can try to help.
+
+3. (Optional) run the redis http chache for the crawler:
+```
+kubectl apply -k config/redis/http_cache
+```
+  This will create a deployment for the cache, and a service. The crawler should
+  be configured to connect to the `http_cache` if it exists, but you can always
+  check the logs to make sure it connects, and that the identifiers match in the
+  crawler configuration and for the service endpoint.
+
+  The please be aware that the cache does not have a persistent volume.
+
+4. Configure the main redis instance:
+```
+kubectl apply -k config/redis/document_keystore
+```
+  This will create a StatefulSet with a volume of 4GiB for a redis instance.
+
+5. Get an access token from GitHub.
+
+To be able to kindly ask GitHub for it's data on k8s config files, you'll need
+to create an access\_token. From my understanding, this is the only way to do
+these code search queries (without first specifying a repository).
+
+To generate a token, go to your GitHub's account in Settings > Developer
+Settings > Personal access tokens. It should look like this.
+
+![GitHub Token 1](
+https://sigs.k8s.io/kustomize/internal/tools/pictures/github_token.png)
+
+From here you want to generate a new token and  have the following
+configuration:
+
+![GitHub Token 1](
+https://sigs.k8s.io/kustomize/internal/tools/pictures/token_config.png)
+
+If you have uses for any other data from this token, (org data, or something
+else) you can pick and choose, but be careful since it can grant this
+application access to your notifications, etc. However, any such extension
+is explicitly a non-goal and would not be maintained by this project.
+
+6. Launch the crawler:
+```
+kustomize build config/crawler/cronjob | kubectl apply -f -
+```
+This will periodically run the crawler every day according to the cron timing
+rules in the cronjob.yaml file.
+
+Instead, to get the crawler running now, you can run:
+```
+kustomize build config/crawler/cronjob | kubectl apply -f -
+```
+which will launch a non-periodic version of the crawler. It will take a few
+minutes for the crawler to split the search, but then config files should
+start to get populated within 20 minutes. It may take a while to do the
+first crawl, since it has to fetch rate-limited endpoints for each new file it
+finds. It should get significantly faster to update in the future.
+
+5. Launch the search backend
+```
+kustomize build config/webapp/backend | kubectl apply -f -
+```
+
+6. Launch the search frontend
+```
+kustomize build config/webapp/frontend | kubectl apply -f -
+```
+
+## Notes about the components
+
+### Elasticsearch
+I will add a basic working setup soon. I just did the lazy thing and used an
+already packaged solution. Most clouds will provide their own elastic
+environments, however, Elasticsearch is also working on their own
+implementation of a
+![k8s operator](https://www.elastic.co/elasticsearch-kubernetes), which might
+be worth checking out. Please note that it comes with its own license
+agreement.
+
+### Redis
+There are two Redis instances that are used in this application.
+
+One of them is configured to have on disk persistence, so make sure to have
+that set up in your kubernetes cluster. Also note that it is running on a
+single master node (i.e. it does not automatically shard keys to multiple head
+nodes as part of a highly available cluster). Since it's storing a sparse
+graph, I can't imagine this being much of an issue, but it's probably worth
+mentioning.
+
+The other Redis instance is running as a HTTP (RFC 7234) cache for etags from
+GitHub (or any other document store from which we could crawl/index). This one
+does not require full persistent storage on disk. The caching strategy is an
+LRU cache which is probably a good starting point. It might be worth it to
+investigate other cache policies, but I think LRU will work well since
+documents may or may not expire anyway, and the amount of memory allocated for
+keys is fairly large, so eviction of frequently used documents seems unlikely
+anyway.
+
+### Nginx + Angular
+There is a Dockerfile included for generating the container image with Nginx
+(using the default package) and adding all of the supporting compiled angular
+files. Any modifications to the code-base should be compatible with this setup,
+so all that's needed is to rebuild the container image, and possibly modify
+the image tags in the k8s file.
+
+### Supporting Go binaries
+There are a few go binaries that each have their own Dockerfile to build
+containers in which to run them on k8s, namely the crawler and the search
+service. Their configurations are not optimal (read: needs to be cleaned up),
+but they are functional.
+
+## Technical details
+
+### Overall design and imlpementation
+
+There are a few components that are all running together in order to get
+the overall application to work smoothly. This section will provide a brief
+overview of each component with the following sections going into more details.
+
+The overall structure is outlined in the following figure:
+![overview](
+https://sigs.k8s.io/kustomize/internal/tools/pictures/sys_arch.png)
+
+#### Crawler
+The leftmost component consists of a crawler with an http cache of GitHub
+queries does two things, it first looks at the list of documents in
+elasticsearch and tries to update them. In doing so, it maintains a set of
+newly updated files to exclude them from other parts of the crawl.
+
+To find newly added documents, the crawler crawls any new dependencies
+introduced in the document updating step and it also queries GitHub for the
+most recently indexed kustomization.\* files. Each new file will be processed
+for efficient text queries and put into the document index. Any new dependency
+will also incur more crawl operations. Finally, a graphical
+representation of the documents and their dependencies is built in Redis to be
+used for graph algorithms such as PageRank and component analysis.
+
+#### Data library
+There are a few helper libaries for dealing with Elasticsearch, Redis and
+documents. This is not persistent, nor is it centralized. They act as small
+components that help to package common pieces of code. Eventually it may make
+sense to merge all of it together and make a proper persistent model around
+this while providing an external API for document insertion/deletion. But
+that is definitely out of scope in terms of getting this to run. However
+there are limitations with the current model in terms of minimizing the
+API surface for the different components of the application. For now this
+problem is mostly mitigated by having the query server only connected to
+a data node of the Elasticsearch cluster, but the problem of knowing what
+is accessible and what isn't is left to the programmer instead of being
+clearly and explicitly supported by the API.
+
+#### Server
+Uses the data library to communicate with the data store and answer queries.
+Processes the user entered text queries into somewhat optimized elasticsearch
+queries. Provides a few endpoints to get different metrics and to eventually
+allow for registration of remote repositories.
+
+This application has an exposing service in order to allow users of the
+application access to queries and the results.
+
+#### Nginx + Angular
+Communicates directly with the backend server to forward user queries and
+their results. Presents the results on an interface. It's still pretty simple
+looking but it seems usable (to me).
+
+
+### Crawling GitHub
+With the use of API keys, GitHub allows account owners to search for files
+using their API.
+
+The search endpoints allow for the use of metadata search
+that is fairly useful/powerful. For instance they provide a `filename:` keyword
+that permits us to look for `kustomization.yaml`, `kustomization.yml`, etc.
+This enables the fetching of a list of kustomization documents, from which
+we can get the actual content from another endpoint
+(raw.githubusercontent.com).
+
+However, the search API is fairly limited. There is a restriction to the number
+of documents that can be retrieved from this method. One possible way to
+mitigate this would be to periodically query GitHub for results, sorted by the
+last indexed time. This would allow you to collect most documents from this
+point forwards. The downside to this is that it may require a large number of
+requests to their API since you cannot know when new files will be added.
+Furthermore, there is a possibility that you would not be able to get all of
+files either, depending on the velocity of growth.
+
+The approach that was taken to mitigate this is to use the `filesize:` keyword
+and to shard the search space into contiguous buckets of appropriate size in
+order to get all of the documents. This is fairly efficient, since you can find
+a good enough way to shard the documents in
+`lg(max file size) * number of documents / 1000` API queries. Moreover, since
+queries are paginated with at most 100 results per query, this solution is
+competitive with getting the optimal (non-contiguous) sharding of result sets.
+Furthermore, filesize queries can be cached to minimize the total number of
+queries called to the API in order to shard the search space. This is done by
+querying for file size intervals that always start with 0..X and binary
+searching over the `filesize:` space. This will allow you to reuse a lot of
+queries when you're looking for the next range, since it is upper bounded and
+lower bounded to a smaller number of queries within a range that has also been
+queried. I think this is only true because filesizes are power law distributed,
+so searches will typically require less queries as they progress from left to
+right.
+
+However, this method in no way depends on intervals of the form 0..X, as
+the number of documents in the many intervals of the range search could be
+added together to also make this work. This approach just seemed simpler to
+implement, maintain, and debug so it was preferred.
+
+To get an idea of how efficient this method is, to shard the search space of
+7000 documents, it will only take ~90 API range queries which should only take
+a few minutes. While actually fetching the documents and their relevant
+metadata (creation time, etc.) will take several hours. Furthermore, this
+could be made more efficient if a prior distribution is approximated.
+This prior could be scaled to the number of documents that need to be fetched,
+and then finding a shard that has an adequate number of requests, will only
+take a few queries per shard. It could probably be supported in a constant
+number of size queries if the size of each shard is halved which shouldn't
+have terrible performance impact for the retrieval. However, there where
+more pressing things to implement. I might revisit this later.
+
+### Document Indexing and Processing
+In order to support simple text queries the structured documents must be
+processed in some way that makes searching them easy. The current method
+is to recursively traverse the map of configurations to generate each sub-path
+and each key-value pair for the leaf nodes of the recursion tree.
+
+However, note that this means that a document has to be valid yaml/json
+format in order for indexing to happen. The rest of the document is treated
+as mostly text and uses default text settings from Elasticsearch.
+
+What this means is that for the following yaml document:
+
+```yaml
+resources:
+- service.yaml
+- deployment.yaml
+
+configmapGenerator:
+- name: app-configuration
+  files:
+  - config.yaml
+
+patchesJson6902:
+- target:
+    version: v1
+    kind: StatefulSet
+    name: ss-name
+  path: ss-patch.yaml
+- target:
+    version: v1
+    kind: Deployment
+    name: dep-name
+  path: dep-patch.yaml
+```
+
+the following flattened structure would look like:
+```json
+{
+  "identifiers": [
+    "resources",
+    "configmapGenerator",
+    "configmapGenerator:name",
+    "configmapGenerator:files",
+    "patchesJson6902",
+    "patchesJson6902:target",
+    "patchesJson6902:target:version",
+    "patchesJson6902:target:kind",
+    "patchesJson6902:target:name",
+    "patchesJson6902:path",
+  ],
+  "values": [
+    "resources=service.yaml"
+    "resources=deployment.yaml"
+    "configmapGenerator:name=app-configuration"
+    "configmapGenerator:files=config.yaml"
+    "patchesJson6902:target:version=v1",
+    "patchesJson6902:target:kind=StatefulSet",
+    "patchesJson6902:target:name=ss-name",
+    "patchesJson6902:path=ss-patch.yaml",
+    "patchesJson6902:target:kind=Deployment",
+    "patchesJson6902:target:name=dep-name",
+    "patchesJson6902:path=dep-patch.yaml",
+  ],
+  ...
+}
+```
+
+Note that unique paths and values are deduplicated.
+
+On the search side, exact queries will be prioritized, but the document paths
+and key=value pairs will also be analyzed with 3-grams to have some amount of
+fuzzy search. The reason that a Levenshtein-Distance was not used instead, is due
+to searching multiple fields at the same time, which is a use case where
+Elasticsearch does not support proper fuzzy searching.
+
+### Document Search
+Given a text query, each token is considered separately. Each token will be fed
+through a handful of analyzers on the Elasticsearch side, and will be compared
+with the reverse document index of each document fields. It will then determine
+the best matching documents. Text ordering is largely insignificant. This makes
+sense for the structured search, but may leave room for improvement for the
+text only search within the document.
+
+Each token _must_ be matched, so each white space character acts as a
+conjunction of individual queries. There are also ways of telling
+Elasticsearch that some things _should_ match, but I think for now it makes
+more sense to leave it as is.
+
+I think this behavior is sufficient to make the search feel fairly intuitive
+while providing support for fairly complex use cases.
+
+### Metrics Computation
+From the each kustomization document that is indexed, we can find it's
+resources that are publicly available. This includes other kustomizations.
+From this, we can build a directed graph of dependencies and reverse
+dependencies.
+
+This opens up the possibility to add a plethora of graph metrics that can
+give the project maintainers feedback and insight into how people are using
+their tools.
+
+Some of these are useful such as getting an idea for how large the dependency
+graphs actually grow in practice, and can be used to find _popular_
+kustomizations within the corpus. This lends itself to implementing PageRank
+to help bubble up popular results as good search results. I unfortunately
+did not have the time to implement the algorithm, but I do plan to revisit
+this sometime soon to add a few good and efficient implementations of useful
+graph algorithms that would be useful to have. See the Roadmap.md for a more
+complete list of features that could be added and how I think they could be
+implemented.

internal/tools/ROADMAP.md

@@ -0,0 +1,176 @@
+# Road map and comments about this work
+
+From working on this project, here is a collection of thoughts and suggestions
+for future improvements. For any questions about this, or to request help do
+not hesitate to contact @damienr74 on GitHub, my email should be listed.
+
+I think this project has the potential for the K8s community to promote best
+practices. If this becomes popular, It could become easier to find
+*subjectively good* configurations. This can act as a way to guide newcomers
+to k8s config features that are easy to maintain, practical, and tested in some
+real world environment. However, a lot of work remains to be made if this is
+to happen. Extracting and ranking semantic-level information from the open
+source configuration files, is definitely not trivial, and will require a lot of
+though and consideration from the experts and the patterns that successful k8s
+project follow. This, is outside of my scope having little to no experience with
+k8s other than working on this project; however, if you have ideas I can
+probably suggest approaches in order to implement it, having worked a lot on
+this project.
+
+### Improving configuration files and container configs
+I did not have a lot of time to refactor the images to use configmaps for
+everything. This is a good thing to improve, should be fairly easy. Another
+thing that could make the user experience of launcing this could be to make all
+of the go utilities be subcommands to the same binary/container image. This
+would reduce the number of things that would have to be rebuilt, in order to get
+it running, and it would make the application (and its components) more self
+contained. (also has some disadvantages, so I'll let someone else decide.
+
+### Adding graph metrics
+From the Redis graph representation, we are able to run a multitude of graph
+algorithms (not all of which are implemented).
+
+The simplest one would be to run kruskal's algorithm to find connected
+components, and to compute graph metrics on each component. Here are some of the
+metrics that may be useful:
+
++ Average size and histograms of the sizes of each components.
+
++ Average size and histograms of the node with the highest in degree (rdeps) of
+  each component.
+
++ Average size and histograms of the number of repositories in a connected
+  component.
+
++ Any other metric that may be helpful to measure the scale of the kustomize
+  import graph.
+
+Another cool thing that may be helpful, would be to output the graph
+representation of deps/rdeps. This should be fairly easy to do with graphviz/dot
+so if anyone really wants this, I (damienr74) should be able to do it. Feel free
+to send me an email or to @ mention me in an issue.
+
+Note: dfs could also be used to find connected components, but I think union
+find is preferable, since the results can be stored and modified very
+efficiently. The only challenging part would be to implement deleting of edges
+and nodes from a component efficiently, but I know it is possible to support
+these operations with a union find structure.
+
+### Implementing PageRank
+The graph is set up to be able to efficiently compute PageRank since the edge
+weights are real valued, and the graph representation is sparse which means that
+it will fit in the memory of a single machine which will make the processing
+much more efficient.
+
+It could also be implemented as a Redis script, but I feel like there's
+something fundamentally wrong with implementing PageRank in lua. :P
+
+### Implement feature tracking
+Each day, when the crawler finds and indexes these structured documents,
+it should insert aggregate data to a separate index. This data could look like the
+following:
+
+```json
+{
+  "kind": "kustomization",
+  "added_identifiers": [
+    {
+      "identifier": "some:new:k8s:feature",
+      "addedIn": [
+        "docID1",
+        "docID100",
+        "docID45",
+        ...
+      ],
+    }
+    {
+      "identifier": "another:k8s:feature",
+      "documents": [
+        ...
+      ],
+    }
+    ...
+  ]
+
+  "removed_identifiers": [
+    {
+      "identifier": "some:deprecated:field",
+      "documents": [
+        ...
+      ]
+    }
+  ]
+}
+```
+
+This would make it fairly easy to get deep insight into:
+- the speed at which things can effectively be deprecated.
+- how many people are migrating to current best practices.
+- how many documents get updated frequently/rarely.
+- detailed cross sections of growth/regression over conjunctions of features.
+- a world of possibilities.
+
+This is also something that I would be interested to work on sometime soon, so
+feel free to contact me (damienr74) or ask questions about this.
+
+As needed, it could be a good idea to also aggregate past data with a larger
+granularity. for instance each month, the past 30 days can be aggregated into
+weekish durations, And every year these weekly aggregations can be converted
+into monthly summaries depending on how much data this ends up being, and how
+much you want to pay for the storage of this data.
+
+Another cool way to compress this data would be to dynamically compress this
+data into a logarithmic number of buckets with decreasing granularity. But it
+seems like overkill for the amount of data that we'd likely get.
+
+### The UI probably needs a lot of work
+I'm not much of a UI/UX person and have little to no experience in developing
+these types of applications. If anyone with Angular experience wants to dive in
+and completely restructure the app to make the UI/UX/Code health better that
+would be greatly appreciated.
+
+### Query tuning probably still has to be adjusted
+I'm also not an expert in Elasticsearch. From what I could read in the docs,
+I think I've made sane decisions in converting user queries into meaningful
+Elasticsearch queries, but I'm sure there are a lot of improvements that remain
+to be done in order to get more accurate results.
+
+
+### Some other signals that indicate the presence of a good configuration file
+There are lots of heuristics that could be used to achieve this. Here are a
+couple in no particular order:
+
++ Penalize for the number of yaml `---` document splits. I'm not sure what the
+  general consensus is, but I think it's better to separate them, since it
+  makes git commits less noisy, it's a trivial transformation, and it makes
+  config files smaller. However, I can understand the argument that its somewhat
+  practical to keep an overall view of the configurations together (maybe).
+
++ Penalize the number of unique identifiers in a structured document. I think
+  this makes sense, since we don't want to have someone game the search engine
+  to match documents with every possible path from the k8s docs. PageRank might
+  help with this to some extent, but with a small corpus it would be fairly easy
+  to game.
+
++ Assign weights to the usefulness of certain fields. It would be good to
+  promote documents that use `keyRefFromConfigMap`, liveness probes, etc.
+
+These are the main ones I can think of, but I'm sure there are a *ton* of
+ways to achieve this.
+
+If the corpus gets large enough, we might even be able to use *blockchains*,
+*machine learning*, and maybe even self-driving cars.
+
+### Add more support for indexing of other k8s/kustomize related data
+One thing that jumps to mind is the use of kustomize plugins. They are easy
+to track since they all have an unused global variable: `var KustomizePluggin`
+it would be easy to run the pluginator command and generate godocs for each
+go file with this unique identifier.
+
+For the sake of completeness, here is the full GitHub query that we can use to
+find these:
+`api.github.com/search/code?q=var+KustomizePlugin+extension%3A.go&access_token=access_token`
+
+Godoc will not show much, since most packages will be using package main, but
+using pluginator we can make it a properly named package such that Godoc would
+actually generate the relevant documentation.

internal/tools/backend/search_backend.go

@@ -0,0 +1,195 @@
+package server
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"log"
+	"net/http"
+	"os"
+	"strconv"
+	"strings"
+
+	"github.com/gorilla/mux"
+	"github.com/rs/cors"
+
+	"sigs.k8s.io/kustomize/internal/tools/index"
+)
+
+type kustomizeSearch struct {
+	ctx context.Context
+	// Eventually pIndex *index.PlugginIndex
+	idx    *index.KustomizeIndex
+	router *mux.Router
+	log    *log.Logger
+}
+
+// New server. Creating a server does not launch it. To launch simply:
+//	srv, _ := NewKustomizeSearch(context.Backgroud())
+//	err := srv.Serve()
+//	if err != nil {
+//		// Handle server issues.
+//	}
+//
+// The server has three enpoints, two of which are functional:
+//
+// /search: processes the ?q= parameter for a text query and
+// returns a list of 10 resutls starting from the ?from= value provided,
+// with the default being zero.
+//
+// /metrics: returns overall metrics about the files indexed. Returns
+// timeseries data for kustomization files, and returns breakdown of file
+// counts by their 'kind' fields
+//
+// /register: not implemented, but meant as an endpoint for adding new
+// kustomization files to the corpus.
+func NewKustomizeSearch(ctx context.Context) (*kustomizeSearch, error) {
+	idx, err := index.NewKustomizeIndex(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	ks := &kustomizeSearch{
+		ctx:    ctx,
+		idx:    idx,
+		router: mux.NewRouter(),
+		log: log.New(os.Stdout, "Kustomize server: ",
+			log.LstdFlags|log.Llongfile|log.LUTC),
+	}
+
+	return ks, nil
+}
+
+// Set up common middleware and the routes for the server.
+func (ks *kustomizeSearch) routes() {
+
+	// Setup middleware.
+	ks.router.Use(func(handler http.Handler) http.Handler {
+		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			w.Header().Set("Content-Type", "application/json")
+			handler.ServeHTTP(w, r)
+		})
+	})
+
+	ks.router.HandleFunc("/liveness", ks.liveness()).Methods(http.MethodGet)
+	ks.router.HandleFunc("/readiness", ks.readiness()).Methods(http.MethodGet)
+	ks.router.HandleFunc("/search", ks.search()).Methods(http.MethodGet)
+	ks.router.HandleFunc("/metrics", ks.metrics()).Methods(http.MethodGet)
+	ks.router.HandleFunc("/register", ks.register()).Methods(http.MethodPost)
+}
+
+// Start listening and serving on the provided port.
+func (ks *kustomizeSearch) Serve(port int) error {
+	ks.routes()
+	handler := cors.Default().Handler(ks.router)
+	s := &http.Server{
+		Addr:    fmt.Sprintf(":%d", port),
+		Handler: handler,
+		// Timeouts/Limits
+	}
+
+	return s.ListenAndServe()
+}
+
+// /liveness endpoint
+func (ks *kustomizeSearch) liveness() http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusOK)
+	}
+}
+
+// /readyness endpoint
+func (ks *kustomizeSearch) readiness() http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		opt := index.KustomizeSearchOptions{}
+		_, err := ks.idx.Search("", opt)
+		if err != nil {
+			http.Error(w,
+				`{ "error": "could not connect to database" }`,
+				http.StatusInternalServerError)
+			return
+		}
+		w.WriteHeader(http.StatusOK)
+	}
+}
+
+// /register endpoint.
+func (ks *kustomizeSearch) register() http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		http.Error(w, "not implemented", http.StatusInternalServerError)
+	}
+}
+
+// /search endpoint.
+func (ks *kustomizeSearch) search() http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		values := r.URL.Query()
+
+		queries := values["q"]
+		ks.log.Println("Query: ", values)
+
+		var from int
+		fromParam := values["from"]
+		if len(fromParam) > 0 {
+			from, _ = strconv.Atoi(fromParam[0])
+			if from < 0 {
+				from = 0
+			}
+		}
+		_, noKinds := values["nokinds"]
+
+		opt := index.KustomizeSearchOptions{
+			SearchOptions: index.SearchOptions{
+				Size: 10,
+				From: from,
+			},
+			KindAggregation: !noKinds,
+		}
+
+		results, err := ks.idx.Search(strings.Join(queries, " "), opt)
+		if err != nil {
+			ks.log.Println("Error: ", err)
+			http.Error(w, fmt.Sprintf(
+				`{ "error": "could not complete the query" }`),
+				http.StatusInternalServerError)
+			return
+		}
+
+		enc := json.NewEncoder(w)
+		setIndent(enc)
+		if err = enc.Encode(results); err != nil {
+			http.Error(w, `{ "error": "failed to send back results" }`,
+				http.StatusInternalServerError)
+			return
+		}
+		return
+	}
+}
+
+// metrics endpoint.
+func (ks *kustomizeSearch) metrics() http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		res, err := ks.idx.Search("", index.KustomizeSearchOptions{
+			KindAggregation:       true,
+			TimeseriesAggregation: true,
+		})
+		if err != nil {
+			http.Error(w, `{ "error": "could not perform the search."}`,
+				http.StatusInternalServerError)
+			return
+		}
+
+		enc := json.NewEncoder(w)
+		setIndent(enc)
+		if err := enc.Encode(res); err != nil {
+			http.Error(w, `{ "error": "could not format return value" }`,
+				http.StatusInternalServerError)
+			return
+		}
+	}
+}
+
+// make json response human readable.
+func setIndent(e *json.Encoder) {
+	e.SetIndent("", "  ")
+}

internal/tools/cmd/backend/Dockerfile

@@ -0,0 +1,14 @@
+FROM golang:1.11 AS build
+
+ARG GO111MODULE=on
+
+WORKDIR /go/src/sigs.k8s.io/kustomize/internal/tools
+COPY . /go/src/sigs.k8s.io/kustomize/internal/tools
+
+RUN go mod download
+RUN CGO_ENABLED=0 go install sigs.k8s.io/kustomize/internal/tools/cmd/backend/
+
+FROM scratch
+COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
+COPY --from=build /go/bin/backend /
+ENTRYPOINT ["/backend"]

internal/tools/cmd/backend/main.go

@@ -0,0 +1,30 @@
+package main
+
+import (
+	"context"
+	"log"
+	"os"
+	"strconv"
+
+	"sigs.k8s.io/kustomize/internal/tools/backend"
+)
+
+func main() {
+	portStr := os.Getenv("PORT")
+	port, err := strconv.Atoi(portStr)
+	if portStr == "" || err != nil {
+		log.Fatalf("$PORT(%s) must be set to an integer\n", portStr)
+	}
+
+	ctx := context.Background()
+
+	ks, err := server.NewKustomizeSearch(ctx)
+	if err != nil {
+		log.Fatalf("Error creating kustomize server: %v", ks)
+	}
+
+	err = ks.Serve(port)
+	if err != nil {
+		log.Fatalf("Error while running server: %v", err)
+	}
+}

internal/tools/config/base/kustomization.yaml

@@ -0,0 +1,6 @@
+configmapGenerator:
+- name: elasticsearch-config
+  literals:
+  - es-url="http://esbasic-master:9200"
+  - kustomize-index-name="kustomize"
+  - plugin-index-name="plugin"

internal/tools/config/crawler/base/kustomization.yaml

@@ -0,0 +1,13 @@
+resources:
+- ../../base
+
+configmapGenerator:
+- name: crawler-http-cache
+  literals:
+  - redis-cache-url="redis://redis-http-cache:6379"
+
+
+secretGenerator:
+- name: github-access-token
+  files:
+  - token=github_api_secret.txt

internal/tools/config/crawler/cronjob/cronjob.yaml

@@ -0,0 +1,30 @@
+apiVersion: batch/v1beta1
+kind: CronJob
+metadata:
+  name: crawler
+spec:
+  schedule: "5 0 * * */1"
+  jobTemplate:
+    spec:
+      template:
+        spec:
+          restartPolicy: OnFailure
+          containers:
+          - name: crawler
+            image: gcr.io/kustomize-search/crawler:latest
+            env:
+            - name: GITHUB_ACCESS_TOKEN
+              valueFrom:
+                secretKeyRef:
+                  name: github-access-token
+                  key: token
+            - name: ELASTICSEARCH_URL
+              valueFrom:
+                configMapKeyRef:
+                  name: elasticsearch-config
+                  key: es-url
+            - name: REDIS_URL
+              valueFrom:
+                configMapKeyRef:
+                  name: crawler-http-cache
+                  key: redis-cache-url

internal/tools/config/crawler/cronjob/kustomization.yaml

@@ -0,0 +1,3 @@
+resources:
+- ../base
+- cronjob.yaml

internal/tools/config/crawler/job/job.yaml

@@ -0,0 +1,32 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: crawler
+spec:
+  template:
+    spec:
+      restartPolicy: OnFailure
+      containers:
+      - name: crawler
+        image: gcr.io/kustomize-search/crawler:latest
+        env:
+        - name: GITHUB_ACCESS_TOKEN
+          valueFrom:
+            secretKeyRef:
+              name: github-access-token
+              key: token
+        - name: ELASTICSEARCH_URL
+          valueFrom:
+            configMapKeyRef:
+              name: elasticsearch-config
+              key: es-url
+        - name: REDIS_CACHE_URL
+          valueFrom:
+            configMapKeyRef:
+              name: crawler-http-cache
+              key: redis-cache-url
+        - name: REDIS_KEY_URL
+          valueFrom:
+            configMapKeyRef:
+              name: redis-keystore
+              key: keystore-url

internal/tools/config/crawler/job/kustomization.yaml

@@ -0,0 +1,3 @@
+resources:
+- ../base
+- job.yaml

internal/tools/config/redis/document_keystore/kustomization.yaml

@@ -0,0 +1,7 @@
+resources:
+- redis.yaml
+- service.yaml
+
+commonLabels:
+  app: redis
+  tier: document-keystore

internal/tools/config/redis/document_keystore/redis.yaml

@@ -0,0 +1,37 @@
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: redis-docs-keystore
+spec:
+  serviceName: "redis-docs-keystore"
+  template:
+    spec:
+      containers:
+      - name: redis
+        image: redis:5-alpine
+        imagePullPolicy: Always
+        args:
+          - "--save"
+          - "900"
+          - "1"
+          - "--save"
+          - "30"
+          - "100"
+          - "--appendonly"
+          - "yes"
+        ports:
+        - name: redis-docs-port
+          containerPort: 6379
+        volumeMounts:
+        - mountPath: /data
+          name: redis-docs-keystore-data
+      restartPolicy: Always
+  volumeClaimTemplates:
+  - metadata:
+      name: redis-docs-keystore-data
+    spec:
+      accessModes:
+      - ReadWriteOnce
+      resources:
+        requests:
+          storage: 4Gi

internal/tools/config/redis/document_keystore/service.yaml

@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: redis-docs-keystore
+spec:
+  clusterIP: None
+  ports:
+  - protocol: "TCP"
+    port: 6379
+    targetPort: redis-docs-port

internal/tools/config/redis/http_cache/kustomization.yaml

@@ -0,0 +1,7 @@
+resources:
+- redis.yaml
+- service.yaml
+
+commonLabels:
+  app: redis
+  tier: http-cache

internal/tools/config/redis/http_cache/redis.yaml

@@ -0,0 +1,16 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: redis-http-cache
+spec:
+  template:
+    spec:
+      containers:
+      - name: redis
+        image: redis:5-alpine
+        imagePullPolicy: Always
+        # see redis.io/topics/lru-cache for other policy options.
+        args: ["--maxmemory", "1gb", "--maxmemory-policy", "allkeys-lru"]
+        ports:
+        - name: http-cache-port
+          containerPort: 6379

internal/tools/config/redis/http_cache/service.yaml

@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: redis-http-cache
+spec:
+  clusterIP: None
+  ports:
+  - protocol: "TCP"
+    port: 6379
+    targetPort: http-cache-port

internal/tools/config/webapp/backend/deployment.yaml

@@ -0,0 +1,38 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: kustomize-search
+spec:
+  selector:
+    matchLabels:
+      app: kustomize-search
+      tier: backend
+  replicas: 1
+  template:
+    metadata:
+      labels:
+        app: kustomize-search
+        tier: backend
+    spec:
+      containers:
+      - name: kustomize-search
+        image: gcr.io/kustomize-search/backend:latest
+        livenessProbe:
+          httpGet:
+            path: /liveness
+            port: backend-port
+        readinessProbe:
+          httpGet:
+            path: /readiness
+            port: backend-port
+        ports:
+        - name: backend-port
+          containerPort: 8080
+        env:
+        - name: ELASTICSEARCH_URL
+          valueFrom:
+            configMapKeyRef:
+              name: elasticsearch-config
+              key: es-url
+        - name: PORT
+          value: "8080"

internal/tools/config/webapp/backend/kustomization.yaml

@@ -0,0 +1,4 @@
+resources:
+- ../../base
+- deployment.yaml
+- service.yaml

internal/tools/config/webapp/backend/service.yaml

@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: kustomize-search
+spec:
+  selector:
+    app: kustomize-search
+    tier: backend
+  ports:
+  - protocol: "TCP"
+    port: 80
+    targetPort: backend-port
+  type: LoadBalancer
+  loadBalancerIP: ""

internal/tools/config/webapp/frontend/deployment.yaml

@@ -0,0 +1,22 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: kustomize-search-ui
+spec:
+  selector:
+    matchLabels:
+      app: kustomize-search
+      tier: frontend
+  replicas: 1
+  template:
+    metadata:
+      labels:
+        app: kustomize-search
+        tier: frontend
+    spec:
+      containers:
+      - name: frontend
+        image: gcr.io/kustomize-search/frontend:latest
+        ports:
+        - name: frontend-port
+          containerPort: 80

internal/tools/config/webapp/frontend/kustomization.yaml

@@ -0,0 +1,4 @@
+resources:
+- ../../base
+- deployment.yaml
+- service.yaml

internal/tools/config/webapp/frontend/service.yaml

@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: kustomize-search-ui
+spec:
+  selector:
+    app: kustomize-search
+    tier: frontend
+  ports:
+  - protocol: "TCP"
+    port: 80
+    targetPort: frontend-port
+  type: LoadBalancer
+  loadBalancerIP: ""

internal/tools/crawler/crawler.go

@@ -0,0 +1,236 @@
+// Package crawler provides helper methods and defines an interface for lauching
+// source repository crawlers that retrieve files from a source and forwards
+// to a channel for indexing and retrieval.
+package crawler
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"os"
+	"sync"
+
+	_ "github.com/gomodule/redigo/redis"
+
+	"sigs.k8s.io/kustomize/internal/tools/doc"
+)
+
+var (
+	logger = log.New(os.Stdout, "Crawler: ", log.LstdFlags|log.LUTC|log.Llongfile)
+)
+
+// Crawler forwards documents from source repositories to index and store them
+// for searching. Each crawler is responsible for querying it's source of
+// information, and forwarding files that have not been seen before or that need
+// updating.
+type Crawler interface {
+	// Crawl returns when it is done processing. This method does not take
+	// ownership of the channel. The channel is write only, and it
+	// designates where the crawler should forward the documents.
+	Crawl(ctx context.Context, output chan<- CrawlerDocument) error
+
+	// Get the document data given the FilePath, Repo, and Ref/Tag/Branch.
+	FetchDocument(context.Context, *doc.Document) error
+	// Write to the document what the created time is.
+	SetCreated(context.Context, *doc.Document) error
+
+	Match(*doc.Document) bool
+}
+
+type CrawlerDocument interface {
+	ID() string
+	GetDocument() *doc.Document
+	GetResources() ([]*doc.Document, error)
+	WasCached() bool
+}
+
+type CrawlerSeed []*doc.Document
+
+type IndexFunc func(CrawlerDocument, Crawler) error
+type Converter func(*doc.Document) (CrawlerDocument, error)
+
+// Cleaner, more efficient, and more extensible crawler implementation.
+// The seed must include the ids of each document in the index.
+func CrawlFromSeed(ctx context.Context, seed CrawlerSeed,
+	crawlers []Crawler, conv Converter, indx IndexFunc) {
+
+	seen := make(map[string]struct{})
+
+	logIfErr := func(err error) {
+		if err == nil {
+			return
+		}
+		logger.Println("error: ", err)
+	}
+
+	stack := make(CrawlerSeed, 0)
+
+	findMatch := func(d *doc.Document) Crawler {
+		for _, crawl := range crawlers {
+			if crawl.Match(d) {
+				return crawl
+			}
+		}
+
+		return nil
+	}
+
+	addBranches := func(cdoc CrawlerDocument, match Crawler) {
+		if _, ok := seen[cdoc.ID()]; ok {
+			return
+		}
+
+		seen[cdoc.ID()] = struct{}{}
+		// Insert into index
+		err := indx(cdoc, match)
+		logIfErr(err)
+		if err != nil {
+			return
+		}
+
+		deps, err := cdoc.GetResources()
+		logIfErr(err)
+		if err != nil {
+			return
+		}
+		for _, dep := range deps {
+			if _, ok := seen[dep.ID()]; ok {
+				continue
+			}
+			stack = append(stack, dep)
+		}
+	}
+
+	doCrawl := func(docsPtr *CrawlerSeed) {
+		for len(*docsPtr) > 0 {
+			back := len(*docsPtr) - 1
+			next := (*docsPtr)[back]
+			*docsPtr = (*docsPtr)[:back]
+
+			match := findMatch(next)
+			if match == nil {
+				logIfErr(fmt.Errorf(
+					"%v could not match any crawler", next))
+				continue
+			}
+
+			err := match.FetchDocument(ctx, next)
+			logIfErr(err)
+			// If there was no change or there is an error, we don't have
+			// to branch out, since the dependencies are already in the
+			// index, or we cannot find the document.
+			if err != nil || next.WasCached() {
+				continue
+			}
+
+			cdoc, err := conv(next)
+			logIfErr(err)
+			if err != nil {
+				continue
+			}
+
+			addBranches(cdoc, match)
+		}
+	}
+	// Exploit seed to update bulk of corpus.
+	logger.Printf("updating %d documents from seed\n", len(seed))
+	doCrawl(&seed)
+	// Traverse any new links added while updating corpus.
+	logger.Printf("crawling %d new documents found in the seed\n", len(stack))
+	doCrawl(&stack)
+
+	ch := make(chan CrawlerDocument, 1<<10)
+	wg := sync.WaitGroup{}
+
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		for cdoc := range ch {
+			if _, ok := seen[cdoc.ID()]; ok {
+				continue
+			}
+			match := findMatch(cdoc.GetDocument())
+			if match == nil {
+				logIfErr(fmt.Errorf(
+					"%v could not match any crawler", cdoc))
+				continue
+			}
+			addBranches(cdoc, match)
+		}
+	}()
+
+	// Exploration through APIs.
+	errs := CrawlerRunner(ctx, ch, crawlers)
+	if errs != nil {
+		for _, err := range errs {
+			logIfErr(err)
+		}
+	}
+	close(ch)
+	logger.Println("Processing the new documents from the crawlers' exploration.")
+	wg.Wait()
+	// Handle deps of newly discovered documents.
+	logger.Printf("crawling the %d new documents from the crawlers' exploration.",
+		len(stack))
+	doCrawl(&stack)
+}
+
+// CrawlerRunner is a blocking function and only returns once all of the
+// crawlers are finished with execution.
+//
+// This function uses the output channel to forward kustomization documents
+// from a list of crawlers. The output is to be consumed by a database/search
+// indexer for later retrieval.
+//
+// The return value is an array of errors in which each index represents the
+// index of the crawler that emitted the error. Although the errors themselves
+// can be nil, the array will always be exactly the size of the crawlers array.
+//
+// Crawler Runner takes in a seed, which represents the documents stored in an
+// index somewhere. The document data is not required to be populated. If there
+// are many documents, this is preferable. The order of iteration over the seed
+// is not garanteed, but the CrawlerRunner does guarantee that every element
+// from the seed will be processed before any other documents from the
+// crawlers.
+func CrawlerRunner(ctx context.Context,
+	output chan<- CrawlerDocument, crawlers []Crawler) []error {
+
+	errs := make([]error, len(crawlers))
+	wg := sync.WaitGroup{}
+
+	for i, crawler := range crawlers {
+		// Crawler implementations get their own channels to prevent a
+		// crawler from closing the main output channel.
+		docs := make(chan CrawlerDocument)
+		wg.Add(2)
+
+		// Forward all of the documents from this crawler's channel to
+		// the main output channel.
+		go func(docs <-chan CrawlerDocument) {
+			defer wg.Done()
+			for doc := range docs {
+				output <- doc
+			}
+		}(docs)
+
+		// Run this crawler and capture its returned error.
+		go func(idx int, crawler Crawler,
+			docs chan<- CrawlerDocument) {
+
+			defer func() {
+				wg.Done()
+				if r := recover(); r != nil {
+					errs[idx] = fmt.Errorf(
+						"%+v panicked: %v, additional error %v",
+						crawler, r, errs[idx],
+					)
+				}
+			}()
+			defer close(docs)
+			errs[idx] = crawler.Crawl(ctx, docs)
+		}(i, crawler, docs) // Copies the index and the crawler
+	}
+
+	wg.Wait()
+	return errs
+}

internal/tools/crawler/crawler_test.go

@@ -0,0 +1,356 @@
+package crawler
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"reflect"
+	"sort"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"sigs.k8s.io/kustomize/internal/tools/doc"
+	"sigs.k8s.io/kustomize/v3/pkg/pgmconfig"
+)
+
+const (
+	kustomizeRepo = "https://github.com/kubernetes-sigs/kustomize"
+)
+
+// Simple crawler that forwards it's list of documents to a provided channel and
+// returns it's error to the caller.
+type testCrawler struct {
+	matchPrefix string
+	err         error
+	docs        []doc.KustomizationDocument
+	lukp        map[string]int
+}
+
+func (c testCrawler) Match(d *doc.Document) bool {
+	return d != nil && strings.HasPrefix(d.ID(), c.matchPrefix)
+}
+
+func (c testCrawler) FetchDocument(ctx context.Context, d *doc.Document) error {
+	if i, ok := c.lukp[d.ID()]; ok {
+		d.DocumentData = c.docs[i].DocumentData
+		return nil
+	}
+	for _, suffix := range pgmconfig.KustomizationFileNames {
+		fmt.Println(d.ID(), "/", suffix)
+		i, ok := c.lukp[d.ID()+"/"+suffix]
+		if !ok {
+			continue
+		}
+		d.FilePath += "/" + suffix
+		d.DocumentData = c.docs[i].DocumentData
+		return nil
+	}
+	return fmt.Errorf("Document %v does not exist for matcher: %s",
+		d, c.matchPrefix)
+}
+
+func (c testCrawler) SetCreated(ctx context.Context, d *doc.Document) error {
+	d.CreationTime = &time.Time{}
+	return nil
+}
+
+func newCrawler(matchPrefix string, err error,
+	docs []doc.KustomizationDocument) testCrawler {
+	c := testCrawler{
+		matchPrefix: matchPrefix,
+		err:         err,
+		docs:        docs,
+		lukp:        make(map[string]int),
+	}
+	for i, d := range docs {
+		c.lukp[d.ID()] = i
+	}
+	return c
+}
+
+// Crawl implements the Crawler interface for testing.
+func (c testCrawler) Crawl(ctx context.Context,
+	output chan<- CrawlerDocument) error {
+
+	for i, d := range c.docs {
+		isResource := true
+		for _, suffix := range pgmconfig.KustomizationFileNames {
+			if strings.HasSuffix(d.FilePath, suffix) {
+				isResource = false
+				break
+			}
+		}
+		if isResource {
+			continue
+		}
+		output <- &c.docs[i]
+	}
+	return c.err
+}
+
+// Used to make sure that we're comparing documents in order. This is needed
+// since these documents will be sent concurrently.
+type sortableDocs []doc.KustomizationDocument
+
+func (s sortableDocs) Less(i, j int) bool {
+	return s[i].FilePath < s[j].FilePath
+}
+
+func (s sortableDocs) Swap(i, j int) {
+	s[i], s[j] = s[j], s[i]
+}
+
+func (s sortableDocs) Len() int {
+	return len(s)
+}
+
+func TestCrawlerRunner(t *testing.T) {
+	fmt.Println("testing CrawlerRunner")
+	tests := []struct {
+		tc   []Crawler
+		errs []error
+		docs sortableDocs
+	}{
+		{
+			tc: []Crawler{
+				testCrawler{
+					docs: []doc.KustomizationDocument{
+						{Document: doc.Document{
+							FilePath: "crawler1/doc1/kustomization.yaml",
+						}},
+						{Document: doc.Document{
+							FilePath: "crawler1/doc2/kustomization.yaml",
+						}},
+						{Document: doc.Document{
+							FilePath: "crawler1/doc3/kustomization.yaml",
+						}},
+					},
+				},
+				testCrawler{err: errors.New("crawler2")},
+				testCrawler{},
+				testCrawler{
+					docs: []doc.KustomizationDocument{
+						{Document: doc.Document{
+							FilePath: "crawler4/doc1/kustomization.yaml",
+						}},
+						{Document: doc.Document{
+							FilePath: "crawler4/doc2/kustomization.yaml",
+						}},
+					},
+					err: errors.New("crawler4"),
+				},
+			},
+			errs: []error{
+				nil,
+				errors.New("crawler2"),
+				nil,
+				errors.New("crawler4"),
+			},
+			docs: sortableDocs{
+				{Document: doc.Document{
+					FilePath: "crawler1/doc1/kustomization.yaml",
+				}},
+				{Document: doc.Document{
+					FilePath: "crawler1/doc2/kustomization.yaml",
+				}},
+				{Document: doc.Document{
+					FilePath: "crawler1/doc3/kustomization.yaml",
+				}},
+				{Document: doc.Document{
+					FilePath: "crawler4/doc1/kustomization.yaml",
+				}},
+				{Document: doc.Document{
+					FilePath: "crawler4/doc2/kustomization.yaml",
+				}},
+			},
+		},
+	}
+
+	for _, test := range tests {
+		output := make(chan CrawlerDocument)
+		wg := sync.WaitGroup{}
+		wg.Add(1)
+
+		// Run the Crawler runner with a list of crawlers.
+		go func() {
+			defer close(output)
+			defer wg.Done()
+
+			errs := CrawlerRunner(context.Background(),
+				output, test.tc)
+
+			// Check that errors are returned as they should be.
+			if !reflect.DeepEqual(errs, test.errs) {
+				t.Errorf("Expected errs (%v) to equal (%v)",
+					errs, test.errs)
+			}
+
+		}()
+
+		// Iterate over the output channel of Crawler runner.
+		returned := make(sortableDocs, 0, len(test.docs))
+		for o := range output {
+			d, ok := o.(*doc.KustomizationDocument)
+			if !ok || d == nil {
+				t.Errorf("%T not expected type (%T)",
+					o, d)
+			}
+			returned = append(returned, *d)
+		}
+
+		// Check that all documents are received.
+		sort.Sort(returned)
+		if !reflect.DeepEqual(returned, test.docs) {
+			t.Errorf("Expected docs (%v) to equal (%v)\n",
+				returned, test.docs)
+		}
+
+		wg.Wait()
+	}
+}
+
+func TestCrawlFromSeed(t *testing.T) {
+	fmt.Println("testing CrawlFromSeed")
+
+	tests := []struct {
+		seed    CrawlerSeed
+		matcher string
+		corpus  []doc.KustomizationDocument
+	}{
+		{
+			seed: CrawlerSeed{
+				{
+					RepositoryURL: kustomizeRepo,
+					FilePath:      "examples/helloWorld/kustomization.yaml",
+				},
+				{
+					RepositoryURL: kustomizeRepo,
+					FilePath:      "examples/other/kustomization.yaml",
+				},
+			},
+			matcher: kustomizeRepo,
+			corpus: []doc.KustomizationDocument{
+				// Visited from the seed, will be ignored in the crawl.
+				{Document: doc.Document{
+					RepositoryURL: kustomizeRepo,
+					FilePath:      "examples/helloWorld/kustomization.yaml",
+					DocumentData: `
+resources:
+- deployment.yaml
+`,
+				}},
+				// Also visited from the seed as a relative resource.
+				{Document: doc.Document{
+					RepositoryURL: kustomizeRepo,
+					FilePath:      "examples/helloWorld/deployment.yaml",
+					DocumentData: `
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: hello
+`,
+				}},
+				// Visited from the seed. Has a remote import.
+				{Document: doc.Document{
+					RepositoryURL: kustomizeRepo,
+					FilePath:      "examples/other/kustomization.yaml",
+					DocumentData: `
+resources:
+- https://github.com/kubernetes-sigs/kustomize/examples/other/overlay
+- service.yaml
+`,
+				}},
+				// Imported as a base from the seed.
+				{Document: doc.Document{
+					RepositoryURL: kustomizeRepo,
+					FilePath:      "examples/other/overlay/kustomization.yaml",
+					DocumentData: `
+resources:
+- https://github.com/kubernetes-sigs/kustomize/examples/seedcrawl1
+- https://github.com/kubernetes-sigs/kustomize/examples/seedcrawl2
+`,
+				}},
+				// Imported as a resource from the seed.
+				{Document: doc.Document{
+					RepositoryURL: kustomizeRepo,
+					FilePath:      "examples/other/service.yaml",
+				}},
+				// Visited from crawling seed.
+				{Document: doc.Document{
+					RepositoryURL: kustomizeRepo,
+					FilePath:      "examples/seedcrawl1/kustomization.yml",
+				}},
+				// Visited from crawling seed.
+				{Document: doc.Document{
+					RepositoryURL: kustomizeRepo,
+					FilePath:      "examples/seedcrawl2/kustomization.yaml",
+					DocumentData: `
+resources:
+- ../base
+- job.yaml
+`,
+				}},
+				// Visited from crawling seed.
+				{Document: doc.Document{
+					RepositoryURL: kustomizeRepo,
+					FilePath:      "examples/base/kustomization.yml",
+				}},
+				// Visited from crawling seed imported as resource.
+				{Document: doc.Document{
+					RepositoryURL: kustomizeRepo,
+					FilePath:      "examples/seedcrawl2/job.yaml",
+				}},
+				// Visited from the crawler runner.
+				{Document: doc.Document{
+					RepositoryURL: kustomizeRepo,
+					FilePath:      "examples/other/base/kustomization.yaml",
+					DocumentData: `
+resources:
+- ../app
+`,
+				}},
+				// Visited from the crawler runner.
+				{Document: doc.Document{
+					RepositoryURL: kustomizeRepo,
+					FilePath:      "examples/other/app/kustomization.yaml",
+					DocumentData: `
+resources:
+- resource.yaml
+`,
+				}},
+				// Visited from crawling runner imported as resource.
+				{Document: doc.Document{
+					RepositoryURL: kustomizeRepo,
+					FilePath:      "examples/other/app/resource.yaml",
+				}},
+			},
+		},
+	}
+
+	for _, tc := range tests {
+		cr := newCrawler(tc.matcher, nil, tc.corpus)
+		visited := make(map[string]int)
+		CrawlFromSeed(context.Background(), tc.seed, []Crawler{cr},
+			func(d *doc.Document) (CrawlerDocument, error) {
+				return &doc.KustomizationDocument{
+					Document: *d,
+				}, nil
+			},
+			func(d CrawlerDocument, cr Crawler) error {
+				visited[d.ID()]++
+				return nil
+			},
+		)
+		if lv, lc := len(visited), len(tc.corpus); lv != lc {
+			t.Errorf("error: %d of %d documents visited.", lv, lc)
+			t.Errorf("\nvisited (%v)\nexpected (%v).", visited, cr.lukp)
+		}
+		for id, cnt := range visited {
+			if cnt != 1 {
+				t.Errorf("%s not visited once (%d)", id, cnt)
+			}
+		}
+	}
+}

internal/tools/crawler/github/crawler.go

@@ -0,0 +1,582 @@
+// Package github implements the crawler.Crawler interface, getting data
+// from the Github search API.
+package github
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"math"
+	"net/http"
+	"os"
+	"regexp"
+	"strconv"
+	"strings"
+	"time"
+
+	"sigs.k8s.io/kustomize/internal/tools/crawler"
+	"sigs.k8s.io/kustomize/internal/tools/doc"
+	"sigs.k8s.io/kustomize/internal/tools/httpclient"
+	"sigs.k8s.io/kustomize/v3/pkg/git"
+	"sigs.k8s.io/kustomize/v3/pkg/pgmconfig"
+)
+
+var logger = log.New(os.Stdout, "Github Crawler: ",
+	log.LstdFlags|log.LUTC|log.Llongfile)
+
+// Implements crawler.Crawler.
+type githubCrawler struct {
+	client GitHubClient
+	query  Query
+}
+
+type GitHubClient struct {
+	RequestConfig
+	retryCount uint64
+	client     *http.Client
+}
+
+func NewClient(accessToken string, retryCount uint64, client *http.Client) GitHubClient {
+	return GitHubClient{
+		retryCount: retryCount,
+		client:     client,
+		RequestConfig: RequestConfig{
+			perPage:     githubMaxPageSize,
+			accessToken: accessToken,
+		},
+	}
+}
+
+func NewCrawler(accessToken string, retryCount uint64, client *http.Client,
+	query Query) githubCrawler {
+
+	return githubCrawler{
+		client: GitHubClient{
+			retryCount: retryCount,
+			client:     client,
+			RequestConfig: RequestConfig{
+				perPage:     githubMaxPageSize,
+				accessToken: accessToken,
+			},
+		},
+		query: query,
+	}
+}
+
+// Implements crawler.Crawler.
+func (gc githubCrawler) Crawl(
+	ctx context.Context, output chan<- crawler.CrawlerDocument) error {
+
+	noETagClient := GitHubClient{
+		RequestConfig: gc.client.RequestConfig,
+		client:        &http.Client{Timeout: gc.client.client.Timeout},
+		retryCount:    gc.client.retryCount,
+	}
+
+	// Since Github returns a max of 1000 results per query, we can use
+	// multiple queries that split the search space into chunks of at most
+	// 1000 files to get all of the data.
+	ranges, err := FindRangesForRepoSearch(newCache(noETagClient, gc.query))
+	if err != nil {
+		return fmt.Errorf("could not split %v into ranges, %v\n",
+			gc.query, err)
+	}
+
+	logger.Println("ranges: ", ranges)
+
+	// Query each range for files.
+	errs := make(multiError, 0)
+	for _, query := range ranges {
+		err := processQuery(ctx, gc.client, query, output)
+		if err != nil {
+			errs = append(errs, err)
+		}
+	}
+
+	if len(errs) > 0 {
+		return errs
+	}
+
+	return nil
+}
+
+func (gc githubCrawler) FetchDocument(ctx context.Context, d *doc.Document) error {
+	repoURL := d.RepositoryURL + "/" + d.FilePath + "?ref=" + d.DefaultBranch
+	repoSpec, err := git.NewRepoSpecFromUrl(repoURL)
+	if err != nil {
+		return fmt.Errorf("invalid repospec: %v", err)
+	}
+
+	url := "https://raw.githubusercontent.com/" + repoSpec.OrgRepo +
+		"/" + repoSpec.Ref + "/" + repoSpec.Path
+
+	handle := func(resp *http.Response, err error, path string) error {
+		if err == nil && resp.StatusCode == http.StatusOK {
+			d.IsSame = httpclient.FromCache(resp.Header)
+			defer resp.Body.Close()
+			data, err := ioutil.ReadAll(resp.Body)
+			if err != nil {
+				return err
+			}
+			d.DocumentData = string(data)
+			d.FilePath = d.FilePath + path
+			return nil
+		}
+		return err
+	}
+	resp, err := gc.client.GetRawUserContent(url)
+	if err := handle(resp, err, ""); err == nil {
+		return nil
+	}
+
+	for _, file := range pgmconfig.KustomizationFileNames {
+		resp, err = gc.client.GetRawUserContent(url + "/" + file)
+		err := handle(resp, err, "/"+file)
+		if err != nil {
+			continue
+		}
+	}
+	return fmt.Errorf("File Not Found: %s", url)
+}
+
+func (gc githubCrawler) SetCreated(ctx context.Context, d *doc.Document) error {
+	fs := GithubFileSpec{}
+	fs.Repository.FullName = d.RepositoryURL + "/" + d.FilePath
+	creationTime, err := gc.client.GetFileCreationTime(fs)
+	if err != nil {
+		return err
+	}
+	d.CreationTime = &creationTime
+	return nil
+}
+
+func (gc githubCrawler) Match(d *doc.Document) bool {
+	url := d.RepositoryURL + "/" + d.FilePath + "?ref=" + "/" +
+		d.DefaultBranch
+	repoSpec, err := git.NewRepoSpecFromUrl(url)
+	if err != nil {
+		return false
+	}
+
+	return strings.Contains(repoSpec.Host, "github.com")
+}
+
+// processQuery follows all of the pages in a query, and updates/adds the
+// documents from the crawl to the datastore/index.
+func processQuery(ctx context.Context, gcl GitHubClient, query string,
+	output chan<- crawler.CrawlerDocument) error {
+
+	queryPages := make(chan GithubResponseInfo)
+
+	go func() {
+		// Forward the document metadata to the retrieval channel.
+		// This separation allows for concurrent requests for the code
+		// search, and the retrieval portions of the API.
+		err := gcl.ForwardPaginatedQuery(ctx, query, queryPages)
+		if err != nil {
+			// TODO(damienr74) handle this error with redis?
+			logger.Println(err)
+		}
+		close(queryPages)
+	}()
+
+	errs := make(multiError, 0)
+	errorCnt := 0
+	totalCnt := 0
+	for page := range queryPages {
+		if page.Error != nil {
+			errs = append(errs, page.Error)
+			continue
+		}
+
+		for _, file := range page.Parsed.Items {
+			k, err := kustomizationResultAdapter(gcl, file)
+			if err != nil {
+				errs = append(errs, err)
+				errorCnt++
+				continue
+			}
+			output <- k
+			totalCnt++
+		}
+
+		logger.Printf("got %d files out of %d from API. %d of %d had errors\n",
+			totalCnt, page.Parsed.TotalCount, errorCnt, totalCnt)
+	}
+
+	return errs
+}
+
+func kustomizationResultAdapter(gcl GitHubClient, k GithubFileSpec) (
+	crawler.CrawlerDocument, error) {
+
+	data, err := gcl.GetFileData(k)
+	if err != nil {
+		return nil, err
+	}
+
+	if err != nil {
+		logger.Printf(
+			"(error: %v) initializing to current time.\n", err)
+	}
+
+	url := gcl.ReposRequest(k.Repository.FullName)
+	defaultBranch, err := gcl.GetDefaultBranch(url)
+	if err != nil {
+		logger.Printf(
+			"(error: %v) setting default_branch to master\n", err)
+		defaultBranch = "master"
+	}
+
+	doc := doc.KustomizationDocument{
+		Document: doc.Document{
+			DocumentData:  string(data),
+			FilePath:      k.Path,
+			DefaultBranch: defaultBranch,
+			RepositoryURL: k.Repository.URL,
+		},
+	}
+
+	return &doc, nil
+}
+
+// ForwardPaginatedQuery follows the links to the next pages and performs all of
+// the queries for a given search query, relaying the data from each request
+// back to an output channel.
+func (gcl GitHubClient) ForwardPaginatedQuery(ctx context.Context, query string,
+	output chan<- GithubResponseInfo) error {
+
+	logger.Println("querying: ", query)
+	response := gcl.parseGithubResponse(query)
+
+	if response.Error != nil {
+		return response.Error
+	}
+
+	output <- response
+
+	for response.LastURL != "" && response.NextURL != "" {
+		select {
+		case <-ctx.Done():
+			return nil
+		default:
+			response = gcl.parseGithubResponse(response.NextURL)
+			if response.Error != nil {
+				return response.Error
+			}
+
+			output <- response
+		}
+	}
+
+	return nil
+}
+
+// GetFileData gets the bytes from a file.
+func (gcl GitHubClient) GetFileData(k GithubFileSpec) ([]byte, error) {
+
+	url := gcl.ContentsRequest(k.Repository.FullName, k.Path)
+
+	resp, err := gcl.GetReposData(url)
+	if err != nil {
+		return nil, fmt.Errorf("%+v: could not get '%s' metadata: %v",
+			k, url, err)
+	}
+
+	data, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return nil, fmt.Errorf("%+v: could not read '%s' metadata: %v",
+			k, url, err)
+	}
+	resp.Body.Close()
+
+	type githubContentRawURL struct {
+		DownloadURL string `json:"download_url,omitempty"`
+	}
+	var rawURL githubContentRawURL
+	err = json.Unmarshal(data, &rawURL)
+	if err != nil {
+		return nil, fmt.Errorf(
+			"%+v: could not get 'download_url' from '%s' response: %v",
+			k, data, err)
+	}
+
+	resp, err = gcl.GetRawUserContent(rawURL.DownloadURL)
+	if err != nil {
+		return nil, fmt.Errorf("%+v: could not fetch file raw data '%s': %v",
+			k, rawURL.DownloadURL, err)
+	}
+
+	defer resp.Body.Close()
+	data, err = ioutil.ReadAll(resp.Body)
+	return data, err
+}
+
+func (gcl GitHubClient) GetDefaultBranch(url string) (string, error) {
+	resp, err := gcl.GetReposData(url)
+	if err != nil {
+		return "", fmt.Errorf(
+			"'%s' could not get default_branch: %v", url, err)
+	}
+	defer resp.Body.Close()
+	data, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return "", fmt.Errorf(
+			"could not read default_branch: %v", err)
+	}
+
+	type defaultBranch struct {
+		DefaultBranch string `json:"default_branch,omitempty"`
+	}
+	var branch defaultBranch
+	err = json.Unmarshal(data, &branch)
+	if err != nil {
+		return "", fmt.Errorf(
+			"default_branch json malformed: %v", err)
+	}
+
+	return branch.DefaultBranch, nil
+}
+
+// GetFileCreationTime gets the earliest date of a file.
+func (gcl GitHubClient) GetFileCreationTime(
+	k GithubFileSpec) (time.Time, error) {
+
+	url := gcl.CommitsRequest(k.Repository.FullName, k.Path)
+
+	defaultTime := time.Now()
+
+	resp, err := gcl.GetReposData(url)
+	if err != nil {
+		return defaultTime, fmt.Errorf(
+			"%+v: '%s' could not get metadata: %v", k, url, err)
+	}
+
+	type DateSpec struct {
+		Commit struct {
+			Author struct {
+				Date string `json:"date,omitempty"`
+			} `json:"author,omitempty"`
+		} `json:"commit,omitempty"`
+	}
+
+	_, lastURL := parseGithubLinkFormat(resp.Header.Get("link"))
+	if lastURL != "" {
+		resp, err = gcl.GetReposData(lastURL)
+		if err != nil {
+			return defaultTime, fmt.Errorf(
+				"%+v: '%s' could not get metadata: %v",
+				k, lastURL, err)
+		}
+	}
+
+	defer resp.Body.Close()
+	data, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return defaultTime, fmt.Errorf(
+			"%+v: failed to read metadata: %v", k, err)
+	}
+	earliestDate := []DateSpec{}
+	err = json.Unmarshal(data, &earliestDate)
+	size := len(earliestDate)
+	if err != nil || size == 0 {
+		return defaultTime, fmt.Errorf(
+			"%+v: server response '%s' not in expected format: %v",
+			k, data, err)
+	}
+
+	return time.Parse(time.RFC3339, earliestDate[size-1].Commit.Author.Date)
+}
+
+// TODO(damienr74) change the tickers to actually check api rate limits, reset
+// times, and throttle requests dynamically based off of current utilization,
+// instead of hardcoding the documented values, these calls are not quota'd.
+// This is now especially important, since caching the API requests will reduce
+// API quota use (so we can actually make more requests in the allotted time
+// period).
+//
+// See https://developer.github.com/v3/rate_limit/ for details.
+var (
+	searchRateTicker  = time.NewTicker(time.Second * 2)
+	contentRateTicker = time.NewTicker(time.Second * 1)
+)
+
+func throttleSearchAPI() {
+	<-searchRateTicker.C
+}
+
+func throttleRepoAPI() {
+	<-contentRateTicker.C
+}
+
+type multiError []error
+
+func (me multiError) Error() string {
+	size := len(me) + 2
+	strs := make([]string, size)
+	strs[0] = "Errors ["
+	for i, err := range me {
+		strs[i+1] = "\t" + err.Error()
+	}
+	strs[size-1] = "]"
+	return strings.Join(strs, "\n")
+}
+
+type GithubFileSpec struct {
+	Path       string `json:"path,omitempty"`
+	Repository struct {
+		API      string `json:"url,omitempty"`
+		URL      string `json:"html_url,omitempty"`
+		FullName string `json:"full_name,omitempty"`
+	} `json:"repository,omitempty"`
+}
+
+type githubResponse struct {
+	// MaxUint is reserved as a sentinel value.
+	// This is the number of files that match the query.
+	TotalCount uint64 `json:"total_count,omitempty"`
+
+	// Github representation of a file.
+	Items []GithubFileSpec `json:"items,omitempty"`
+}
+
+type GithubResponseInfo struct {
+	*http.Response
+	Parsed  *githubResponse
+	Error   error
+	NextURL string
+	LastURL string
+}
+
+func parseGithubLinkFormat(links string) (string, string) {
+	const (
+		linkNext    = "next"
+		linkLast    = "last"
+		linkInfoURL = 1
+		linkInfoRel = 2
+	)
+
+	next, last := "", ""
+	linkInfo := regexp.MustCompile(`<(.*)>.*; rel="(last|next)"`)
+
+	for _, link := range strings.Split(links, ",") {
+		linkParse := linkInfo.FindStringSubmatch(link)
+		if len(linkParse) != 3 {
+			continue
+		}
+
+		url := linkParse[linkInfoURL]
+		switch linkParse[linkInfoRel] {
+		case linkNext:
+			next = url
+		case linkLast:
+			last = url
+		default:
+		}
+	}
+
+	return next, last
+}
+
+func (gcl GitHubClient) parseGithubResponse(getRequest string) GithubResponseInfo {
+	resp, err := gcl.SearchGithubAPI(getRequest)
+	requestInfo := GithubResponseInfo{
+		Response: resp,
+		Error:    err,
+		Parsed:   nil,
+	}
+
+	if err != nil || resp == nil {
+		return requestInfo
+	}
+
+	var data []byte
+	defer resp.Body.Close()
+	data, requestInfo.Error = ioutil.ReadAll(resp.Body)
+	if requestInfo.Error != nil {
+		return requestInfo
+	}
+
+	if resp.StatusCode != http.StatusOK {
+		logger.Println("query: ", getRequest)
+		logger.Println("status not OK at the source")
+		logger.Println("header dump", resp.Header)
+		logger.Println("body dump", string(data))
+		requestInfo.Error = fmt.Errorf("request rejected, status '%s'",
+			resp.Status)
+		return requestInfo
+	}
+
+	requestInfo.NextURL, requestInfo.LastURL =
+		parseGithubLinkFormat(resp.Header.Get("link"))
+
+	resultCount := githubResponse{
+		TotalCount: math.MaxUint64,
+	}
+	requestInfo.Error = json.Unmarshal(data, &resultCount)
+	if requestInfo.Error != nil {
+		return requestInfo
+	}
+
+	requestInfo.Parsed = &resultCount
+
+	return requestInfo
+
+}
+
+// SearchGithubAPI performs a search query and handles rate limitting for
+// the 'code/search?' endpoint as well as timed retries in the case of abuse
+// prevention.
+func (gcl GitHubClient) SearchGithubAPI(query string) (*http.Response, error) {
+	throttleSearchAPI()
+	return gcl.getWithRetry(query)
+}
+
+// GetReposData performs a search query and handles rate limitting for
+// the '/repos' endpoint as well as timed retries in the case of abuse
+// prevention.
+func (gcl GitHubClient) GetReposData(query string) (*http.Response, error) {
+	throttleRepoAPI()
+	return gcl.getWithRetry(query)
+}
+
+// User content (file contents) is not API rate limited, so there's no use in
+// throttling this call.
+func (gcl GitHubClient) GetRawUserContent(query string) (*http.Response, error) {
+	return gcl.getWithRetry(query)
+}
+
+func (gcl GitHubClient) getWithRetry(
+	query string) (resp *http.Response, err error) {
+
+	resp, err = gcl.client.Get(query)
+	retryCount := gcl.retryCount
+
+	for err == nil &&
+		resp.StatusCode == http.StatusForbidden &&
+		retryCount > 0 {
+
+		retryTime := resp.Header.Get("Retry-After")
+		i, err := strconv.Atoi(retryTime)
+		if err != nil {
+			return resp, fmt.Errorf(
+				"query '%s' forbidden without 'Retry-After'", query)
+		}
+		logger.Printf(
+			"status forbidden, retring %d more times\n", retryCount)
+
+		logger.Printf("waiting %d seconds before retrying\n", i)
+		time.Sleep(time.Second * time.Duration(i))
+		retryCount--
+		resp, err = gcl.client.Get(query)
+	}
+
+	if err != nil {
+		return resp, fmt.Errorf("query '%s' could not be processed, %v",
+			query, err)
+	}
+
+	return resp, err
+}

internal/tools/crawler/github/queries.go

@@ -0,0 +1,224 @@
+package github
+
+import (
+	"fmt"
+	"net/url"
+	"strings"
+)
+
+const (
+	perPageArg     = "per_page"
+	accessTokenArg = "access_token"
+
+	githubMaxPageSize = 100
+)
+
+// Implementation detail, not important to external API.
+type queryField struct {
+	name  string
+	value interface{}
+}
+
+// Formats a query field.
+func (qf queryField) String() string {
+	var value string
+	switch v := qf.value.(type) {
+	case string:
+		value = v
+	case rangeFormatter:
+		value = v.RangeString()
+	default:
+		value = fmt.Sprint(v)
+	}
+
+	if qf.name == "" {
+		return value
+	}
+	return fmt.Sprint(qf.name, ":", value)
+}
+
+// Example of formating a query:
+// QueryWith(
+//	Filename("kustomization.yaml"),
+//	Filesize(RangeWithin{64, 192}),
+//	Keyword("copyright"),
+//	Keyword("2019"),
+// ).String()
+//
+// Outputs "q=filename:kustomization.yaml+size:64..192+copyright+2018" which
+// would search for files that have [64, 192] bytes (inclusive range) and that
+// contain the keywords 'copyright' and '2019' somewhere in the file.
+type Query []queryField
+
+func QueryWith(qfs ...queryField) Query {
+	return Query(qfs)
+}
+
+func (q Query) String() string {
+	strs := make([]string, 0, len(q))
+	for _, elem := range q {
+		str := elem.String()
+		if str == "" {
+			continue
+		}
+		strs = append(strs, str)
+	}
+
+	query := strings.Join(strs, "+")
+	if query == "" {
+		return query
+	}
+	return "q=" + query
+}
+
+// Keyword takes a single word, and formats it according to the Github API.
+func Keyword(k string) queryField {
+	return queryField{value: k}
+}
+
+// Filesize takes a rangeFormatter and formats it according to the Github API.
+func Filesize(r rangeFormatter) queryField {
+	return queryField{name: "size", value: r}
+}
+
+// Filename takes a filename and formats it according to the Github API.
+func Filename(f string) queryField {
+	return queryField{name: "filename", value: f}
+}
+
+// Path takes a filepath and formats it according to the Github API.
+func Path(p string) queryField {
+	return queryField{name: "path", value: p}
+}
+
+// RequestConfig stores common variables that must be present for the queries.
+// - CodeSearchRequests: ask Github to check the code indices given a query.
+// - ContentsRequests: ask Github where to download a resource given a repo and a
+// file path.
+// - CommitsRequests: asks Github to list commits made one a file. Useful to
+// determine the date of a file.
+type RequestConfig struct {
+	perPage     uint64
+	accessToken string
+}
+
+func NewRequestConfig(perPage uint64, accessToken string) RequestConfig {
+	return RequestConfig{
+		perPage:     perPage,
+		accessToken: accessToken,
+	}
+}
+
+// CodeSearchRequestWith given a list of query parameters that specify the
+// (patial) query, returns a request object with the (parital) query. Must call
+// the URL method to get the string value of the URL. See request.CopyWith, to
+// understand why the request object is useful.
+func (rc RequestConfig) CodeSearchRequestWith(query Query) request {
+	req := rc.makeRequest("search/code", query)
+	req.vals.Set("sort", "indexed")
+	req.vals.Set("order", "desc")
+	return req
+}
+
+// ContentsRequest given the repo name, and the filepath returns a formatted
+// query for the Github API to find the dowload information of this filepath.
+func (rc RequestConfig) ContentsRequest(fullRepoName, path string) string {
+	uri := fmt.Sprintf("repos/%s/contents/%s", fullRepoName, path)
+	return rc.makeRequest(uri, Query{}).URL()
+}
+
+func (rc RequestConfig) ReposRequest(fullRepoName string) string {
+	uri := fmt.Sprintf("repos/%s", fullRepoName)
+	return rc.makeRequest(uri, Query{}).URL()
+}
+
+// CommitsRequest given the repo name, and a filepath returns a formatted query
+// for the Github API to find the commits that affect this file.
+func (rc RequestConfig) CommitsRequest(fullRepoName, path string) string {
+	uri := fmt.Sprintf("repos/%s/commits", fullRepoName)
+	return rc.makeRequest(uri, Query{Path(path)}).URL()
+}
+
+func (rc RequestConfig) makeRequest(path string, query Query) request {
+	vals := url.Values{}
+	if rc.accessToken != "" {
+		vals.Set(accessTokenArg, rc.accessToken)
+	}
+	vals.Set(perPageArg, fmt.Sprint(rc.perPage))
+
+	return request{
+		url: url.URL{
+			Scheme: "https",
+			Host:   "api.github.com",
+			Path:   path,
+		},
+		vals:  vals,
+		query: query,
+	}
+}
+
+type request struct {
+	url   url.URL
+	vals  url.Values
+	query Query
+}
+
+// CopyWith copies the requests and adds the extra query parameters. Usefull
+// for dynamically adding sizes to a filename only query without modifying it.
+func (r request) CopyWith(queryParams ...queryField) request {
+	cpy := r
+	cpy.query = append(cpy.query, queryParams...)
+	return cpy
+}
+
+// URL encodes the variables and the URL representation into a string.
+func (r request) URL() string {
+	// Github does not handle URL encoding properly in its API for the
+	// q='...', so the query parameter is added without any encoding
+	// manually.
+	encoded := r.vals.Encode()
+	query := r.query.String()
+	sep := "&"
+	if query == "" {
+		sep = ""
+	}
+	if encoded == "" && query != "" {
+		sep = "?"
+	}
+	r.url.RawQuery = encoded + sep + query
+	return r.url.String()
+}
+
+// Allows to define a range of numbers and print it in the github range
+// query format https://help.github.com/en/articles/understanding-the-search-syntax.
+type rangeFormatter interface {
+	RangeString() string
+}
+
+// RangeLessThan is a range of values strictly less than (<) size.
+type RangeLessThan struct {
+	size uint64
+}
+
+func (r RangeLessThan) RangeString() string {
+	return fmt.Sprintf("<%d", r.size)
+}
+
+// RangeLessThan is a range of values strictly greater than (>) size.
+type RangeGreaterThan struct {
+	size uint64
+}
+
+func (r RangeGreaterThan) RangeString() string {
+	return fmt.Sprintf(">%d", r.size)
+}
+
+// RangeWithin is an inclusive range from start to end.
+type RangeWithin struct {
+	start uint64
+	end   uint64
+}
+
+func (r RangeWithin) RangeString() string {
+	return fmt.Sprintf("%d..%d", r.start, r.end)
+}

internal/tools/crawler/github/queries_test.go

@@ -0,0 +1,119 @@
+package github
+
+import (
+	"testing"
+)
+
+func TestQueryFields(t *testing.T) {
+	testCases := []struct {
+		formatter queryField
+		expected  string
+	}{
+		{
+			formatter: Keyword("keyword"),
+			expected:  "keyword",
+		},
+		{
+			formatter: Filesize(RangeLessThan{23}),
+			expected:  "size:<23",
+		},
+		{
+			formatter: Filesize(RangeWithin{24, 64}),
+			expected:  "size:24..64",
+		},
+		{
+			formatter: Filesize(RangeGreaterThan{64}),
+			expected:  "size:>64",
+		},
+		{
+			formatter: Path("some/path/to/file"),
+			expected:  "path:some/path/to/file",
+		},
+		{
+			formatter: Filename("kustomization.yaml"),
+			expected:  "filename:kustomization.yaml",
+		},
+	}
+
+	for _, test := range testCases {
+		if result := test.formatter.String(); result != test.expected {
+			t.Errorf("got (%#v = %s), expected %s", test.formatter, result, test.expected)
+		}
+	}
+}
+
+func TestQueryType(t *testing.T) {
+	testCases := []struct {
+		query    Query
+		expected string
+	}{
+		{
+			query: QueryWith(
+				Filesize(RangeWithin{24, 64}),
+				Filename("kustomization.yaml"),
+				Keyword("keyword1"),
+				Keyword("keyword2"),
+			),
+			expected: "q=size:24..64+filename:kustomization.yaml+keyword1+keyword2",
+		},
+	}
+
+	for _, test := range testCases {
+		if queryStr := test.query.String(); queryStr != test.expected {
+			t.Errorf("got (%#v = %s), expected %s", test.query, queryStr, test.expected)
+		}
+
+	}
+}
+
+func TestGithubSearchQuery(t *testing.T) {
+	const (
+		accessToken = "random_token"
+		perPage     = 100
+	)
+
+	testCases := []struct {
+		rc                    RequestConfig
+		codeQuery             Query
+		fullRepoName          string
+		path                  string
+		expectedCodeQuery     string
+		expectedContentsQuery string
+		expectedCommitsQuery  string
+	}{
+		{
+			rc: RequestConfig{
+				perPage:     perPage,
+				accessToken: accessToken,
+			},
+			codeQuery: Query{
+				Filename("kustomization.yaml"),
+				Filesize(RangeWithin{64, 128}),
+			},
+			fullRepoName: "kubernetes-sigs/kustomize",
+			path:         "examples/helloWorld/kustomization.yaml",
+
+			expectedCodeQuery: "https://api.github.com/search/code?" +
+				"access_token=random_token&order=desc&per_page=100&sort=indexed&q=filename:kustomization.yaml+size:64..128",
+
+			expectedContentsQuery: "https://api.github.com/repos/kubernetes-sigs/kustomize/contents/" +
+				"examples/helloWorld/kustomization.yaml?access_token=random_token&per_page=100",
+
+			expectedCommitsQuery: "https://api.github.com/repos/kubernetes-sigs/kustomize/commits?" +
+				"access_token=random_token&per_page=100&q=path:examples/helloWorld/kustomization.yaml",
+		},
+	}
+
+	for _, test := range testCases {
+		if result := test.rc.CodeSearchRequestWith(test.codeQuery).URL(); result != test.expectedCodeQuery {
+			t.Errorf("Got code query: %s, expected %s", result, test.expectedCodeQuery)
+		}
+
+		if result := test.rc.ContentsRequest(test.fullRepoName, test.path); result != test.expectedContentsQuery {
+			t.Errorf("Got contents query: %s, expected %s", result, test.expectedContentsQuery)
+		}
+		if result := test.rc.CommitsRequest(test.fullRepoName, test.path); result != test.expectedCommitsQuery {
+			t.Errorf("Got commits query: %s, expected %s", result, test.expectedCommitsQuery)
+		}
+	}
+}

internal/tools/crawler/github/split_search_ranges.go

@@ -0,0 +1,378 @@
+package github
+
+// GitHub only returns at most 1000 results per search query,
+// this is problematic if you want to retrieve all the results for a given
+// search query. However, GitHub allows you to specify as much as you want per
+// query to make things more specific. Specifically for files, GitHub allows
+// you to specify their sizes with range queries. This is very convenient
+// since it allows us to split the search into disjoint sets/shards of results
+// from the different file size ranges.
+//
+// Some important factors to consider:
+//
+// -  These queries are rate limited by the API to roughly once query every two
+//    seconds.
+//
+// -  The search space for file sizes is in bytes, from 0B to < 512KiB (this is
+//    a huge search space that cannot be probed linearly in a timely manner if
+//    granularity is to be expected).
+//
+// -  If you have K files there will likely be ~K/1000 sets that you have find
+//    from this search space in order to get all of the results.
+//
+// -  If you have O(K) sets it is unlikely that they are all of the same size,
+//    since (most files are power law distributed). That means that the range
+//    might be significantly smaller for 1000 small files, than it is for
+//    1000 large files.
+//
+// -  This method is a best effort approach. There are some limitations to what
+//    it can and can't do, so please note the following:
+//
+//    +  There may very well be a filesize that has more than 1000 results.
+//       this method cannot help in this case. However, requerying over time
+//       (days/weeks/months) while sorting by last indexed values may be
+//       sufficient to eventually get all of the results.
+//
+//    +  It's possible that the github API returns inconsistent counts. This
+//       is problematic in most cases, since it can cause many issues if the
+//       case is not handled properly. For instance, if you requested the
+//       number of files of an interval from size:0..64 and get that there
+//       are 900 results, you may query at size:0..96 and get that there
+//       are 800 results. To guarantee that this approach completes and does
+//       not get into a query loop over the same intervals, it will retry a few
+//       times and take the largest of the results or the largest previously
+//       queried value from another range (in this case, the implementation
+//       could decide that size:0..96 must have 900) results. This makes the
+//       approach best effort even if there are no single file sizes of over
+//       1000 results.
+//
+//
+// The approach that was taken to solve this problem is the following:
+//
+// 1. Determine the total number of results by querying from the lower bound
+//    to the upper bound (size:0..max). If there are less than 1000 files,
+//    return a single range of values (size:0..max) since all results can be
+//    retrieved.
+//
+// 2. Otherwise, set a target number of files to be 1000.
+//
+// 3. Binary search for the range from 0..r that provides a file count that is
+//    less than or equal to the target. Once this value is found, store the
+//    upper bound of range (r). If r is the same as the previous value, (or 0)
+//    increase r by one (this guarantees progress, but will miss out on some
+//    results).
+//
+// 4. Increase the target by 1000.
+//
+// 5. Repeat steps 3 and 4 until the target is at or exceeds the total number
+//    of files.
+//
+//
+// In general there are other ways to get all of the files from GitHub. In
+// some cases it would be sufficient to just get the files that are being
+// updated/indexed by github periodically to update the corpus, so this
+// complicated approach does not have to be run every time. However, for
+// some searches, there may be too many results on a time interval to do
+// this simple update search limited to only 1000 results.
+//
+// There is also a more sophisticated approach that may yield better
+// performance:
+// -  Perform this search once and create a prior distribution of file sizes.
+//    Each time you want to retrieve the results of the query, scale the
+//    prior of expected ranges to the current number of files. From each
+//    expected range of 1000 files, perform a exponential search to find the
+//    lower bound of the range. This would likely reduce the total number
+//    of queries by a significant amount since it would only have to search
+//    for a small set of values around each likely range boundary.
+//
+// However, actually retrieving the files will be the bottleneck operation
+// since the number of queries to find the ranges will be close to:
+//   log2(maxFileSize) * totalResults / 1000 ~= totalResults / 50
+// whereas the number of queries to actually get all of the search results
+// are close to:
+//   apiCallsPerResult * 10(pages) * 100(resultsPerPage) * totalResults / 1000
+//   = apiCallsPerResult * totalResults.
+//
+// So it could very well take apiCallsPerResult * 50 times longer to acutally
+// fetch the results (assuming the quotas for the API calls are the same as the
+// search API), than it does to perform these range searches.
+
+import (
+	"fmt"
+	"math/bits"
+)
+
+// Files cannot be more than 2^19 bytes, according to
+// https://help.github.com/en/articles/searching-code#considerations-for-code-search
+const (
+	githubMaxFileSize        = uint64(1 << 19)
+	githubMaxResultsPerQuery = uint64(1000)
+)
+
+// Interface instead of struct for testing purposes.
+// Not expecting to have multiple implementations.
+type cachedSearch interface {
+	CountResults(uint64) (uint64, error)
+	RequestString(filesize rangeFormatter) string
+}
+
+// cachedSearch is a simple data structure that maps the upper bound (r) of a
+// range from 0 to r to the number of files that have between 0 and r files
+// (inclusive). It also guarantees that the counts are monotonically increasing
+// (not strict) as the value for r increases, by looking at the maximal
+// previous file count for the value that precedes r in the cache.
+//
+// It uses a bit trick to be more efficient in detecting
+// inconsistencies in the returned data from the Github API.
+// Therefore, the cache expects a search to always start at 0, and
+// it expects the max file size to be a power of 2. If this is to be changed
+// there are a few considerations to keep in mind:
+//
+// 1. The cache is only efficient if the queries can be reused, so if
+//    the first chunk of files lives in the range 0..x, continuing the
+//    search for the next chunk from x+1..max (while asymptotically sane)
+//    may actually be less efficient since the cache is essentially reset
+//    at every interval. This leads to a larger number of requests in
+//    practice, and requests are what's expensive (rate limits).
+//
+// 2. The github API is not perfectly monotonic.. (this is somewhat
+//    problematic). The current cache implementation looks at the
+//    predecessor entry to find out if the current value is monotonic.
+//    This is where the bit trick is used, since each step in the binary
+//    search is adding or ommiting to add a decreasing power of 2 to the query
+//    value, we can remove the least significant set bit to find the
+//    predecessor in constant time. Ultimately since the search is rate
+//    limited, we could also easily afford to compute this in linear time
+//    by iterating over cached values. So this trick is not crucial to the
+//    cache's performance.
+type githubCachedSearch struct {
+	cache       map[uint64]uint64
+	gcl         GitHubClient
+	baseRequest request
+}
+
+func newCache(client GitHubClient, query Query) githubCachedSearch {
+	return githubCachedSearch{
+		cache: map[uint64]uint64{
+			0: 0,
+		},
+		gcl:         client,
+		baseRequest: client.CodeSearchRequestWith(query),
+	}
+}
+
+func (c githubCachedSearch) CountResults(upperBound uint64) (uint64, error) {
+	count, cached := c.cache[upperBound]
+	if cached {
+		return count, nil
+	}
+
+	sizeRange := RangeWithin{0, upperBound}
+	rangeRequest := c.RequestString(sizeRange)
+
+	result := c.gcl.parseGithubResponse(rangeRequest)
+	if result.Error != nil {
+		return count, result.Error
+	}
+
+	// As range search uses powers of 2 for binary search, the previously
+	// cached value is easy to find by removing the least significant set
+	// bit from the current upperBound, since each step of the search adds
+	// least significant set bit.
+	//
+	// Finding the predecessor could also be implemented by iterating over
+	// the map to find the largest key that is smaller than upperBound if
+	// this approach deemed too complex.
+	trail := bits.TrailingZeros64(upperBound)
+	prev := uint64(0)
+	if trail != 64 {
+		prev = upperBound - (1 << uint64(trail))
+	}
+
+	// Sometimes the github API is not monotonically increasing, or ouputs
+	// an erroneous value of 0, or 1. This logic makes sure that it was not
+	// erroneous, and that the sequence continues to be monotonic by setting
+	// the current query count to match the previous value. which at least
+	// guarantees that the range search terminates.
+	//
+	// On the other hand, if files are added, then we way loose out on some
+	// files in a reviously completed range, but these files should be there
+	// the next time the crawler runs, so this is not really problematic.
+	retryMonotonicCount := 4
+	for result.Parsed.TotalCount < c.cache[prev] {
+		logger.Printf(
+			"Retrying query... current lower bound: %d, got: %d\n",
+			c.cache[prev], result.Parsed.TotalCount)
+
+		result = c.gcl.parseGithubResponse(rangeRequest)
+		if result.Error != nil {
+			return count, result.Error
+		}
+
+		retryMonotonicCount--
+		if retryMonotonicCount <= 0 {
+			result.Parsed.TotalCount = c.cache[prev]
+			logger.Println(
+				"Retries for monotonic check exceeded,",
+				" setting value to match predecessor")
+		}
+	}
+
+	count = result.Parsed.TotalCount
+	logger.Printf("Caching new query %s, with count %d\n",
+		sizeRange.RangeString(), count)
+	c.cache[upperBound] = count
+	return count, nil
+}
+
+func (c githubCachedSearch) RequestString(filesize rangeFormatter) string {
+	return c.baseRequest.CopyWith(Filesize(filesize)).URL()
+}
+
+// Outputs a (possibly incomplete) list of ranges to query to find most search
+// results as permissible by the search github search API. Github search only
+// allows 1,000 results per query (paginated).
+// Source: https://developer.github.com/v3/search/
+//
+// This leaves the possibility of having file sizes with more than 1000 results,
+// This would mean that the search as it is could not find all files. If queries
+// are sorted by last indexed, and retrieved on regular intervals, it should be
+// sufficient to get most if not all documents.
+func FindRangesForRepoSearch(cache cachedSearch) ([]string, error) {
+	totalFiles, err := cache.CountResults(githubMaxFileSize)
+	if err != nil {
+		return nil, err
+	}
+	logger.Println("total files: ", totalFiles)
+
+	if githubMaxResultsPerQuery >= totalFiles {
+		return []string{
+			cache.RequestString(RangeWithin{0, githubMaxFileSize}),
+		}, nil
+	}
+
+	// Find all the ranges of file sizes such that all files are queryable
+	// using the Github API. This does not compute an optimal ranges, since
+	// the number of queries needed to get the information required to
+	// compute an optimal range is expected to be much larger than the
+	// number of queries performed this way.
+	//
+	// The number of ranges is k = (number of files)/1000, and finding a
+	// range is logarithmic in the max file size (n = filesize). This means
+	// that preprocessing takes O(k * lg n) queries to find the ranges with
+	// a binary search over file sizes.
+	//
+	// My intuition is that this approach is competitive to a perfectly
+	// optimal solution, but I didn't actually take the time to do a
+	// rigorous proof. Intuitively, since files sizes are typically power
+	// law distibuted the binary search will be very skewed towards the
+	// smaller file ranges. This means that in practice this approach will
+	// make fewer than (#files/1000)*(log(n) = 19) queries for
+	// preprocessing, since it reuses a lot of the queries in the denser
+	// ranges. Furthermore, because of the distribution, it should be very
+	// easy to find ranges that are very close to the upper bound, up to
+	// the limiting factor of having no more than 1000 files accessible per
+	// range.
+	filesAccessible := uint64(0)
+	sizes := make([]uint64, 0)
+	for filesAccessible < totalFiles {
+		target := filesAccessible + githubMaxResultsPerQuery
+		if target >= totalFiles {
+			break
+		}
+
+		logger.Printf("%d accessible files, next target = %d\n",
+			filesAccessible, target)
+
+		cur, err := lowerBoundFileCount(cache, target)
+		if err != nil {
+			return nil, err
+		}
+
+		// If there are more than 1000 files in the next bucket, we must
+		// advance anyway and lose out on some files :(.
+		if l := len(sizes); l > 0 && sizes[l-1] == cur {
+			cur++
+		}
+
+		nextAccessible, err := cache.CountResults(cur)
+		if err != nil {
+			return nil, fmt.Errorf(
+				"cache should be populated at %d already, got %v",
+				cur, err)
+		}
+		if nextAccessible < filesAccessible {
+			return nil, fmt.Errorf(
+				"Number of results dropped from %d to %d within range search",
+				filesAccessible, nextAccessible)
+		}
+
+		filesAccessible = nextAccessible
+		if nextAccessible < totalFiles {
+			sizes = append(sizes, cur)
+		}
+	}
+
+	return formatFilesizeRanges(cache, sizes), nil
+}
+
+// lowerBoundFileCount finds the filesize range from [0, return value] that has
+// the largest file count that is smaller than or equal to
+// githubMaxResultsPerQuery. It is important to note that this returned value
+// could already be in a previous range if the next file size has more than 1000
+// results. It is left to the caller to handle this bit of logic and guarantee
+// forward progession in this case.
+func lowerBoundFileCount(
+	cache cachedSearch, targetFileCount uint64) (uint64, error) {
+
+	// Binary search for file sizes that make up the next <=1000 element
+	// chunk.
+	cur := uint64(0)
+	increase := githubMaxFileSize / 2
+
+	for increase > 0 {
+		mid := cur + increase
+
+		count, err := cache.CountResults(mid)
+		if err != nil {
+			return count, err
+		}
+
+		if count <= targetFileCount {
+			cur = mid
+		}
+
+		if count == targetFileCount {
+			break
+		}
+
+		increase /= 2
+	}
+
+	return cur, nil
+}
+
+func formatFilesizeRanges(cache cachedSearch, sizes []uint64) []string {
+	ranges := make([]string, 0, len(sizes)+1)
+
+	if len(sizes) > 0 {
+		ranges = append(ranges, cache.RequestString(
+			RangeLessThan{sizes[0] + 1},
+		))
+	}
+
+	for i := 0; i < len(sizes)-1; i += 1 {
+		ranges = append(ranges, cache.RequestString(
+			RangeWithin{sizes[i] + 1, sizes[i+1]},
+		))
+
+		if i != len(sizes)-2 {
+			continue
+		}
+		ranges = append(ranges, cache.RequestString(
+			RangeGreaterThan{sizes[i+1]},
+		))
+	}
+
+	return ranges
+}

internal/tools/crawler/github/split_search_ranges_test.go

@@ -0,0 +1,90 @@
+package github
+
+import (
+	"fmt"
+	"reflect"
+	"testing"
+)
+
+type testCachedSearch struct {
+	cache map[uint64]uint64
+}
+
+func (c testCachedSearch) CountResults(upperBound uint64) (uint64, error) {
+	fmt.Printf("CountResults(%05x)\n", upperBound)
+	count, ok := c.cache[upperBound]
+	if !ok {
+		return count, fmt.Errorf("cache not set at %x", upperBound)
+	}
+	return count, nil
+}
+
+func (c testCachedSearch) RequestString(filesize rangeFormatter) string {
+	return filesize.RangeString()
+}
+
+// TODO(damienr74) make tests easier to write.. I'm thinking I can make the test
+// cache take in a list of (filesize, count) pairs and it can populate the cache
+// without relying on how the implementation will create queries. This was only
+// a quick and dirty test to make sure that modifications are not going to break
+// the functionality.
+func TestRangeSplitting(t *testing.T) {
+	// Keys follow the binary search depending on whether or not the range
+	// is too small/large to find close to optimal filesize ranges. This
+	// test is heavily tied to the fact that the search is using powers of two
+	// to make progress in the search (hence the use of hexadecimal values).
+	cache := testCachedSearch{
+		map[uint64]uint64{
+			0x80000: 5000,
+			0x40000: 5000,
+			0x20000: 5000,
+			0x10000: 5000,
+			0x08000: 5000,
+			0x04000: 5000,
+			0x02000: 5000,
+			0x01000: 5000,
+			0x00fff: 3950,
+			0x00ffe: 3950,
+			0x00ffc: 3950,
+			0x00ff8: 3950,
+			0x00ff0: 3950,
+			0x00fe0: 3950,
+			0x00fc0: 3950,
+			0x00f80: 3950,
+			0x00f00: 3950,
+			0x00e00: 3950,
+			0x00c00: 3950,
+			0x00800: 3950,
+			0x00400: 3950,
+			0x00200: 3688,
+			0x00180: 3028,
+			0x00100: 2999,
+			0x000c0: 2448,
+			0x00080: 1999,
+			0x00070: 1600,
+			0x0006c: 1003,
+			0x0006b: 1001,
+			0x0006a: 999,
+			0x00068: 999,
+			0x00060: 999,
+			0x00040: 999,
+			0x00000: 0,
+		},
+	}
+
+	requests, err := FindRangesForRepoSearch(cache)
+	if err != nil {
+		t.Errorf("Error while finding ranges: %v", err)
+	}
+	expected := []string{
+		"<107",      // cache.RequestString(RangeLessThan{0x6b}),
+		"107..128",  // cache.RequestString(RangeWithin{0x6b, 0x80}),
+		"129..256",  // cache.RequestString(RangeWithin{0x81, 0x100}),
+		"257..4095", // cache.RequestString(RangeWithin{0x101, 0xfff}),
+		">4095",     // cache.RequestString(RangeGreaterThan{0xfff}),
+	}
+
+	if !reflect.DeepEqual(requests, expected) {
+		t.Errorf("Expected requests (%v) to equal (%v)", requests, expected)
+	}
+}

internal/tools/doc/doc.go

@@ -0,0 +1,202 @@
+package doc
+
+import (
+	"fmt"
+	"strings"
+
+	"sigs.k8s.io/kustomize/v3/k8sdeps/kunstruct"
+	"sigs.k8s.io/kustomize/v3/pkg/ifc"
+	"sigs.k8s.io/kustomize/v3/pkg/pgmconfig"
+	"sigs.k8s.io/kustomize/v3/pkg/types"
+	"sigs.k8s.io/yaml"
+)
+
+var fileReader ifc.KunstructuredFactory = kunstruct.NewKunstructuredFactoryImpl()
+
+// This document is meant to be used at the elasticsearch document type.
+// Fields are serialized as-is to elasticsearch, where indices are built
+// to facilitate text search queries. Identifiers, Values, FilePath,
+// RepositoryURL and DocumentData are meant to be searched for text queries
+// directly, while the other fields can either be used as a filter, or as
+// additional metadata displayed in the UI.
+//
+// The fields of the document and their purpose are listed below:
+// - DocumentData contains the contents of the kustomization file.
+// - Kinds Represents the kubernetes Kinds that are in this file.
+// - Identifiers are a list of (partial and full) identifier paths that can be
+//   found by users. Each part of a path is delimited by ":" e.g. spec:replicas.
+// - Values are a list of identifier paths and their values that can be found by
+//   search queries. The path is delimited by ":" and the value follows the "="
+//   symbol e.g. spec:replicas=4.
+// - FilePath is the path of the file.
+// - RepositoryURL is the URL of the source repository.
+// - CreationTime is the time at which the file was created.
+//
+// Representing each Identifier and Value as a flat string representation
+// facilitates the use of complex text search features from elasticsearch such
+// as fuzzy searching, regex, wildcards, etc.
+type KustomizationDocument struct {
+	Document
+	Kinds       []string `json:"kinds,omitempty"`
+	Identifiers []string `json:"identifiers,omitempty"`
+	Values      []string `json:"values,omitempty"`
+}
+
+type set map[string]struct{}
+
+// Implements the CrawlerDocument interface.
+func (doc *KustomizationDocument) GetResources() ([]*Document, error) {
+	isResource := true
+	for _, suffix := range pgmconfig.KustomizationFileNames {
+		if strings.HasSuffix(doc.FilePath, "/"+suffix) {
+			isResource = false
+		}
+	}
+	if isResource {
+		return []*Document{}, nil
+	}
+
+	content := []byte(doc.DocumentData)
+	content, err := FixKustomizationPreUnmarshallingNonFatal(content)
+	if err != nil {
+		return nil, fmt.Errorf("could not fix kustomize file: %v", err)
+	}
+
+	var k types.Kustomization
+	err = yaml.Unmarshal(content, &k)
+	if err != nil {
+		return nil, fmt.Errorf(
+			"could not parse kustomization: %v", err)
+	}
+	k.FixKustomizationPostUnmarshalling()
+
+	res := make([]*Document, 0, len(k.Resources))
+	for _, r := range k.Resources {
+		next, err := doc.Document.FromRelativePath(r)
+		if err != nil {
+			fmt.Printf("GetResources error: %v\n", err)
+			continue
+		}
+		res = append(res, &next)
+	}
+
+	return res, nil
+}
+
+func (doc *KustomizationDocument) readBytes() ([]map[string]interface{}, error) {
+	data := []byte(doc.DocumentData)
+
+	for _, suffix := range pgmconfig.KustomizationFileNames {
+		if !strings.HasSuffix(doc.FilePath, "/"+suffix) {
+			continue
+		}
+		var config map[string]interface{}
+		err := yaml.Unmarshal(data, &config)
+		if err != nil {
+			return nil, fmt.Errorf(
+				"unable to parse kustomization: %v", err)
+		}
+		return []map[string]interface{}{config}, nil
+	}
+
+	configs := make([]map[string]interface{}, 0)
+	ks, err := fileReader.SliceFromBytes(data)
+	if err != nil {
+		return nil, fmt.Errorf("unable to parse resource: %v", err)
+	}
+	for _, k := range ks {
+		configs = append(configs, k.Map())
+	}
+
+	return configs, nil
+}
+
+func (doc *KustomizationDocument) ParseYAML() error {
+	doc.Identifiers = make([]string, 0)
+	doc.Values = make([]string, 0)
+	doc.Kinds = make([]string, 0, 1)
+
+	identifierSet := make(set)
+	valueSet := make(set)
+	getKind := func(m map[string]interface{}) string {
+		const defaultStr = "Kustomization"
+		kind, ok := m["kind"]
+		if !ok {
+			return defaultStr
+		}
+		if str, ok := kind.(string); ok && str != "" {
+			return str
+		}
+		return defaultStr
+	}
+
+	ks, err := doc.readBytes()
+	if err != nil {
+		return err
+	}
+
+	for _, contents := range ks {
+		doc.Kinds = append(doc.Kinds, getKind(contents))
+		createFlatStructure(identifierSet, valueSet, contents)
+	}
+
+	for val := range valueSet {
+		doc.Values = append(doc.Values, val)
+	}
+
+	for key := range identifierSet {
+		doc.Identifiers = append(doc.Identifiers, key)
+	}
+
+	return nil
+}
+
+func createFlatStructure(identifierSet set, valueSet set, contents map[string]interface{}) {
+	type Map struct {
+		data   map[string]interface{}
+		prefix string
+	}
+
+	toVisit := []Map{
+		{
+			data:   contents,
+			prefix: "",
+		},
+	}
+
+	for i := 0; i < len(toVisit); i++ {
+		visiting := toVisit[i]
+		for k, v := range visiting.data {
+			identifier := fmt.Sprintf("%s:%s", visiting.prefix, k)
+			// noop after the first iteration.
+			identifier = strings.TrimLeft(identifier, ":")
+
+			// Recursive function traverses structure to find
+			// identifiers and values. These later get formatted
+			// into doc.Identifiers and doc.Values respectively.
+			var traverseStructure func(interface{})
+			traverseStructure = func(arg interface{}) {
+				switch value := arg.(type) {
+				case map[string]interface{}:
+					toVisit = append(toVisit, Map{
+						data:   value,
+						prefix: identifier,
+					})
+				case []interface{}:
+					for _, val := range value {
+						traverseStructure(val)
+					}
+				case interface{}:
+					esc := fmt.Sprintf("%v", value)
+
+					valuePath := fmt.Sprintf("%s=%v",
+						identifier, esc)
+					valueSet[valuePath] = struct{}{}
+				}
+			}
+			traverseStructure(v)
+
+			identifierSet[identifier] = struct{}{}
+		}
+	}
+}

internal/tools/doc/doc_test.go

@@ -0,0 +1,254 @@
+package doc
+
+import (
+	"reflect"
+	"sort"
+	"strings"
+	"testing"
+)
+
+func TestParseYAML(t *testing.T) {
+	testCases := []struct {
+		identifiers []string
+		values      []string
+		kinds       []string
+		filepath    string
+		yaml        string
+	}{
+		{
+			identifiers: []string{
+				"namePrefix",
+				"metadata",
+				"metadata:name",
+				"kind",
+			},
+			values: []string{
+				"kind=",
+				"namePrefix=dev-",
+				"metadata:name=app",
+			},
+			kinds: []string{
+				"Kustomization",
+			},
+			filepath: "some/path/to/kustomization.yaml",
+			yaml: `
+namePrefix: dev-
+metadata:
+  name: app
+kind: ""
+`,
+		},
+		{
+			identifiers: []string{
+				"namePrefix",
+				"metadata",
+				"metadata:name",
+				"metadata:spec",
+				"metadata:spec:replicas",
+				"kind",
+				"replicas",
+				"replicas:name",
+				"replicas:count",
+				"resource",
+			},
+			values: []string{
+				"namePrefix=dev-",
+				"metadata:name=n1",
+				"metadata:spec:replicas=3",
+				"kind=Kustomization",
+				"replicas:name=n1",
+				"replicas:name=n2",
+				"replicas:count=3",
+				"resource=file1.yaml",
+				"resource=file2.yaml",
+			},
+			kinds: []string{
+				"Kustomization",
+			},
+			filepath: "./kustomization.yaml",
+			yaml: `
+namePrefix: dev-
+# map of map
+metadata:
+  name: n1
+  spec:
+    replicas: 3
+kind: Kustomization
+
+#list of map
+replicas:
+- name: n1
+  count: 3
+- name: n2
+  count: 3
+
+# list
+resource:
+- file1.yaml
+- file2.yaml
+`,
+		},
+		{
+			identifiers: []string{
+				"kind",
+				"metadata",
+				"metadata:name",
+			},
+			values: []string{
+				"kind=Deployment",
+				"kind=Service",
+				"kind=Custom",
+				"metadata:name=app",
+				"metadata:name=app-service",
+				"metadata:name=app-crd",
+			},
+			kinds: []string{
+				"Deployment",
+				"Service",
+				"Custom",
+			},
+			filepath: "resources.yaml",
+			yaml: `
+---
+kind: Deployment
+metadata:
+  name: app
+---
+kind: Service
+metadata:
+  name: app-service
+---
+kind: Custom
+metadata:
+  name: app-crd
+`,
+		},
+	}
+
+	for _, test := range testCases {
+		doc := KustomizationDocument{
+			Document: Document{
+				DocumentData: test.yaml,
+				FilePath:     test.filepath,
+			},
+		}
+
+		err := doc.ParseYAML()
+		if err != nil {
+			t.Errorf("Document error error: %s", err)
+		}
+
+		cmpStrings := func(got, expected []string, label string) {
+			sort.Strings(got)
+			sort.Strings(expected)
+
+			if !reflect.DeepEqual(got, expected) {
+				t.Errorf("Expected %s (%v) to be equal to (%v)\n",
+					label,
+					strings.Join(got, ","),
+					strings.Join(expected, ","))
+			}
+
+		}
+
+		cmpStrings(doc.Identifiers, test.identifiers, "identifiers")
+		cmpStrings(doc.Values, test.values, "values")
+		cmpStrings(doc.Kinds, test.kinds, "kinds")
+	}
+}
+
+func TestGetResources(t *testing.T) {
+	tests := []struct {
+		doc       KustomizationDocument
+		resources []*Document
+	}{
+		{
+			doc: KustomizationDocument{
+				Document: Document{
+					RepositoryURL: "sigs.k8s.io/kustomize",
+					FilePath:      "some/path/to/kdir/kustomization.yaml",
+					DocumentData: `
+bases:
+- ../base
+- ../otherbase
+
+resources:
+- file.yaml
+- https://github.com/kubernetes-sigs/kustomize/examples/helloWorld?ref=v3.1.0
+`},
+			},
+			resources: []*Document{
+				{
+					RepositoryURL: "sigs.k8s.io/kustomize",
+					FilePath:      "some/path/to/base",
+				},
+				{
+					RepositoryURL: "sigs.k8s.io/kustomize",
+					FilePath:      "some/path/to/otherbase",
+				},
+				{
+					RepositoryURL: "sigs.k8s.io/kustomize",
+					FilePath:      "some/path/to/kdir/file.yaml",
+				},
+				{
+					RepositoryURL: "https://github.com/kubernetes-sigs/kustomize",
+					FilePath:      "examples/helloWorld",
+					DefaultBranch: "v3.1.0",
+				},
+			},
+		},
+		{
+			doc: KustomizationDocument{
+				Document: Document{
+					RepositoryURL: "https://github.com/some/repo",
+					FilePath:      "some/resource.yaml",
+					DocumentData: `
+bases:
+- ../base
+- ../overlay
+
+resources:
+- https://github.com/kubernetes-sigs/kustomize/examples/helloWorld?ref=v3.1.0
+- some/file.yaml
+`,
+				},
+			},
+			resources: []*Document{},
+		},
+	}
+
+	for _, test := range tests {
+		res, err := test.doc.GetResources()
+		if err != nil {
+			t.Errorf("Unexpected error: %v\n", err)
+			continue
+		}
+		if len(test.resources) != len(res) {
+			t.Errorf("Number of resources does not match.")
+			continue
+		}
+		cmp := func(docs []*Document) func(i, j int) bool {
+			return func(i, j int) bool {
+				if docs[i].RepositoryURL != docs[j].RepositoryURL {
+					return docs[i].RepositoryURL <
+						docs[j].RepositoryURL
+				}
+
+				if docs[i].FilePath != docs[j].FilePath {
+					return docs[i].FilePath <
+						docs[j].FilePath
+				}
+
+				return docs[i].DefaultBranch < docs[j].DefaultBranch
+			}
+		}
+		sort.Slice(test.resources, cmp(test.resources))
+		sort.Slice(res, cmp(res))
+		for i, r := range test.resources {
+			if !reflect.DeepEqual(res[i], r) {
+				t.Errorf("Expected '%+v' to equal '%+v'\n",
+					res[i], r)
+			}
+		}
+	}
+}

internal/tools/doc/docname.go

@@ -0,0 +1,58 @@
+package doc
+
+import (
+	"path"
+	"time"
+
+	"sigs.k8s.io/kustomize/v3/pkg/git"
+)
+
+type Document struct {
+	RepositoryURL string     `json:"repositoryUrl,omitempty"`
+	FilePath      string     `json:"filePath,omitempty"`
+	DefaultBranch string     `json:"defaultBranch,omitempty"`
+	DocumentData  string     `json:"document,omitempty"`
+	CreationTime  *time.Time `json:"creationTime,omitempty"`
+	IsSame        bool       `json:"-"`
+}
+
+// Implements the CrawlerDocument interface.
+func (doc *Document) GetDocument() *Document {
+	return doc
+}
+
+// Implements the CrawlerDocument interface.
+func (doc *Document) WasCached() bool {
+	return doc.IsSame
+}
+
+func (doc *Document) FromRelativePath(newFile string) (Document, error) {
+	repoSpec, err := git.NewRepoSpecFromUrl(newFile)
+	if err == nil {
+		return Document{
+			RepositoryURL: repoSpec.Host + path.Clean(repoSpec.OrgRepo),
+			FilePath:      path.Clean(repoSpec.Path),
+			DefaultBranch: repoSpec.Ref,
+		}, nil
+	}
+	// else document is probably relative path.
+
+	ret := Document{
+		RepositoryURL: doc.RepositoryURL,
+		DefaultBranch: doc.DefaultBranch,
+	}
+	ogDir, _ := path.Split(doc.FilePath)
+
+	cleaned := path.Clean(newFile)
+	if !path.IsAbs(cleaned) {
+		cleaned = path.Clean(ogDir + "/" + cleaned)
+	}
+
+	ret.FilePath = cleaned
+	return ret, nil
+}
+
+func (doc *Document) ID() string {
+	return doc.RepositoryURL + "/" +
+		doc.DefaultBranch + "/" + doc.FilePath
+}

internal/tools/doc/docname_test.go

@@ -0,0 +1,64 @@
+package doc
+
+import (
+	"reflect"
+	"testing"
+)
+
+func TestFromRelativePath(t *testing.T) {
+	type Case struct {
+		RelativePath string
+		Expected     Document
+	}
+
+	testCases := []struct {
+		BaseDoc Document
+		Cases   []Case
+	}{
+		{
+			BaseDoc: Document{
+				RepositoryURL: "example.com/repo",
+				FilePath:      "path/to/file/kustomization.yaml",
+				DefaultBranch: "master",
+			},
+			Cases: []Case{
+				{
+					RelativePath: "../other/file/resource.yaml",
+					Expected: Document{
+						RepositoryURL: "example.com/repo",
+						FilePath:      "path/to/other/file/resource.yaml",
+						DefaultBranch: "master",
+					},
+				},
+				{
+					RelativePath: "../file/../../something/../to/other/file/patch.yaml",
+					Expected: Document{
+						RepositoryURL: "example.com/repo",
+						FilePath:      "path/to/other/file/patch.yaml",
+						DefaultBranch: "master",
+					},
+				},
+				{
+					RelativePath: "service.yaml",
+					Expected: Document{
+						RepositoryURL: "example.com/repo",
+						FilePath:      "path/to/file/service.yaml",
+						DefaultBranch: "master",
+					},
+				},
+			},
+		},
+	}
+
+	for _, tc := range testCases {
+		for _, c := range tc.Cases {
+			rd, err := tc.BaseDoc.FromRelativePath(c.RelativePath)
+			if err != nil {
+				t.Errorf("unexpected error: %v", err)
+			}
+			if !reflect.DeepEqual(rd, c.Expected) {
+				t.Errorf("document mismatch expected %v, got %v", c.Expected, rd)
+			}
+		}
+	}
+}

internal/tools/doc/kustomize_util.go

@@ -0,0 +1,51 @@
+package doc
+
+import (
+	"fmt"
+	"regexp"
+
+	"sigs.k8s.io/yaml"
+)
+
+func FixKustomizationPreUnmarshallingNonFatal(data []byte) ([]byte, error) {
+	deprecateFieldsMap := map[string]string{
+		"imageTags:": "images:",
+	}
+	for oldname, newname := range deprecateFieldsMap {
+		pattern := regexp.MustCompile(oldname)
+		data = pattern.ReplaceAll(data, []byte(newname))
+	}
+
+	found, err := useLegacyPatch(data)
+	if err == nil && found {
+		pattern := regexp.MustCompile("patches:")
+		data = pattern.ReplaceAll(data, []byte("patchesStrategicMerge:"))
+	}
+
+	return data, err
+}
+
+func useLegacyPatch(data []byte) (bool, error) {
+	found := false
+
+	var object map[string]interface{}
+	err := yaml.Unmarshal(data, &object)
+	if err != nil {
+		return false, fmt.Errorf("invalid content from %s",
+			string(data))
+	}
+	if rawPatches, ok := object["patches"]; ok {
+		patches, ok := rawPatches.([]interface{})
+		if !ok {
+			return false, fmt.Errorf("invalid patches from %v",
+				rawPatches)
+		}
+		for _, p := range patches {
+			_, ok := p.(string)
+			if ok {
+				found = true
+			}
+		}
+	}
+	return found, nil
+}

internal/tools/go.mod

@@ -0,0 +1,13 @@
+module sigs.k8s.io/kustomize/internal/tools
+
+go 1.12
+
+require (
+	github.com/elastic/go-elasticsearch/v6 v6.8.2
+	github.com/gomodule/redigo v2.0.0+incompatible
+	github.com/gorilla/mux v1.7.3
+	github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79
+	github.com/rs/cors v1.7.0
+	sigs.k8s.io/kustomize/v3 v3.1.1-0.20190826160027-84519c236bac
+	sigs.k8s.io/yaml v1.1.0
+)

internal/tools/go.sum

@@ -0,0 +1,146 @@
+github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ=
+github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
+github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
+github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
+github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
+github.com/davecgh/go-spew v0.0.0-20151105211317-5215b55f46b2/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/elastic/go-elasticsearch/v6 v6.8.2 h1:rp5DGrd63V5c6nHLjF6QEXUpZSvs0+QM3ld7m9VhV2g=
+github.com/elastic/go-elasticsearch/v6 v6.8.2/go.mod h1:UwaDJsD3rWLM5rKNFzv9hgox93HoX8utj1kxD9aFUcI=
+github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
+github.com/emicklei/go-restful v2.9.6+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
+github.com/evanphx/json-patch v4.5.0+incompatible h1:ouOWdg56aJriqS0huScTkVXPC5IcNrDCXZ6OoTAWu7M=
+github.com/evanphx/json-patch v4.5.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
+github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
+github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
+github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
+github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+35s3my2LFTysnkMfxsJBAMHj/DoqoB9knIWoYG/Vk0=
+github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg=
+github.com/go-openapi/jsonreference v0.0.0-20160704190145-13c6e3589ad9/go.mod h1:W3Z9FmVs9qj+KR4zFKmDPGiLdk1D9Rlm7cyMvf57TTg=
+github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc=
+github.com/go-openapi/spec v0.0.0-20160808142527-6aced65f8501/go.mod h1:J8+jY1nAiCcj+friV/PDoE1/3eeccG9LYBs0tYvLOWc=
+github.com/go-openapi/spec v0.19.2/go.mod h1:sCxk3jxKgioEJikev4fgkNmwS+3kuYdJtcsZsD5zxMY=
+github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dpr1UfpPtxFw+EFuQ41HhCWZfha5jSVRG7C7I=
+github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
+github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE=
+github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
+github.com/golang/protobuf v0.0.0-20161109072736-4bd1920723d7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg=
+github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/gomodule/redigo v2.0.0+incompatible h1:K/R+8tc58AaqLkqG2Ol3Qk+DR/TlNuhuh457pBFPtt0=
+github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4=
+github.com/google/gofuzz v0.0.0-20161122191042-44d81051d367/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI=
+github.com/google/gofuzz v1.0.0 h1:A8PeW59pxE9IoFRqBp37U+mSNaQoZ46F1f0f863XSXw=
+github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/googleapis/gnostic v0.0.0-20170426233943-68f4ded48ba9/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY=
+github.com/googleapis/gnostic v0.3.0 h1:CcQijm0XKekKjP/YCz28LXVSpgguuB+nCxaSjCe09y0=
+github.com/googleapis/gnostic v0.3.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY=
+github.com/gorilla/mux v1.7.3 h1:gnP5JzjVOuiZD07fKKToCAOjS0yOpj/qPETTXCCS6hw=
+github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
+github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 h1:+ngKgrYPPJrOjhax5N+uePQ0Fh1Z7PheYoUI/0nzkPA=
+github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
+github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
+github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
+github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
+github.com/json-iterator/go v0.0.0-20180612202835-f2b4162afba3/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
+github.com/json-iterator/go v1.1.6 h1:MrUvLMLTMxbqFJ9kzlvat/rYZqZnW3u4wkLzWTaFwKs=
+github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
+github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
+github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA=
+github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
+github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
+github.com/mailru/easyjson v0.0.0-20190620125010-da37f6c1e481/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/reflect2 v0.0.0-20180320133207-05fbef0ca5da/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI=
+github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
+github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.8.0 h1:VkHVNpR4iVnU8XQR6DBm8BqYjN7CRzw+xKUbVVbbW9w=
+github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
+github.com/onsi/gomega v1.5.0 h1:izbySO9zDPmjJ8rDjLvkA2zJHIo+HkYXHnf7eN7SSyo=
+github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik=
+github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU=
+github.com/spf13/cobra v0.0.2/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
+github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
+github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg=
+github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
+github.com/stretchr/testify v0.0.0-20151208002404-e3a8ff8ce365/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190621203818-d432491b9138 h1:t8BZD9RDjkm9/h7yYN6kE8oaeov5r9aztkB7zKA5Tkg=
+golang.org/x/sys v0.0.0-20190621203818-d432491b9138/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
+golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20181011042414-1f849cf54d09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
+gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
+gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
+gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
+gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+k8s.io/api v0.0.0-20190313235455-40a48860b5ab h1:DG9A67baNpoeweOy2spF1OWHhnVY5KR7/Ek/+U1lVZc=
+k8s.io/api v0.0.0-20190313235455-40a48860b5ab/go.mod h1:iuAfoD4hCxJ8Onx9kaTIt30j7jUFS00AXQi6QMi99vA=
+k8s.io/apimachinery v0.0.0-20190313205120-d7deff9243b1 h1:IS7K02iBkQXpCeieSiyJjGoLSdVOv2DbPaWHJ+ZtgKg=
+k8s.io/apimachinery v0.0.0-20190313205120-d7deff9243b1/go.mod h1:ccL7Eh7zubPUSh9A3USN90/OzHNSVN6zxzde07TDCL0=
+k8s.io/client-go v11.0.0+incompatible h1:LBbX2+lOwY9flffWlJM7f1Ct8V2SRNiMRDFeiwnJo9o=
+k8s.io/client-go v11.0.0+incompatible/go.mod h1:7vJpHMYJwNQCWgzmNV+VYUl1zCObLyodBc8nIyt8L5s=
+k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
+k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk=
+k8s.io/klog v0.3.3 h1:niceAagH1tzskmaie/icWd7ci1wbG7Bf2c6YGcQv+3c=
+k8s.io/klog v0.3.3/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk=
+k8s.io/kube-openapi v0.0.0-20190603182131-db7b694dc208 h1:5sW+fEHvlJI3Ngolx30CmubFulwH28DhKjGf70Xmtco=
+k8s.io/kube-openapi v0.0.0-20190603182131-db7b694dc208/go.mod h1:nfDlWeOsu3pUf4yWGL+ERqohP4YsZcBJXWMK+gkzOA4=
+sigs.k8s.io/kustomize/v3 v3.1.1-0.20190826160027-84519c236bac h1:kmfwkekoKBD2cZCiold2zvyTDzycW6ieBstiBo352bk=
+sigs.k8s.io/kustomize/v3 v3.1.1-0.20190826160027-84519c236bac/go.mod h1:ztX4zYc/QIww3gSripwF7TBOarBTm5BvyAMem0kCzOE=
+sigs.k8s.io/structured-merge-diff v0.0.0-20190525122527-15d366b2352e/go.mod h1:wWxsB5ozmmv/SG7nM11ayaAW51xMvak/t1r0CSlcokI=
+sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs=
+sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=

internal/tools/httpclient/httpclient.go

@@ -0,0 +1,23 @@
+package httpclient
+
+import (
+	"net/http"
+	"time"
+
+	"github.com/gomodule/redigo/redis"
+	"github.com/gregjones/httpcache"
+	redis_cache "github.com/gregjones/httpcache/redis"
+)
+
+func FromCache(header http.Header) bool {
+	return header.Get(httpcache.XFromCache) != ""
+}
+
+func NewClient(conn redis.Conn) *http.Client {
+	etagCache := redis_cache.NewWithClient(conn)
+	tr := httpcache.NewTransport(etagCache)
+	return &http.Client{
+		Transport: tr,
+		Timeout:   10 * time.Second,
+	}
+}

internal/tools/index/elasticsearch.go

@@ -0,0 +1,266 @@
+package index
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"time"
+
+	es "github.com/elastic/go-elasticsearch/v6"
+	"github.com/elastic/go-elasticsearch/v6/esapi"
+)
+
+// TODO(damienr74) Split index into reader and writer?
+type index struct {
+	ctx    context.Context
+	client *es.Client
+	name   string
+}
+
+func newIndex(ctx context.Context, name string) (*index, error) {
+	client, err := es.NewDefaultClient()
+	if err != nil {
+		return nil, err
+	}
+
+	return &index{
+		ctx:    ctx,
+		client: client,
+		name:   name,
+	}, nil
+}
+
+type readerFunc func(io.Reader) error
+
+func ignoreResponseBody(reader io.Reader) error {
+	return nil
+}
+
+// checks that elastic returned successfully. If it has not, it will read the
+// body and return it in an error message.
+//
+// Otherwise, it will use the readerFunc to read the body. This function is a
+// mechanism for getting relevant data from the response only if it was successful.
+func (idx *index) responseErrorOrNil(info string, res *esapi.Response,
+	err error, reader readerFunc) error {
+
+	messageStart := fmt.Sprintf("index %s error: %s", idx.name, info)
+	if err != nil || res == nil {
+		return fmt.Errorf("%s: %v", messageStart, err)
+	}
+
+	defer res.Body.Close()
+	if res.IsError() {
+		return fmt.Errorf("%s: %s", messageStart, res.String())
+	}
+
+	if reader != nil {
+		err = reader(res.Body)
+		if err != nil {
+			return fmt.Errorf("%s: %v", messageStart, err)
+		}
+	}
+
+	return nil
+}
+
+func byteJoin(bts ...interface{}) []byte {
+	ret := make([][]byte, len(bts))
+	for i, v := range bts {
+		switch bt := v.(type) {
+		case []byte:
+			ret[i] = bt
+		case string:
+			ret[i] = []byte(bt)
+		default:
+			ret[i] = []byte(fmt.Sprintf("%v", bt))
+		}
+	}
+
+	return bytes.Join(ret, []byte(` `))
+}
+
+// Update the elasticsearch index mappings. (describes how to index/search for the documents).
+func (idx *index) UpdateMapping(mappings []byte) error {
+	request := byteJoin(`{ "mappings":`, mappings, `}`)
+
+	op := idx.client.Indices.PutMapping
+	res, err := op(
+		bytes.NewReader(request),
+		op.WithContext(idx.ctx),
+		op.WithIndex(idx.name),
+		op.WithIncludeTypeName(true),
+		op.WithPretty(),
+	)
+
+	return idx.responseErrorOrNil(
+		fmt.Sprintf("could not update index mappings '%s'", request),
+		res, err, ignoreResponseBody)
+}
+
+// Update the elasticsearch index settings. (describes default parameters and
+// some analyzer definitions, etc.)
+func (idx *index) UpdateSetting(settings []byte) error {
+	request := byteJoin(`{ "settings": `, settings, `}`)
+	op := idx.client.Indices.PutSettings
+	res, err := op(
+		bytes.NewReader(request),
+		op.WithContext(idx.ctx),
+		op.WithIndex(idx.name),
+		op.WithPretty(),
+	)
+
+	return idx.responseErrorOrNil(
+		fmt.Sprintf("could not update index settings '%s'", request),
+		res, err, ignoreResponseBody)
+}
+
+// Create an index providing both the mappings and the settings.
+func (idx *index) CreateIndex(mappings []byte, settings []byte) error {
+	request := byteJoin(`{ "mappings":`, mappings, `, "settings":`, settings, `}`)
+	op := idx.client.Indices.Create
+	res, err := op(
+		idx.name,
+		op.WithBody(bytes.NewReader(request)),
+		op.WithContext(idx.ctx),
+		op.WithHuman(),
+		op.WithPretty(),
+		op.WithIncludeTypeName(true),
+	)
+
+	return idx.responseErrorOrNil(
+		fmt.Sprintf("could not create index with config '%s'", request),
+		res, err, ignoreResponseBody)
+}
+
+// Delete an index.
+func (idx *index) DeleteIndex() error {
+	res, err := idx.client.Indices.Delete(
+		[]string{idx.name},
+	)
+
+	return idx.responseErrorOrNil("could not delete index",
+		res, err, ignoreResponseBody)
+}
+
+// Insert or update the document by ID.
+func (idx *index) Put(uniqueID string, doc interface{}) (string, error) {
+	body, err := json.Marshal(doc)
+	if err != nil {
+		return "", err
+	}
+
+	req := esapi.IndexRequest{
+		Index:      idx.name,
+		Body:       bytes.NewReader(body),
+		DocumentID: uniqueID,
+	}
+	res, err := req.Do(idx.ctx, idx.client)
+
+	var id string
+	readId := func(reader io.Reader) error {
+		type InsertResult struct {
+			ID string `json:"_id,omitempty"`
+		}
+		var ir InsertResult
+		data, err := ioutil.ReadAll(reader)
+		if err != nil {
+			return err
+		}
+
+		err = json.Unmarshal(data, &ir)
+		if err != nil {
+			return err
+		}
+		id = ir.ID
+
+		return nil
+	}
+
+	// populates the id field.
+	err = idx.responseErrorOrNil("could not insert document",
+		res, err, readId)
+
+	return id, err
+}
+
+type scrollUpdater func(string, readerFunc) error
+
+// Update the scroll for iteration. If no scroll exists, create one.
+func (idx *index) scrollUpdater(query []byte, batchSize int,
+	timeout time.Duration) scrollUpdater {
+
+	return func(scrollID string, reader readerFunc) error {
+		var res *esapi.Response
+		var err error
+
+		if scrollID == "" {
+			search := idx.client.Search
+			res, err = search(
+				search.WithContext(idx.ctx),
+				search.WithIndex(idx.name),
+				search.WithBody(bytes.NewBuffer(query)),
+				search.WithScroll(timeout),
+				search.WithSize(batchSize),
+			)
+		} else {
+			scroll := idx.client.Scroll
+			res, err = scroll(
+				scroll.WithContext(idx.ctx),
+				scroll.WithScroll(timeout),
+				scroll.WithScrollID(scrollID),
+			)
+		}
+
+		return idx.responseErrorOrNil(
+			fmt.Sprintf("could not scroll for query %s", query),
+			res, err, reader)
+	}
+}
+
+// Simple search options. Size is the number of elements to return, From is the
+// rank of the results according to the query. Used as a simple (stateless)
+// pagination technique.
+type SearchOptions struct {
+	Size int
+	From int
+}
+
+// Search for a query (json query dsl) with some options, and use the reader func
+// to extract the response.
+func (idx *index) Search(query []byte, opts SearchOptions,
+	responseReader readerFunc) error {
+
+	op := idx.client.Search
+	res, err := op(
+		op.WithContext(idx.ctx),
+		op.WithIndex(idx.name),
+		op.WithBody(bytes.NewBuffer(query)),
+		op.WithTrackTotalHits(true),
+		op.WithSize(opts.Size),
+		op.WithFrom(opts.From),
+		op.WithPretty(),
+	)
+
+	return idx.responseErrorOrNil(
+		fmt.Sprintf("could not complete search query %v", query),
+		res, err, responseReader)
+}
+
+// Delete an element from elasticsearch by Id.
+func (idx *index) Delete(id string) error {
+	op := idx.client.Delete
+	res, err := op(
+		idx.name,
+		id,
+		op.WithContext(idx.ctx),
+		op.WithPretty(),
+	)
+
+	return idx.responseErrorOrNil(
+		fmt.Sprintf("could not delete id(%s) from index(%s)", id, idx.name),
+		res, err, ignoreResponseBody)
+}

internal/tools/index/kustomize.go

@@ -0,0 +1,337 @@
+package index
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"strings"
+	"time"
+
+	"sigs.k8s.io/kustomize/internal/tools/doc"
+)
+
+const (
+	AggregationKeyword = "aggs"
+)
+
+// Redefinition of Hits structure. Must match the json string of
+// KustomizeResult.Hits.Hits. Declared as a convenience for iteration.
+type KustomizeHits []struct {
+	ID       string                    `json:"id"`
+	Document doc.KustomizationDocument `json:"result"`
+}
+
+type KustomizeResult struct {
+	ScrollID *string `json:"-"`
+
+	Hits *struct {
+		Total int `json:"total"`
+		Hits  []struct {
+			ID       string                    `json:"id"`
+			Document doc.KustomizationDocument `json:"result"`
+		} `json:"hits"`
+	} `json:"hits,omitempty"`
+
+	Aggregations *struct {
+		Timeseries *struct {
+			Buckets []struct {
+				Key   string `json:"key"`
+				Count int    `json:"count"`
+			} `json:"buckets"`
+		} `json:"timeseries,omitempty"`
+
+		Kinds *struct {
+			OtherCount int `json:"otherResults"`
+			Buckets    []struct {
+				Key   string `json:"key"`
+				Count int    `json:"count"`
+			} `json:"buckets"`
+		} `json:"kinds,omitempty"`
+	} `json:"aggregations,omitempty"`
+}
+
+// Elasticsearch has some sometimes inconsistent labels, and some pretty ugly label choices.
+// However, the structure seems reasonable, so I wanted to use it if possible. This method
+// needs two copies of the types to make the json strings different. The Copies must be the
+// exact same type/structure, so the types must be declared inline. Go will check that these
+// are convertible at compile time, and converting at runtime is a noop.
+type ElasticKustomizeResult struct {
+	ScrollID *string `json:"_scroll_id,omitempty"`
+
+	Hits *struct {
+		Total int `json:"total"`
+		Hits  []struct {
+			ID       string                    `json:"_id"`
+			Document doc.KustomizationDocument `json:"_source"`
+		} `json:"hits"`
+	} `json:"hits,omitempty"`
+
+	Aggregations *struct {
+		Timeseries *struct {
+			Buckets []struct {
+				Key   string `json:"key_as_string"`
+				Count int    `json:"doc_count"`
+			}
+		} `json:"timeseries,omitempty"`
+
+		Kinds *struct {
+			OtherCount int `json:"sum_other_doc_count"`
+			Buckets    []struct {
+				Key   string `json:"key"`
+				Count int    `json:"doc_count"`
+			}
+		} `json:"kinds,omitempty"`
+	} `json:"aggregations,omitempty"`
+}
+
+type KustomizeIndex struct {
+	*index
+}
+
+// Create index reference to the index containing the kustomize documents.
+func NewKustomizeIndex(ctx context.Context) (*KustomizeIndex, error) {
+	idx, err := newIndex(ctx, "kustomize")
+	if err != nil {
+		return nil, err
+	}
+	return &KustomizeIndex{idx}, nil
+}
+
+// Return a timeseries of kustomization file counts.
+func TimeseriesAggregation() (string, map[string]interface{}) {
+	return "timeseries", map[string]interface{}{
+		"date_histogram": map[string]interface{}{
+			"field":    "creationTime",
+			"interval": "day",
+			/// XXX Only return values with counts, otherwise
+			// every day is added to the output...
+			// This matters if ever a zero valued time would
+			// be stored in the creationTime field... it would
+			// return >600k entries (for every day since year 0).
+			// IDK why this is default, but I would not want this
+			// to happen...
+			"min_doc_count": 1,
+		},
+	}
+}
+
+// Return aggregation of results based off of their kinds.
+func KindAggregation(maxBuckets int) (string, map[string]interface{}) {
+	if maxBuckets < 1 {
+		maxBuckets = 1
+	}
+	return "kinds", map[string]interface{}{
+		"terms": map[string]interface{}{
+			"field": "kinds.keyword",
+			"size":  maxBuckets,
+		},
+	}
+}
+
+// The multi_match search type in elasticsearch will check each field according
+// to their respective analyzers for the identifier.
+func multiMatch(query string) map[string]interface{} {
+	return map[string]interface{}{
+		"multi_match": map[string]interface{}{
+			"type": "cross_fields",
+			"fields": []string{
+				"values.keyword^3",
+				"identifiers.keyword^3",
+				"values.ngram",
+				"identifiers.ngram",
+				// TODO(damienr74) remove document with default
+				// analyzer. It does not handle special (=,: etc)
+				// characters properly, and matches with false
+				// positives. document.whitespace does not exist
+				// yet, but should use the whitespace analyzer.
+				"document",
+				"document.whitespace",
+			},
+			"query": query,
+		},
+	}
+}
+
+// Build an elasticsearch query from a user query.
+func BuildQuery(query string) map[string]interface{} {
+	queryTokens := strings.Fields(query)
+	if len(queryTokens) == 0 {
+		return map[string]interface{}{
+			"size": 0,
+		}
+	}
+
+	mustMatch := make([]map[string]interface{}, len(queryTokens))
+
+	for i, tok := range queryTokens {
+		if strings.HasPrefix(strings.ToLower(tok), "kind=") {
+			mustMatch[i] = map[string]interface{}{
+				"term": map[string]interface{}{
+					"kinds.keyword": tok[5:],
+				},
+			}
+			continue
+		}
+		mustMatch[i] = multiMatch(tok)
+	}
+
+	structuredQuery := map[string]interface{}{
+		"query": map[string]interface{}{
+			"bool": map[string]interface{}{
+				"must": mustMatch,
+			},
+		},
+	}
+
+	return structuredQuery
+}
+
+// Iterator based off of the way bufio.Scanner works.
+//
+// Example:
+//	for it.Next() {
+//		for _, doc := range it.Value().Hits {
+//			// Handle KustomizationDocument.
+//		}
+//	}
+//
+//	if err := it.Err(); err != nil {
+//		// Handle err.
+//	}
+type KustomizeIterator struct {
+	update scrollUpdater
+	err    error
+	// Matches the return definition of elasticsearch search results. The
+	// scroll ID is practically a database cursor.
+	scrollImpl KustomizeResult
+}
+
+// Get the next batch of results. Note that this returns multiple results that
+// can be iterated.
+func (it *KustomizeIterator) Next() bool {
+	reader := func(reader io.Reader) error {
+		data, err := ioutil.ReadAll(reader)
+		if err != nil {
+			return fmt.Errorf("could not read from body: %v", err)
+		}
+		var scrollInput ElasticKustomizeResult
+		err = json.Unmarshal(data, &scrollInput)
+		if err != nil {
+			return fmt.Errorf("cloud not marshal %s into %T: %v",
+				data, scrollInput, err)
+		}
+		it.scrollImpl = KustomizeResult(scrollInput)
+
+		return nil
+	}
+
+	if it.err == nil {
+		fmt.Printf("updating scroll: %s\n", *it.scrollImpl.ScrollID)
+		it.err = it.update(*it.scrollImpl.ScrollID, reader)
+	}
+
+	// if there is no error and the array is not empty, then Value is
+	// obligated to return a valid result.
+	return it.err == nil &&
+		it.scrollImpl.Hits != nil &&
+		len(it.scrollImpl.Hits.Hits) > 0
+}
+
+// Get the value from this batch of iterations.
+func (it *KustomizeIterator) Value() KustomizeResult {
+	return it.scrollImpl
+}
+
+// Check if any errors have occured.
+func (it *KustomizeIterator) Err() error {
+	return it.err
+}
+
+// Create an iterator over query. Iterate in chunks of batchSize, each batch
+// should take no longer than timeout to read (otherwise, elasticsearch will
+// delete the context).
+//
+// XXX Important to set a reasonable amount of time to read the documents. If
+// a lot of processing must be done, consider loading everything in memory
+// before doing it so that, a short timeout period can be set. Scrolling creates
+// a consistent DB context, so this can be costly.
+//
+// Scrolling is also not meant to be used for real time purposes. If you need
+// results quickly, consider using the From: field in SearchOptions and a normal
+// search. This will not guarantee that the values will not change but is more
+// suitable for lower latencies/long execution timeouts.
+func (ki *KustomizeIndex) IterateQuery(query []byte, batchSize int,
+	timeout time.Duration) *KustomizeIterator {
+
+	emptyScroll := ""
+	return &KustomizeIterator{
+		update: ki.scrollUpdater(query, batchSize, timeout),
+		scrollImpl: KustomizeResult{
+			ScrollID: &emptyScroll,
+		},
+	}
+}
+
+// type specific Put for inserting structured kustomization documents.
+func (ki *KustomizeIndex) Put(id string, doc *doc.KustomizationDocument) (string, error) {
+	id, err := ki.index.Put(id, doc)
+	if err != nil {
+		return id, fmt.Errorf("could not insert in elastic: %v", err)
+	}
+	return id, nil
+}
+
+// Kustomize search options: What metrics should be returned? Kind Aggregation,
+// TimeseriesAggregation, etc. Also embedds the SearchOptions field to specify
+// the position in the sorted list of results and the number of results to return.
+type KustomizeSearchOptions struct {
+	SearchOptions
+	KindAggregation       bool
+	TimeseriesAggregation bool
+}
+
+// Search the index with the given query string. Returns a structured result and possible
+// aggregates.
+func (ki *KustomizeIndex) Search(query string,
+	opts KustomizeSearchOptions) (*KustomizeResult, error) {
+
+	aggMap := make(map[string]interface{})
+	if opts.KindAggregation {
+		k, kAgg := KindAggregation(15)
+		aggMap[k] = kAgg
+	}
+	if opts.TimeseriesAggregation {
+		t, tAgg := TimeseriesAggregation()
+		aggMap[t] = tAgg
+	}
+
+	esQuery := BuildQuery(query)
+	if len(aggMap) > 0 {
+		esQuery[AggregationKeyword] = aggMap
+	}
+
+	data, err := json.Marshal(&esQuery)
+	if err != nil {
+		return nil, fmt.Errorf("failed to format query %s", query)
+	}
+	fmt.Printf("formated query: %s\n", data)
+
+	var kr ElasticKustomizeResult
+	err = ki.index.Search(data, opts.SearchOptions, func(results io.Reader) error {
+		data, err = ioutil.ReadAll(results)
+		if err != nil {
+			return fmt.Errorf("could not read results from search: %v", err)
+		}
+
+		if err = json.Unmarshal(data, &kr); err != nil {
+			return fmt.Errorf("could not parse results from search: %v", err)
+		}
+
+		return nil
+	})
+	res := KustomizeResult(kr)
+
+	return &res, err
+}

internal/tools/index/kustomize_test.go

@@ -0,0 +1,72 @@
+package index
+
+import (
+	"reflect"
+	"testing"
+)
+
+func TestBuildQuery(t *testing.T) {
+	testCases := []struct {
+		query  string
+		result map[string]interface{}
+	}{
+		{
+			query:  "    \t\n\r",
+			result: map[string]interface{}{"size": 0},
+		},
+		{
+			query: "\tidentifier1 identifier2\nidentifier3\r",
+			result: map[string]interface{}{
+				"query": map[string]interface{}{
+					"bool": map[string]interface{}{
+						"must": []map[string]interface{}{
+							multiMatch("identifier1"),
+							multiMatch("identifier2"),
+							multiMatch("identifier3"),
+						},
+					},
+				},
+			},
+		},
+		{
+			query: "kind=Kustomization",
+			result: map[string]interface{}{
+				"query": map[string]interface{}{
+					"bool": map[string]interface{}{
+						"must": []map[string]interface{}{
+							{
+								"term": map[string]interface{}{
+									"kinds.keyword": "Kustomization",
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+		{
+			query: "kind=Kustomization identifier2",
+			result: map[string]interface{}{
+				"query": map[string]interface{}{
+					"bool": map[string]interface{}{
+						"must": []map[string]interface{}{
+							{
+								"term": map[string]interface{}{
+									"kinds.keyword": "Kustomization",
+								},
+							},
+							multiMatch("identifier2"),
+						},
+					},
+				},
+			},
+		},
+	}
+
+	for _, tc := range testCases {
+		result := BuildQuery(tc.query)
+		if !reflect.DeepEqual(tc.result, result) {
+			t.Errorf("Expected %#v to match %#v", result, tc.result)
+		}
+	}
+}