Merge pull request #2455 from monopole/add6902

Introduce ValueAddTransformer.

.github/workflows/go.yml

@@ -0,0 +1,28 @@
+name: Go
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+
+jobs:
+
+  build:
+    name: Build
+    runs-on: ubuntu-latest
+    steps:
+
+    - name: Set up Go 1.x
+      uses: actions/setup-go@v2
+      with:
+        go-version: ^1.13
+      id: go
+
+    - name: Check out code into the Go module directory
+      uses: actions/checkout@v2
+
+    - name: Build
+      run: ./travis/kyaml-pre-commit.sh
+      env:
+        KUSTOMIZE_DOCKER_E2E: true

.golangci-kustomize.yml

@@ -7,8 +7,13 @@ linters:
     - bodyclose
     - deadcode
     - depguard
+    # - dogsled
     - dupl
+    # - errcheck
+    # - funlen
+    # - gochecknoinits
     - goconst
+    # - gocritic
     - gocyclo
     - gofmt
     - goimports
@@ -21,6 +26,7 @@ linters:
     - lll
     - misspell
     - nakedret
+    # - scopelint
     - staticcheck
     - structcheck
     # stylecheck demands that acronyms not be treated as words
@@ -28,9 +34,10 @@ linters:
     # - stylecheck
     - typecheck
     - unconvert
-    - unused
     - unparam
+    - unused
     - varcheck
+    # - whitespace
 
 linters-settings:
   dupl:

.travis.yml

@@ -1,42 +0,0 @@
-os:
-  - linux
-  - osx
-# TODO: Speed up the slowness of the osx travis runs
-# Maybe cache brew installs?
-#
-# TODO: Uncomment when some gets the tests running on Windows.
-#  - windows
-
-addons:
-  apt:
-    packages:
-    - tree
-  homebrew:
-    packages:
-    - tree
-    update: true
-
-# Only clone the most recent commit.
-git:
-  depth: 1
-
-language: go
-
-go:
-  - "1.13"
-
-go_import_path: sigs.k8s.io/kustomize
-
-before_install:
-  - source ./travis/consider-early-travis-exit.sh
-
-# Skip the install process; let pre-commit.sh do it.
-install: true
-
-script:
-  - make verify-kustomize
-  - ./travis/kyaml-pre-commit.sh
-
-# TBD. Suppressing for now.
-notifications:
-  email: false

Makefile

@@ -17,18 +17,30 @@ verify-kustomize: \
 	test-examples-kustomize-against-HEAD \
 	test-examples-kustomize-against-latest
 
+# The following target referenced by a file in
+# https://github.com/kubernetes/test-infra/tree/master/config/jobs/kubernetes-sigs/kustomize
+.PHONY: prow-presubmit-check
+prow-presubmit-check: \
+	lint-kustomize \
+	test-unit-kustomize-all \
+	test-unit-cmd-all \
+	test-go-mod
+
+.PHONY: verify-kustomize-e2e
+verify-kustomize-e2e: test-examples-e2e-kustomize
+
 # Other builds in this repo might want a different linter version.
 # Without one Makefile to rule them all, the different makes
 # cannot assume that golanci-lint is at the version they want
 # since everything uses the same implicit GOPATH.
 # This installs in a temp dir to avoid overwriting someone else's
 # linter, then installs in MYGOBIN with a new name.
-# Version pinned by api/go.mod
+# Version pinned by hack/go.mod
 $(MYGOBIN)/golangci-lint-kustomize:
 	( \
 		set -e; \
-		export GOBIN=$$(mktemp -d) \
-		cd api; \
+		export GOBIN=$$(mktemp -d); \
+		cd hack; \
 		GO111MODULE=on go install github.com/golangci/golangci-lint/cmd/golangci-lint; \
 		mv $$GOBIN/golangci-lint $(MYGOBIN)/golangci-lint-kustomize \
 	)
@@ -48,6 +60,11 @@ $(MYGOBIN)/goimports:
 	cd api; \
 	go install golang.org/x/tools/cmd/goimports
 
+# Install resource from whatever is checked out.
+$(MYGOBIN)/resource:
+	cd cmd/resource; \
+	go install .
+
 # To pin pluginator, use this recipe instead:
 #   cd api;
 #   go install sigs.k8s.io/kustomize/pluginator/v2
@@ -99,7 +116,6 @@ _builtinplugins = \
 	ConfigMapGenerator.go \
 	HashTransformer.go \
 	ImageTagTransformer.go \
-	InventoryTransformer.go \
 	LabelTransformer.go \
 	LegacyOrderTransformer.go \
 	NamespaceTransformer.go \
@@ -108,7 +124,8 @@ _builtinplugins = \
 	PatchTransformer.go \
 	PrefixSuffixTransformer.go \
 	ReplicaCountTransformer.go \
-	SecretGenerator.go
+	SecretGenerator.go \
+	ValueAddTransformer.go
 
 # Maintaining this explicit list of generated files, and
 # adding it as a dependency to a few targets, to assure
@@ -124,7 +141,6 @@ $(pGen)/AnnotationsTransformer.go: $(pSrc)/annotationstransformer/AnnotationsTra
 $(pGen)/ConfigMapGenerator.go: $(pSrc)/configmapgenerator/ConfigMapGenerator.go
 $(pGen)/HashTransformer.go: $(pSrc)/hashtransformer/HashTransformer.go
 $(pGen)/ImageTagTransformer.go: $(pSrc)/imagetagtransformer/ImageTagTransformer.go
-$(pGen)/InventoryTransformer.go: $(pSrc)/inventorytransformer/InventoryTransformer.go
 $(pGen)/LabelTransformer.go: $(pSrc)/labeltransformer/LabelTransformer.go
 $(pGen)/LegacyOrderTransformer.go: $(pSrc)/legacyordertransformer/LegacyOrderTransformer.go
 $(pGen)/NamespaceTransformer.go: $(pSrc)/namespacetransformer/NamespaceTransformer.go
@@ -134,6 +150,7 @@ $(pGen)/PatchTransformer.go: $(pSrc)/patchtransformer/PatchTransformer.go
 $(pGen)/PrefixSuffixTransformer.go: $(pSrc)/prefixsuffixtransformer/PrefixSuffixTransformer.go
 $(pGen)/ReplicaCountTransformer.go: $(pSrc)/replicacounttransformer/ReplicaCountTransformer.go
 $(pGen)/SecretGenerator.go: $(pSrc)/secretgenerator/SecretGenerator.go
+$(pGen)/ValueAddTransformer.go: $(pSrc)/valueaddtransformer/ValueAddTransformer.go
 
 # The (verbose but portable) Makefile way to convert to lowercase.
 toLowerCase = $(subst A,a,$(subst B,b,$(subst C,c,$(subst D,d,$(subst E,e,$(subst F,f,$(subst G,g,$(subst H,h,$(subst I,i,$(subst J,j,$(subst K,k,$(subst L,l,$(subst M,m,$(subst N,n,$(subst O,o,$(subst P,p,$(subst Q,q,$(subst R,r,$(subst S,s,$(subst T,t,$(subst U,u,$(subst V,v,$(subst W,w,$(subst X,x,$(subst Y,y,$(subst Z,z,$1))))))))))))))))))))))))))
@@ -171,8 +188,14 @@ lint-kustomize: install-tools $(builtinplugins)
 	cd pluginator; \
 	$(MYGOBIN)/golangci-lint-kustomize -c ../.golangci-kustomize.yml run ./...
 
+# Used to add non-default compilation flags when experimenting with
+# plugin-to-api compatibility checks.
+.PHONY: build-kustomize-api
+build-kustomize-api: $(builtinplugins)
+	cd api; go build ./...
+
 .PHONY: test-unit-kustomize-api
-test-unit-kustomize-api: $(builtinplugins)
+test-unit-kustomize-api: build-kustomize-api
 	cd api; go test ./...
 
 .PHONY: test-unit-kustomize-plugins
@@ -189,6 +212,22 @@ test-unit-kustomize-all: \
 	test-unit-kustomize-cli \
 	test-unit-kustomize-plugins
 
+test-unit-cmd-all:
+	./travis/kyaml-pre-commit.sh
+
+test-go-mod:
+	./travis/check-go-mod.sh
+
+.PHONY:
+test-examples-e2e-kustomize: $(MYGOBIN)/mdrip $(MYGOBIN)/kind
+	( \
+		set -e; \
+		/bin/rm -f $(MYGOBIN)/kustomize; \
+		echo "Installing kustomize from ."; \
+		cd kustomize; go install .; cd ..; \
+		./hack/testExamplesE2EAgainstKustomize.sh .; \
+	)
+
 .PHONY:
 test-examples-kustomize-against-HEAD: $(MYGOBIN)/kustomize $(MYGOBIN)/mdrip
 	./hack/testExamplesAgainstKustomize.sh HEAD
@@ -199,7 +238,7 @@ test-examples-kustomize-against-latest: $(MYGOBIN)/mdrip
 		set -e; \
 		/bin/rm -f $(MYGOBIN)/kustomize; \
 		echo "Installing kustomize from latest."; \
-		GO111MODULE=on go install sigs.k8s.io/kustomize/kustomize/v3; \
+		GO111MODULE=on go get sigs.k8s.io/kustomize/kustomize/v3@v3.5.4; \
 		./hack/testExamplesAgainstKustomize.sh latest; \
 		echo "Reinstalling kustomize from HEAD."; \
 		cd kustomize; go install .; \
@@ -222,21 +261,48 @@ $(MYGOBIN)/kubeval:
 	)
 
 # linux only.
-# This is for testing an example plugin that
-# uses helm to inflate a chart for subsequent kustomization.
-# Don't want to add a hard dependence in go.mod file
-# to helm.
-# Instead, download the binary.
-$(MYGOBIN)/helm:
+# This is for testing an example plugin that uses helm to inflate a chart
+# for subsequent kustomization.
+# Don't want to add a hard dependence in go.mod file to helm.
+# Instead, download the binaries.
+$(MYGOBIN)/helmV2:
 	( \
 		set -e; \
 		d=$(shell mktemp -d); cd $$d; \
-		wget https://storage.googleapis.com/kubernetes-helm/helm-v2.13.1-linux-amd64.tar.gz; \
-		tar -xvzf helm-v2.13.1-linux-amd64.tar.gz; \
-		mv linux-amd64/helm $(MYGOBIN); \
+		tgzFile=helm-v2.13.1-linux-amd64.tar.gz; \
+		wget https://storage.googleapis.com/kubernetes-helm/$$tgzFile; \
+		tar -xvzf $$tgzFile; \
+		mv linux-amd64/helm $(MYGOBIN)/helmV2; \
 		rm -rf $$d \
 	)
 
+# Helm V3 differs from helm V2; downloading it to provide coverage for the
+# chart inflator plugin under helm v3.
+$(MYGOBIN)/helmV3:
+	( \
+		set -e; \
+		d=$(shell mktemp -d); cd $$d; \
+		tgzFile=helm-v3.2.0-rc.1-linux-amd64.tar.gz; \
+		wget https://get.helm.sh/$$tgzFile; \
+		tar -xvzf $$tgzFile; \
+		mv linux-amd64/helm $(MYGOBIN)/helmV3; \
+		rm -rf $$d \
+	)
+
+# Default version of helm is v2 for the time being.
+$(MYGOBIN)/helm: $(MYGOBIN)/helmV2
+	ln -s $(MYGOBIN)/helmV2 $(MYGOBIN)/helm
+
+$(MYGOBIN)/kind:
+	( \
+        set -e; \
+        d=$(shell mktemp -d); cd $$d; \
+        wget -O ./kind https://github.com/kubernetes-sigs/kind/releases/download/v0.7.0/kind-$(shell uname)-amd64; \
+        chmod +x ./kind; \
+        mv ./kind $(MYGOBIN); \
+        rm -rf $$d; \
+	)
+
 .PHONY: clean
 clean: kustomize-external-go-plugin-clean
 	go clean --cache
@@ -245,6 +311,7 @@ clean: kustomize-external-go-plugin-clean
 	rm -f $(MYGOBIN)/kustomize
 	rm -f $(MYGOBIN)/golangci-lint-kustomize
 
+# Nuke the site from orbit.  It's the only way to be sure.
 .PHONY: nuke
 nuke: clean
-	sudo rm -rf $(shell go env GOPATH)/pkg/mod/sigs.k8s.io
+	go clean --modcache

OWNERS_ALIASES

@@ -9,3 +9,4 @@ aliases:
     - mengqiy
     - monopole
     - pwittrock
+    - mortent

README.md

@@ -12,8 +12,7 @@ and it's like [`sed`], in that it emits edited text.
 This tool is sponsored by [sig-cli] ([KEP]), and
 inspired by [DAM].
 
-
-[![Build Status](https://travis-ci.org/kubernetes-sigs/kustomize.svg?branch=master)](https://travis-ci.org/kubernetes-sigs/kustomize)
+[![Build Status](https://prow.k8s.io/badge.svg?jobs=kustomize-presubmit-master)](https://prow.k8s.io/job-history/kubernetes-jenkins/pr-logs/directory/kustomize-presubmit-master)
 [![Go Report Card](https://goreportcard.com/badge/github.com/kubernetes-sigs/kustomize)](https://goreportcard.com/report/github.com/kubernetes-sigs/kustomize)
 
 Download a binary from the [release page], or see
@@ -107,7 +106,7 @@ Take the work from step (1) above, move it into a
 `someApp` subdirectory called `base`, then
 place overlays in a sibling directory.
 
-An overlay is just another kustomization, refering to
+An overlay is just another kustomization, referring to
 the base, and referring to patches to apply to that
 base.
 

api/builtins/AnnotationsTransformer.go

@@ -4,9 +4,10 @@
 package builtins
 
 import (
+	"sigs.k8s.io/kustomize/api/filters/annotations"
 	"sigs.k8s.io/kustomize/api/resmap"
-	"sigs.k8s.io/kustomize/api/transform"
 	"sigs.k8s.io/kustomize/api/types"
+	"sigs.k8s.io/kustomize/kyaml/filtersutil"
 	"sigs.k8s.io/yaml"
 )
 
@@ -24,14 +25,16 @@ func (p *AnnotationsTransformerPlugin) Config(
 }
 
 func (p *AnnotationsTransformerPlugin) Transform(m resmap.ResMap) error {
-	t, err := transform.NewMapTransformer(
-		p.FieldSpecs,
-		p.Annotations,
-	)
-	if err != nil {
-		return err
+	for _, r := range m.Resources() {
+		err := filtersutil.ApplyToJSON(annotations.Filter{
+			Annotations: p.Annotations,
+			FsSlice:     p.FieldSpecs,
+		}, r.Kunstructured)
+		if err != nil {
+			return err
+		}
 	}
-	return t.Transform(m)
+	return nil
 }
 
 func NewAnnotationsTransformerPlugin() resmap.TransformerPlugin {

api/builtins/ConfigMapGenerator.go

@@ -13,13 +13,10 @@ import (
 type ConfigMapGeneratorPlugin struct {
 	h                *resmap.PluginHelpers
 	types.ObjectMeta `json:"metadata,omitempty" yaml:"metadata,omitempty" protobuf:"bytes,1,opt,name=metadata"`
-	types.GeneratorOptions
 	types.ConfigMapArgs
 }
 
-func (p *ConfigMapGeneratorPlugin) Config(
-	h *resmap.PluginHelpers, config []byte) (err error) {
-	p.GeneratorOptions = types.GeneratorOptions{}
+func (p *ConfigMapGeneratorPlugin) Config(h *resmap.PluginHelpers, config []byte) (err error) {
 	p.ConfigMapArgs = types.ConfigMapArgs{}
 	err = yaml.Unmarshal(config, p)
 	if p.ConfigMapArgs.Name == "" {
@@ -34,8 +31,7 @@ func (p *ConfigMapGeneratorPlugin) Config(
 
 func (p *ConfigMapGeneratorPlugin) Generate() (resmap.ResMap, error) {
 	return p.h.ResmapFactory().FromConfigMapArgs(
-		kv.NewLoader(p.h.Loader(), p.h.Validator()),
-		&p.GeneratorOptions, p.ConfigMapArgs)
+		kv.NewLoader(p.h.Loader(), p.h.Validator()), p.ConfigMapArgs)
 }
 
 func NewConfigMapGeneratorPlugin() resmap.GeneratorPlugin {

api/builtins/ImageTagTransformer.go

@@ -144,8 +144,10 @@ func (p *ImageTagTransformerPlugin) findContainers(obj map[string]interface{}) e
 }
 
 func isImageMatched(s, t string) bool {
-	// Tag values are limited to [a-zA-Z0-9_.-].
-	pattern, _ := regexp.Compile("^" + t + "(@sha256)?(:[a-zA-Z0-9_.-]*)?$")
+	// Tag values are limited to [a-zA-Z0-9_.{}-].
+	// Some tools like Bazel rules_k8s allow tag patterns with {} characters.
+	// More info: https://github.com/bazelbuild/rules_k8s/pull/423
+	pattern, _ := regexp.Compile("^" + t + "(@sha256)?(:[a-zA-Z0-9_.{}-]*)?$")
 	return pattern.MatchString(s)
 }
 

api/builtins/InventoryTransformer.go

@@ -1,128 +0,0 @@
-// Code generated by pluginator on InventoryTransformer; DO NOT EDIT.
-// pluginator {unknown  1970-01-01T00:00:00Z  }
-
-package builtins
-
-import (
-	"fmt"
-
-	"sigs.k8s.io/kustomize/api/hasher"
-	"sigs.k8s.io/kustomize/api/inventory"
-	"sigs.k8s.io/kustomize/api/kv"
-	"sigs.k8s.io/kustomize/api/resid"
-	"sigs.k8s.io/kustomize/api/resmap"
-	"sigs.k8s.io/kustomize/api/resource"
-	"sigs.k8s.io/kustomize/api/types"
-	"sigs.k8s.io/yaml"
-)
-
-type InventoryTransformerPlugin struct {
-	h                *resmap.PluginHelpers
-	types.ObjectMeta `json:"metadata,omitempty" yaml:"metadata,omitempty" protobuf:"bytes,1,opt,name=metadata"`
-	Policy           string `json:"policy,omitempty" yaml:"policy,omitempty"`
-}
-
-func (p *InventoryTransformerPlugin) Config(
-	h *resmap.PluginHelpers, c []byte) (err error) {
-	p.h = h
-	err = yaml.Unmarshal(c, p)
-	if err != nil {
-		return err
-	}
-	if p.Policy == "" {
-		p.Policy = types.GarbageIgnore.String()
-	}
-	if p.Policy != types.GarbageCollect.String() &&
-		p.Policy != types.GarbageIgnore.String() {
-		return fmt.Errorf(
-			"unrecognized garbagePolicy '%s'", p.Policy)
-	}
-	return nil
-}
-
-// Transform generates an inventory object from the input ResMap.
-// This ConfigMap supports the pruning command in
-// the client side tool proposed here:
-// https://github.com/kubernetes/enhancements/pull/810
-//
-// The inventory data is written to the ConfigMap's
-// annotations, rather than to the key-value pairs in
-// the ConfigMap's data field, since
-//   1. Keys in a ConfigMap's data field are too
-//      constrained for this purpose.
-//   2. Using annotations allow any object to be used,
-//      not just a ConfigMap, should some other object
-//      (e.g. some App object) become more desirable
-//      for this purpose.
-func (p *InventoryTransformerPlugin) Transform(m resmap.ResMap) error {
-
-	inv, h, err := makeInventory(m)
-	if err != nil {
-		return err
-	}
-
-	args := types.ConfigMapArgs{}
-	args.Name = p.Name
-	args.Namespace = p.Namespace
-	opts := &types.GeneratorOptions{
-		Annotations: make(map[string]string),
-	}
-	opts.Annotations[inventory.HashAnnotation] = h
-	err = inv.UpdateAnnotations(opts.Annotations)
-	if err != nil {
-		return err
-	}
-
-	cm, err := p.h.ResmapFactory().RF().MakeConfigMap(
-		kv.NewLoader(p.h.Loader(), p.h.Validator()), opts, &args)
-	if err != nil {
-		return err
-	}
-
-	if p.Policy == types.GarbageCollect.String() {
-		for _, byeBye := range m.AllIds() {
-			m.Remove(byeBye)
-		}
-	}
-	return m.Append(cm)
-}
-
-func makeInventory(m resmap.ResMap) (
-	inv *inventory.Inventory, hash string, err error) {
-	inv = inventory.NewInventory()
-	var keys []string
-	for _, r := range m.Resources() {
-		ns := r.GetNamespace()
-		item := resid.NewResIdWithNamespace(r.GetGvk(), r.GetName(), ns)
-		if _, ok := inv.Current[item]; ok {
-			return nil, "", fmt.Errorf(
-				"item '%v' already in inventory", item)
-		}
-		inv.Current[item], err = computeRefs(r, m)
-		if err != nil {
-			return nil, "", err
-		}
-		keys = append(keys, item.String())
-	}
-	h, err := hasher.SortArrayAndComputeHash(keys)
-	return inv, h, err
-}
-
-func computeRefs(
-	r *resource.Resource, m resmap.ResMap) (refs []resid.ResId, err error) {
-	for _, refid := range r.GetRefBy() {
-		ref, err := m.GetByCurrentId(refid)
-		if err != nil {
-			return nil, err
-		}
-		refs = append(
-			refs,
-			resid.NewResIdWithNamespace(
-				ref.GetGvk(), ref.GetName(), ref.GetNamespace()))
-	}
-	return
-}
-
-func NewInventoryTransformerPlugin() resmap.TransformerPlugin {
-	return &InventoryTransformerPlugin{}
-}

api/builtins/LabelTransformer.go

@@ -4,9 +4,10 @@
 package builtins
 
 import (
+	"sigs.k8s.io/kustomize/api/filters/labels"
 	"sigs.k8s.io/kustomize/api/resmap"
-	"sigs.k8s.io/kustomize/api/transform"
 	"sigs.k8s.io/kustomize/api/types"
+	"sigs.k8s.io/kustomize/kyaml/filtersutil"
 	"sigs.k8s.io/yaml"
 )
 
@@ -24,14 +25,16 @@ func (p *LabelTransformerPlugin) Config(
 }
 
 func (p *LabelTransformerPlugin) Transform(m resmap.ResMap) error {
-	t, err := transform.NewMapTransformer(
-		p.FieldSpecs,
-		p.Labels,
-	)
-	if err != nil {
-		return err
+	for _, r := range m.Resources() {
+		err := filtersutil.ApplyToJSON(labels.Filter{
+			Labels:  p.Labels,
+			FsSlice: p.FieldSpecs,
+		}, r.Kunstructured)
+		if err != nil {
+			return err
+		}
 	}
-	return t.Transform(m)
+	return nil
 }
 
 func NewLabelTransformerPlugin() resmap.TransformerPlugin {

api/builtins/NamespaceTransformer.go

@@ -6,12 +6,13 @@ package builtins
 import (
 	"fmt"
 
-	"sigs.k8s.io/kustomize/api/transform"
-
+	"sigs.k8s.io/kustomize/api/filters/namespace"
 	"sigs.k8s.io/kustomize/api/resid"
 	"sigs.k8s.io/kustomize/api/resmap"
 	"sigs.k8s.io/kustomize/api/resource"
+	"sigs.k8s.io/kustomize/api/transform"
 	"sigs.k8s.io/kustomize/api/types"
+	"sigs.k8s.io/kustomize/kyaml/filtersutil"
 	"sigs.k8s.io/yaml"
 )
 
@@ -19,6 +20,11 @@ import (
 type NamespaceTransformerPlugin struct {
 	types.ObjectMeta `json:"metadata,omitempty" yaml:"metadata,omitempty" protobuf:"bytes,1,opt,name=metadata"`
 	FieldSpecs       []types.FieldSpec `json:"fieldSpecs,omitempty" yaml:"fieldSpecs,omitempty"`
+
+	// YAMLSupport can be set to true to use the kyaml filter instead of the
+	// kunstruct transformer.
+	// TODO: change the default to use kyaml when it is stable
+	YAMLSupport bool `json:"yamlSupport,omitempty" yaml:"yamlSupport,omitempty"`
 }
 
 func (p *NamespaceTransformerPlugin) Config(
@@ -39,27 +45,38 @@ func (p *NamespaceTransformerPlugin) Transform(m resmap.ResMap) error {
 		}
 
 		id := r.OrgId()
-		applicableFs := p.applicableFieldSpecs(id)
 
-		for _, fs := range applicableFs {
-			err := transform.MutateField(
-				r.Map(), fs.PathSlice(), fs.CreateIfNotPresent,
-				p.changeNamespace(r))
+		if p.YAMLSupport {
+			// use the new style transform
+			err := filtersutil.ApplyToJSON(namespace.Filter{
+				Namespace: p.Namespace,
+				FsSlice:   p.FieldSpecs,
+			}, r.Kunstructured)
 			if err != nil {
 				return err
 			}
+		} else {
+			// use the old style transform
+			applicableFs := p.applicableFieldSpecs(id)
+			for _, fs := range applicableFs {
+				err := transform.MutateField(
+					r.Map(), fs.PathSlice(), fs.CreateIfNotPresent,
+					p.changeNamespace(r))
+				if err != nil {
+					return err
+				}
+			}
 		}
 
 		matches := m.GetMatchingResourcesByCurrentId(r.CurId().Equals)
 		if len(matches) != 1 {
-			return fmt.Errorf("namespace tranformation produces ID conflict: %#v", matches)
+			return fmt.Errorf(
+				"namespace transformation produces ID conflict: %+v", matches)
 		}
 	}
 	return nil
 }
 
-const metaNamespace = "metadata/namespace"
-
 // Special casing metadata.namespace since
 // all objects have it, even "ClusterKind" objects
 // that don't exist in a namespace (the Namespace
@@ -67,7 +84,9 @@ const metaNamespace = "metadata/namespace"
 func (p *NamespaceTransformerPlugin) applicableFieldSpecs(id resid.ResId) []types.FieldSpec {
 	var res []types.FieldSpec
 	for _, fs := range p.FieldSpecs {
-		if id.IsSelected(&fs.Gvk) && (fs.Path != metaNamespace || (fs.Path == metaNamespace && id.IsNamespaceableKind())) {
+		if id.IsSelected(&fs.Gvk) &&
+			(fs.Path != types.MetadataNamespacePath ||
+				(fs.Path == types.MetadataNamespacePath && id.IsNamespaceableKind())) {
 			res = append(res, fs)
 		}
 	}

api/builtins/PatchJson6902Transformer.go

@@ -8,10 +8,12 @@ import (
 
 	jsonpatch "github.com/evanphx/json-patch"
 	"github.com/pkg/errors"
+	"sigs.k8s.io/kustomize/api/filters/patchjson6902"
 	"sigs.k8s.io/kustomize/api/ifc"
 	"sigs.k8s.io/kustomize/api/resid"
 	"sigs.k8s.io/kustomize/api/resmap"
 	"sigs.k8s.io/kustomize/api/types"
+	"sigs.k8s.io/kustomize/kyaml/filtersutil"
 	"sigs.k8s.io/yaml"
 )
 
@@ -21,6 +23,8 @@ type PatchJson6902TransformerPlugin struct {
 	Target       types.PatchTarget `json:"target,omitempty" yaml:"target,omitempty"`
 	Path         string            `json:"path,omitempty" yaml:"path,omitempty"`
 	JsonOp       string            `json:"jsonOp,omitempty" yaml:"jsonOp,omitempty"`
+
+	YAMLSupport bool `json:"yamlSupport,omitempty" yaml:"yamlSupport,omitempty"`
 }
 
 func (p *PatchJson6902TransformerPlugin) Config(
@@ -83,16 +87,22 @@ func (p *PatchJson6902TransformerPlugin) Transform(m resmap.ResMap) error {
 	if err != nil {
 		return err
 	}
-	rawObj, err := obj.MarshalJSON()
-	if err != nil {
-		return err
+	if !p.YAMLSupport {
+		rawObj, err := obj.MarshalJSON()
+		if err != nil {
+			return err
+		}
+		modifiedObj, err := p.decodedPatch.Apply(rawObj)
+		if err != nil {
+			return errors.Wrapf(
+				err, "failed to apply json patch '%s'", p.JsonOp)
+		}
+		return obj.UnmarshalJSON(modifiedObj)
+	} else {
+		return filtersutil.ApplyToJSON(patchjson6902.Filter{
+			Patch: p.JsonOp,
+		}, obj.Kunstructured)
 	}
-	modifiedObj, err := p.decodedPatch.Apply(rawObj)
-	if err != nil {
-		return errors.Wrapf(
-			err, "failed to apply json patch '%s'", p.JsonOp)
-	}
-	return obj.UnmarshalJSON(modifiedObj)
 }
 
 func NewPatchJson6902TransformerPlugin() resmap.TransformerPlugin {

api/builtins/PatchStrategicMergeTransformer.go

@@ -4,11 +4,16 @@
 package builtins
 
 import (
+	"encoding/json"
 	"fmt"
 
+	kyaml "sigs.k8s.io/kustomize/kyaml/yaml"
+
+	"sigs.k8s.io/kustomize/api/filters/patchstrategicmerge"
 	"sigs.k8s.io/kustomize/api/resmap"
 	"sigs.k8s.io/kustomize/api/resource"
 	"sigs.k8s.io/kustomize/api/types"
+	"sigs.k8s.io/kustomize/kyaml/filtersutil"
 	"sigs.k8s.io/yaml"
 )
 
@@ -17,6 +22,8 @@ type PatchStrategicMergeTransformerPlugin struct {
 	loadedPatches []*resource.Resource
 	Paths         []types.PatchStrategicMerge `json:"paths,omitempty" yaml:"paths,omitempty"`
 	Patches       string                      `json:"patches,omitempty" yaml:"patches,omitempty"`
+
+	YAMLSupport bool `json:"yamlSupport,omitempty" yaml:"yamlSupport,omitempty"`
 }
 
 func (p *PatchStrategicMergeTransformerPlugin) Config(
@@ -69,22 +76,42 @@ func (p *PatchStrategicMergeTransformerPlugin) Transform(m resmap.ResMap) error
 		if err != nil {
 			return err
 		}
-		err = target.Patch(patch.Kunstructured)
-		if err != nil {
-			return err
-		}
-		// remove the resource from resmap
-		// when the patch is to $patch: delete that target
-		if len(target.Map()) == 0 {
-			err = m.Remove(target.CurId())
+		if !p.YAMLSupport {
+			err = target.Patch(patch.Kunstructured)
 			if err != nil {
 				return err
 			}
+			// remove the resource from resmap
+			// when the patch is to $patch: delete that target
+			if len(target.Map()) == 0 {
+				err = m.Remove(target.CurId())
+				if err != nil {
+					return err
+				}
+			}
+		} else {
+			node, err := getRNode(patch)
+			if err != nil {
+				return err
+			}
+			err = filtersutil.ApplyToJSON(patchstrategicmerge.Filter{
+				Patch: node,
+			}, target.Kunstructured)
 		}
 	}
 	return nil
 }
 
+//TODO: Remove this once the next version of kyaml is released which
+// exposes GetRNode from the filutersutil package.
+func getRNode(k json.Marshaler) (*kyaml.RNode, error) {
+	j, err := k.MarshalJSON()
+	if err != nil {
+		return nil, err
+	}
+	return kyaml.Parse(string(j))
+}
+
 func NewPatchStrategicMergeTransformerPlugin() resmap.TransformerPlugin {
 	return &PatchStrategicMergeTransformerPlugin{}
 }

api/builtins/PatchTransformer.go

@@ -5,12 +5,16 @@ package builtins
 
 import (
 	"fmt"
+	"strings"
 
 	jsonpatch "github.com/evanphx/json-patch"
 	"github.com/pkg/errors"
+	"sigs.k8s.io/kustomize/api/filters/patchjson6902"
+	"sigs.k8s.io/kustomize/api/filters/patchstrategicmerge"
 	"sigs.k8s.io/kustomize/api/resmap"
 	"sigs.k8s.io/kustomize/api/resource"
 	"sigs.k8s.io/kustomize/api/types"
+	"sigs.k8s.io/kustomize/kyaml/filtersutil"
 	"sigs.k8s.io/yaml"
 )
 
@@ -20,69 +24,112 @@ type PatchTransformerPlugin struct {
 	Path         string          `json:"path,omitempty" yaml:"path,omitempty"`
 	Patch        string          `json:"patch,omitempty" yaml:"patch,omitempty"`
 	Target       *types.Selector `json:"target,omitempty" yaml:"target,omitempty"`
+
+	YAMLSupport bool `json:"yamlSupport,omitempty" yaml:"yamlSupport,omitempty"`
 }
 
 func (p *PatchTransformerPlugin) Config(
-	h *resmap.PluginHelpers, c []byte) (err error) {
-	err = yaml.Unmarshal(c, p)
+	h *resmap.PluginHelpers, c []byte) error {
+	err := yaml.Unmarshal(c, p)
 	if err != nil {
 		return err
 	}
+	p.Patch = strings.TrimSpace(p.Patch)
 	if p.Patch == "" && p.Path == "" {
-		err = fmt.Errorf(
+		return fmt.Errorf(
 			"must specify one of patch and path in\n%s", string(c))
-		return
 	}
 	if p.Patch != "" && p.Path != "" {
-		err = fmt.Errorf(
+		return fmt.Errorf(
 			"patch and path can't be set at the same time\n%s", string(c))
-		return
-	}
-	var in []byte
-	if p.Path != "" {
-		in, err = h.Loader().Load(p.Path)
-		if err != nil {
-			return
-		}
-	}
-	if p.Patch != "" {
-		in = []byte(p.Patch)
 	}
 
-	patchSM, errSM := h.ResmapFactory().RF().FromBytes(in)
-	patchJson, errJson := jsonPatchFromBytes(in)
+	if p.Path != "" {
+		loaded, loadErr := h.Loader().Load(p.Path)
+		if loadErr != nil {
+			return loadErr
+		}
+		p.Patch = string(loaded)
+	}
+
+	patchSM, errSM := h.ResmapFactory().RF().FromBytes([]byte(p.Patch))
+	patchJson, errJson := jsonPatchFromBytes([]byte(p.Patch))
+	if (errSM == nil && errJson == nil) ||
+		(patchSM != nil && patchJson != nil) {
+		return fmt.Errorf(
+			"illegally qualifies as both an SM and JSON patch: [%v]",
+			p.Patch)
+	}
 	if errSM != nil && errJson != nil {
-		err = fmt.Errorf(
-			"unable to get either a Strategic Merge Patch or JSON patch 6902 from %s", p.Patch)
-		return
+		return fmt.Errorf(
+			"unable to parse SM or JSON patch from [%v]", p.Patch)
 	}
-	if errSM == nil && errJson != nil {
+	if errSM == nil {
 		p.loadedPatch = patchSM
-	}
-	if errJson == nil && errSM != nil {
+	} else {
 		p.decodedPatch = patchJson
 	}
-	if patchSM != nil && patchJson != nil {
-		err = fmt.Errorf(
-			"a patch can't be both a Strategic Merge Patch and JSON patch 6902 %s", p.Patch)
-	}
-
 	return nil
 }
 
 func (p *PatchTransformerPlugin) Transform(m resmap.ResMap) error {
-	if p.loadedPatch != nil && p.Target == nil {
-		target, err := m.GetById(p.loadedPatch.OrgId())
+	if p.loadedPatch != nil {
+		// The patch was a strategic merge patch
+		return p.transformStrategicMerge(m, p.loadedPatch)
+	} else {
+		return p.transformJson6902(m, p.decodedPatch)
+	}
+}
+
+// transformStrategicMerge applies the provided strategic merge patch
+// to all the resources in the ResMap that match either the Target or
+// the identifier of the patch.
+func (p *PatchTransformerPlugin) transformStrategicMerge(m resmap.ResMap, patch *resource.Resource) error {
+	if p.Target == nil {
+		target, err := m.GetById(patch.OrgId())
 		if err != nil {
 			return err
 		}
-		err = target.Patch(p.loadedPatch.Kunstructured)
-		if err != nil {
-			return err
-		}
-		return nil
+		return p.applySMPatch(target, patch)
 	}
 
+	resources, err := m.Select(*p.Target)
+	if err != nil {
+		return err
+	}
+	for _, res := range resources {
+		patchCopy := patch.DeepCopy()
+		patchCopy.SetName(res.GetName())
+		patchCopy.SetNamespace(res.GetNamespace())
+		patchCopy.SetGvk(res.GetGvk())
+		err := p.applySMPatch(res, patchCopy)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// applySMPatch applies the provided strategic merge patch to the
+// given resource. Depending on the value of YAMLSupport, it will either
+// use the legacy implementation or the kyaml-based solution.
+func (p *PatchTransformerPlugin) applySMPatch(resource, patch *resource.Resource) error {
+	if !p.YAMLSupport {
+		return resource.Patch(patch.Kunstructured)
+	} else {
+		node, err := filtersutil.GetRNode(patch)
+		if err != nil {
+			return err
+		}
+		return filtersutil.ApplyToJSON(patchstrategicmerge.Filter{
+			Patch: node,
+		}, resource.Kunstructured)
+	}
+}
+
+// transformJson6902 applies the provided json6902 patch
+// to all the resources in the ResMap that match the Target.
+func (p *PatchTransformerPlugin) transformJson6902(m resmap.ResMap, patch jsonpatch.Patch) error {
 	if p.Target == nil {
 		return fmt.Errorf("must specify a target for patch %s", p.Patch)
 	}
@@ -92,35 +139,36 @@ func (p *PatchTransformerPlugin) Transform(m resmap.ResMap) error {
 		return err
 	}
 	for _, res := range resources {
-		if p.decodedPatch != nil {
-			rawObj, err := res.MarshalJSON()
-			if err != nil {
-				return err
-			}
-			modifiedObj, err := p.decodedPatch.Apply(rawObj)
-			if err != nil {
-				return errors.Wrapf(
-					err, "failed to apply json patch '%s'", p.Patch)
-			}
-			err = res.UnmarshalJSON(modifiedObj)
-			if err != nil {
-				return err
-			}
-		}
-		if p.loadedPatch != nil {
-			patchCopy := p.loadedPatch.DeepCopy()
-			patchCopy.SetName(res.GetName())
-			patchCopy.SetNamespace(res.GetNamespace())
-			patchCopy.SetGvk(res.GetGvk())
-			err = res.Patch(patchCopy.Kunstructured)
-			if err != nil {
-				return err
-			}
+		err = p.applyJson6902Patch(res, patch)
+		if err != nil {
+			return err
 		}
 	}
 	return nil
 }
 
+// applyJson6902Patch applies the provided patch to the given resource.
+// Depending on the value of YAMLSupport, it will either
+// use the legacy implementation or the kyaml-based solution.
+func (p *PatchTransformerPlugin) applyJson6902Patch(resource *resource.Resource, patch jsonpatch.Patch) error {
+	if !p.YAMLSupport {
+		rawObj, err := resource.MarshalJSON()
+		if err != nil {
+			return err
+		}
+		modifiedObj, err := patch.Apply(rawObj)
+		if err != nil {
+			return errors.Wrapf(
+				err, "failed to apply json patch '%s'", p.Patch)
+		}
+		return resource.UnmarshalJSON(modifiedObj)
+	} else {
+		return filtersutil.ApplyToJSON(patchjson6902.Filter{
+			Patch: p.Patch,
+		}, resource.Kunstructured)
+	}
+}
+
 // jsonPatchFromBytes loads a Json 6902 patch from
 // a bytes input
 func jsonPatchFromBytes(

api/builtins/SecretGenerator.go

@@ -13,12 +13,10 @@ import (
 type SecretGeneratorPlugin struct {
 	h                *resmap.PluginHelpers
 	types.ObjectMeta `json:"metadata,omitempty" yaml:"metadata,omitempty" protobuf:"bytes,1,opt,name=metadata"`
-	types.GeneratorOptions
 	types.SecretArgs
 }
 
 func (p *SecretGeneratorPlugin) Config(h *resmap.PluginHelpers, config []byte) (err error) {
-	p.GeneratorOptions = types.GeneratorOptions{}
 	p.SecretArgs = types.SecretArgs{}
 	err = yaml.Unmarshal(config, p)
 	if p.SecretArgs.Name == "" {
@@ -33,8 +31,7 @@ func (p *SecretGeneratorPlugin) Config(h *resmap.PluginHelpers, config []byte) (
 
 func (p *SecretGeneratorPlugin) Generate() (resmap.ResMap, error) {
 	return p.h.ResmapFactory().FromSecretArgs(
-		kv.NewLoader(p.h.Loader(), p.h.Validator()),
-		&p.GeneratorOptions, p.SecretArgs)
+		kv.NewLoader(p.h.Loader(), p.h.Validator()), p.SecretArgs)
 }
 
 func NewSecretGeneratorPlugin() resmap.GeneratorPlugin {

api/builtins/ValueAddTransformer.go

@@ -0,0 +1,142 @@
+// Code generated by pluginator on ValueAddTransformer; DO NOT EDIT.
+// pluginator {unknown  1970-01-01T00:00:00Z  }
+
+package builtins
+
+import (
+	"fmt"
+	"path/filepath"
+	"strings"
+
+	"sigs.k8s.io/kustomize/api/filters/namespace"
+	"sigs.k8s.io/kustomize/api/filters/valueadd"
+	"sigs.k8s.io/kustomize/api/resmap"
+	"sigs.k8s.io/kustomize/api/resource"
+	"sigs.k8s.io/kustomize/api/types"
+	"sigs.k8s.io/kustomize/kyaml/filtersutil"
+	"sigs.k8s.io/yaml"
+)
+
+// An 'Add' transformer inspired by the IETF RFC 6902 JSON spec Add operation.
+type ValueAddTransformerPlugin struct {
+	// Value is the value to add.
+	// Defaults to base name of encompassing kustomization root.
+	Value string `json:"value,omitempty" yaml:"value,omitempty"`
+
+	// Targets is a slice of targets that should have the value added.
+	Targets []Target `json:"targets,omitempty" yaml:"targets,omitempty"`
+
+	// TargetFilePath is a file path.  If specified, the file will be parsed into
+	// a slice of Target, and appended to anything that was specified in the
+	// Targets field.  This is just a means to share common target specifications.
+	TargetFilePath string `json:"targetFilePath,omitempty" yaml:"targetFilePath,omitempty"`
+}
+
+// Target describes where to put the value.
+type Target struct {
+	// Selector selects the resources to modify.
+	Selector *types.Selector `json:"selector,omitempty" yaml:"selector,omitempty"`
+
+	// NotSelector selects the resources to exclude
+	// from those included by overly broad selectors.
+	// TODO: implement this?
+	// NotSelector *types.Selector `json:"notSelector,omitempty" yaml:"notSelector,omitempty"`
+
+	// FieldPath is a JSON-style path to the field intended to hold the value.
+	FieldPath string `json:"fieldPath,omitempty" yaml:"fieldPath,omitempty"`
+
+	// FilePathPosition is passed to the filter directly.  Look there for doc.
+	FilePathPosition int `json:"filePathPosition,omitempty" yaml:"filePathPosition,omitempty"`
+}
+
+func (p *ValueAddTransformerPlugin) Config(h *resmap.PluginHelpers, c []byte) error {
+	err := yaml.Unmarshal(c, p)
+	if err != nil {
+		return err
+	}
+	p.Value = strings.TrimSpace(p.Value)
+	if p.Value == "" {
+		p.Value = filepath.Base(h.Loader().Root())
+	}
+	if p.TargetFilePath != "" {
+		bytes, err := h.Loader().Load(p.TargetFilePath)
+		if err != nil {
+			return err
+		}
+		var targets struct {
+			Targets []Target `json:"targets,omitempty" yaml:"targets,omitempty"`
+		}
+		err = yaml.Unmarshal(bytes, &targets)
+		if err != nil {
+			return err
+		}
+		p.Targets = append(p.Targets, targets.Targets...)
+	}
+	if len(p.Targets) == 0 {
+		return fmt.Errorf("must specify at least one target")
+	}
+	for _, target := range p.Targets {
+		if err = validateSelector(target.Selector); err != nil {
+			return err
+		}
+		// TODO: call validateSelector(target.NotSelector) if field added.
+		if err = validateJsonFieldPath(target.FieldPath); err != nil {
+			return err
+		}
+		if target.FilePathPosition < 0 {
+			return fmt.Errorf(
+				"value of FilePathPosition (%d) cannot be negative",
+				target.FilePathPosition)
+		}
+	}
+	return nil
+}
+
+// TODO: implement
+func validateSelector(_ *types.Selector) error {
+	return nil
+}
+
+// TODO: Enforce RFC 6902?
+func validateJsonFieldPath(p string) error {
+	if len(p) == 0 {
+		return fmt.Errorf("fieldPath cannot be empty")
+	}
+	return nil
+}
+
+func (p *ValueAddTransformerPlugin) Transform(m resmap.ResMap) (err error) {
+	for _, t := range p.Targets {
+		var resources []*resource.Resource
+		if t.Selector == nil {
+			resources = m.Resources()
+		} else {
+			resources, err = m.Select(*t.Selector)
+			if err != nil {
+				return err
+			}
+		}
+		// TODO: consider t.NotSelector if implemented
+		for _, res := range resources {
+			if t.FieldPath == types.MetadataNamespacePath {
+				err = filtersutil.ApplyToJSON(namespace.Filter{
+					Namespace: p.Value,
+				}, res.Kunstructured)
+			} else {
+				err = filtersutil.ApplyToJSON(valueadd.Filter{
+					Value:            p.Value,
+					FieldPath:        t.FieldPath,
+					FilePathPosition: t.FilePathPosition,
+				}, res.Kunstructured)
+			}
+			if err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+func NewValueAddTransformerPlugin() resmap.TransformerPlugin {
+	return &ValueAddTransformerPlugin{}
+}

api/filesys/util.go

@@ -3,7 +3,11 @@
 
 package filesys
 
-import "path/filepath"
+import (
+	"os"
+	"path/filepath"
+	"strings"
+)
 
 // RootedPath returns a rooted path, e.g. "/foo/bar" as
 // opposed to "foo/bar".
@@ -28,3 +32,94 @@ func StripLeadingSeps(s string) string {
 	}
 	return s[k:]
 }
+
+// PathSplit converts a file path to a slice of string.
+// If the path is absolute (if the path has a leading slash),
+// then the first entry in the result is an empty string.
+// Desired:  path == PathJoin(PathSplit(path))
+func PathSplit(incoming string) []string {
+	if incoming == "" {
+		return []string{}
+	}
+	dir, path := filepath.Split(incoming)
+	if dir == string(os.PathSeparator) {
+		if path == "" {
+			return []string{""}
+		}
+		return []string{"", path}
+	}
+	dir = strings.TrimSuffix(dir, string(os.PathSeparator))
+	if dir == "" {
+		return []string{path}
+	}
+	return append(PathSplit(dir), path)
+}
+
+// PathJoin converts a slice of string to a file path.
+// If the first entry is an empty string, then the returned
+// path is absolute (it has a leading slash).
+// Desired:  path == PathJoin(PathSplit(path))
+func PathJoin(incoming []string) string {
+	if len(incoming) == 0 {
+		return ""
+	}
+	if incoming[0] == "" {
+		return string(os.PathSeparator) + filepath.Join(incoming[1:]...)
+	}
+	return filepath.Join(incoming...)
+}
+
+// InsertPathPart inserts 'part' at position 'pos' in the given filepath.
+// The first position is 0.
+//
+// E.g. if part == 'PEACH'
+//
+//                  OLD : NEW                    : POS
+//      --------------------------------------------------------
+//              {empty} : PEACH                  : irrelevant
+//                    / : /PEACH                 : irrelevant
+//                  pie : PEACH/pie              : 0 (or negative)
+//                 /pie : /PEACH/pie             : 0 (or negative)
+//                  raw : raw/PEACH              : 1 (or larger)
+//                 /raw : /raw/PEACH             : 1 (or larger)
+//      a/nice/warm/pie : a/nice/warm/PEACH/pie  : 3
+//     /a/nice/warm/pie : /a/nice/warm/PEACH/pie : 3
+//
+// * An empty part results in no change.
+//
+// * Absolute paths get their leading '/' stripped, treated like
+//   relative paths, and the leading '/' is re-added on output.
+//   The meaning of pos is intentionally the same in either absolute or
+//   relative paths; if it weren't, this function could convert absolute
+//   paths to relative paths, which is not desirable.
+//
+// * For robustness (liberal input, conservative output) Pos values that
+//   that are too small (large) to index the split filepath result in a
+//   prefix (postfix) rather than an error.  Use extreme position values
+//   to assure a prefix or postfix (e.g. 0 will always prefix, and
+//   9999 will presumably always postfix).
+func InsertPathPart(path string, pos int, part string) string {
+	if part == "" {
+		return path
+	}
+	parts := PathSplit(path)
+	if pos < 0 {
+		pos = 0
+	} else if pos > len(parts) {
+		pos = len(parts)
+	}
+	if len(parts) > 0 && parts[0] == "" && pos < len(parts) {
+		// An empty string at 0 indicates an absolute path, and means
+		// we must increment pos.  This change means that a position
+		// specification has the same meaning in relative and absolute paths.
+		// E.g. in either the path 'a/b/c' or the path '/a/b/c',
+		// 'a' is at 0, 'b' is at 1 and 'c' is at 2, and inserting at
+		// zero means a new first field _without_ changing an absolute
+		// path to a relative path.
+		pos++
+	}
+	result := make([]string, len(parts)+1)
+	copy(result, parts[0:pos])
+	result[pos] = part
+	return PathJoin(append(result, parts[pos:]...))
+}

api/filesys/util_test.go

@@ -1,6 +1,7 @@
 package filesys_test
 
 import (
+	"os"
 	"path/filepath"
 	"testing"
 
@@ -93,6 +94,11 @@ func TestFilePathSplit(t *testing.T) {
 			dir:  "",
 			file: "rabbit.jpg",
 		},
+		{
+			full: "/",
+			dir:  "/",
+			file: "",
+		},
 		{
 			full: "/beans",
 			dir:  "/",
@@ -124,6 +130,169 @@ func TestFilePathSplit(t *testing.T) {
 	}
 }
 
+func TestPathSplitAndJoin(t *testing.T) {
+	cases := map[string]struct {
+		original string
+		expected []string
+	}{
+		"Empty": {
+			original: "",
+			expected: []string{},
+		},
+		"One": {
+			original: "hello",
+			expected: []string{"hello"},
+		},
+		"Two": {
+			original: "hello/there",
+			expected: []string{"hello", "there"},
+		},
+		"Three": {
+			original: "hello/my/friend",
+			expected: []string{"hello", "my", "friend"},
+		},
+	}
+	for n, c := range cases {
+		f := func(t *testing.T, original string, expected []string) {
+			actual := PathSplit(original)
+			if len(actual) != len(expected) {
+				t.Fatalf(
+					"expected len %d, got len %d",
+					len(expected), len(actual))
+			}
+			for i := range expected {
+				if expected[i] != actual[i] {
+					t.Fatalf(
+						"at i=%d, expected '%s', got '%s'",
+						i, expected[i], actual[i])
+				}
+			}
+			joined := PathJoin(actual)
+			if joined != original {
+				t.Fatalf(
+					"when rejoining, expected '%s', got '%s'",
+					original, joined)
+			}
+		}
+		t.Run("relative"+n, func(t *testing.T) {
+			f(t, c.original, c.expected)
+		})
+		t.Run("absolute"+n, func(t *testing.T) {
+			f(t,
+				string(os.PathSeparator)+c.original,
+				append([]string{""}, c.expected...))
+		})
+	}
+}
+
+func TestInsertPathPart(t *testing.T) {
+	cases := map[string]struct {
+		original string
+		pos      int
+		part     string
+		expected string
+	}{
+		"rootOne": {
+			original: "/",
+			pos:      0,
+			part:     "___",
+			expected: "/___",
+		},
+		"rootTwo": {
+			original: "/",
+			pos:      444,
+			part:     "___",
+			expected: "/___",
+		},
+		"rootedFirst": {
+			original: "/apple",
+			pos:      0,
+			part:     "___",
+			expected: "/___/apple",
+		},
+		"rootedSecond": {
+			original: "/apple",
+			pos:      444,
+			part:     "___",
+			expected: "/apple/___",
+		},
+		"rootedThird": {
+			original: "/apple/banana",
+			pos:      444,
+			part:     "___",
+			expected: "/apple/banana/___",
+		},
+		"emptyLow": {
+			original: "",
+			pos:      -3,
+			part:     "___",
+			expected: "___",
+		},
+		"emptyHigh": {
+			original: "",
+			pos:      444,
+			part:     "___",
+			expected: "___",
+		},
+		"peachPie": {
+			original: "a/nice/warm/pie",
+			pos:      3,
+			part:     "PEACH",
+			expected: "a/nice/warm/PEACH/pie",
+		},
+		"rootedPeachPie": {
+			original: "/a/nice/warm/pie",
+			pos:      3,
+			part:     "PEACH",
+			expected: "/a/nice/warm/PEACH/pie",
+		},
+		"longStart": {
+			original: "a/b/c/d/e/f",
+			pos:      0,
+			part:     "___",
+			expected: "___/a/b/c/d/e/f",
+		},
+		"rootedLongStart": {
+			original: "/a/b/c/d/e/f",
+			pos:      0,
+			part:     "___",
+			expected: "/___/a/b/c/d/e/f",
+		},
+		"longMiddle": {
+			original: "a/b/c/d/e/f",
+			pos:      3,
+			part:     "___",
+			expected: "a/b/c/___/d/e/f",
+		},
+		"rootedLongMiddle": {
+			original: "/a/b/c/d/e/f",
+			pos:      3,
+			part:     "___",
+			expected: "/a/b/c/___/d/e/f",
+		},
+		"longEnd": {
+			original: "a/b/c/d/e/f",
+			pos:      444,
+			part:     "___",
+			expected: "a/b/c/d/e/f/___",
+		},
+		"rootedLongEnd": {
+			original: "/a/b/c/d/e/f",
+			pos:      444,
+			part:     "___",
+			expected: "/a/b/c/d/e/f/___",
+		},
+	}
+	for n, c := range cases {
+		t.Run(n, func(t *testing.T) {
+			actual := InsertPathPart(c.original, c.pos, c.part)
+			if actual != c.expected {
+				t.Fatalf("expected '%s', got '%s'", c.expected, actual)
+			}
+		})
+	}
+}
+
 func TestStripTrailingSeps(t *testing.T) {
 	cases := []struct {
 		full string

api/filters/annotations/annotations.go

@@ -0,0 +1,42 @@
+// Copyright 2020 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package annotations
+
+import (
+	"sigs.k8s.io/kustomize/api/filters/filtersutil"
+	"sigs.k8s.io/kustomize/api/filters/fsslice"
+	"sigs.k8s.io/kustomize/api/types"
+	"sigs.k8s.io/kustomize/kyaml/kio"
+	"sigs.k8s.io/kustomize/kyaml/yaml"
+)
+
+type annoMap map[string]string
+
+type Filter struct {
+	// Annotations is the set of annotations to apply to the inputs
+	Annotations annoMap `yaml:"annotations,omitempty"`
+
+	// FsSlice contains the FieldSpecs to locate the namespace field
+	FsSlice types.FsSlice
+}
+
+var _ kio.Filter = Filter{}
+
+func (f Filter) Filter(nodes []*yaml.RNode) ([]*yaml.RNode, error) {
+	keys := filtersutil.SortedMapKeys(f.Annotations)
+	_, err := kio.FilterAll(yaml.FilterFunc(
+		func(node *yaml.RNode) (*yaml.RNode, error) {
+			for _, k := range keys {
+				if err := node.PipeE(fsslice.Filter{
+					FsSlice:    f.FsSlice,
+					SetValue:   fsslice.SetEntry(k, f.Annotations[k]),
+					CreateKind: yaml.MappingNode, // Annotations are MappingNodes.
+				}); err != nil {
+					return nil, err
+				}
+			}
+			return node, nil
+		})).Filter(nodes)
+	return nodes, err
+}

api/filters/annotations/annotations_test.go

@@ -0,0 +1,139 @@
+// Copyright 2020 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package annotations
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"sigs.k8s.io/kustomize/api/internal/plugins/builtinconfig"
+	filtertest_test "sigs.k8s.io/kustomize/api/testutils/filtertest"
+	"sigs.k8s.io/kustomize/api/types"
+)
+
+var annosFs = builtinconfig.MakeDefaultConfig().CommonAnnotations
+
+func TestAnnotations_Filter(t *testing.T) {
+	testCases := map[string]struct {
+		input          string
+		expectedOutput string
+		filter         Filter
+		fsslice        types.FsSlice
+	}{
+		"add": {
+			input: `
+apiVersion: example.com/v1
+kind: Foo
+metadata:
+  name: instance
+  annotations:
+    hero: batman
+    fiend: riddler
+`,
+			expectedOutput: `
+apiVersion: example.com/v1
+kind: Foo
+metadata:
+  name: instance
+  annotations:
+    hero: batman
+    fiend: riddler
+    auto: ford
+    bean: cannellini
+    clown: emmett kelley
+    dragon: smaug
+`,
+			filter: Filter{Annotations: annoMap{
+				"clown":  "emmett kelley",
+				"auto":   "ford",
+				"dragon": "smaug",
+				"bean":   "cannellini",
+			}},
+		},
+		"update": {
+			input: `
+apiVersion: example.com/v1
+kind: Foo
+metadata:
+  name: instance
+  annotations:
+    hero: batman
+    fiend: riddler
+`,
+			expectedOutput: `
+apiVersion: example.com/v1
+kind: Foo
+metadata:
+  name: instance
+  annotations:
+    hero: superman
+    fiend: luthor
+    bean: cannellini
+    clown: emmett kelley
+`,
+			filter: Filter{Annotations: annoMap{
+				"clown": "emmett kelley",
+				"hero":  "superman",
+				"fiend": "luthor",
+				"bean":  "cannellini",
+			}},
+		},
+		"data-fieldspecs": {
+			input: `
+apiVersion: example.com/v1
+kind: Foo
+metadata:
+  name: instance
+---
+apiVersion: example.com/v1
+kind: Bar
+metadata:
+  name: instance
+`,
+			expectedOutput: `
+apiVersion: example.com/v1
+kind: Foo
+metadata:
+  name: instance
+  annotations:
+    sleater: kinney
+a:
+  b:
+    sleater: kinney
+---
+apiVersion: example.com/v1
+kind: Bar
+metadata:
+  name: instance
+  annotations:
+    sleater: kinney
+a:
+  b:
+    sleater: kinney
+`,
+			filter: Filter{Annotations: annoMap{
+				"sleater": "kinney",
+			}},
+			fsslice: []types.FieldSpec{
+				{
+					Path:               "a/b",
+					CreateIfNotPresent: true,
+				},
+			},
+		},
+	}
+
+	for tn, tc := range testCases {
+		t.Run(tn, func(t *testing.T) {
+			filter := tc.filter
+			filter.FsSlice = append(annosFs, tc.fsslice...)
+			if !assert.Equal(t,
+				strings.TrimSpace(tc.expectedOutput),
+				strings.TrimSpace(filtertest_test.RunFilter(t, tc.input, filter))) {
+				t.FailNow()
+			}
+		})
+	}
+}

api/filters/annotations/doc.go

@@ -0,0 +1,6 @@
+// Copyright 2020 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+// Package annotations contains a kio.Filter implementation of the kustomize
+// annotations transformer.
+package annotations

api/filters/annotations/example_test.go

@@ -0,0 +1,55 @@
+// Copyright 2020 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package annotations
+
+import (
+	"bytes"
+	"log"
+	"os"
+
+	"sigs.k8s.io/kustomize/api/internal/plugins/builtinconfig"
+	"sigs.k8s.io/kustomize/kyaml/kio"
+)
+
+func ExampleFilter() {
+	fss := builtinconfig.MakeDefaultConfig().CommonAnnotations
+	err := kio.Pipeline{
+		Inputs: []kio.Reader{&kio.ByteReader{Reader: bytes.NewBufferString(`
+apiVersion: example.com/v1
+kind: Foo
+metadata:
+  name: instance
+---
+apiVersion: example.com/v1
+kind: Bar
+metadata:
+  name: instance
+`)}},
+		Filters: []kio.Filter{Filter{
+			Annotations: map[string]string{
+				"foo": "bar",
+			},
+			FsSlice: fss,
+		}},
+		Outputs: []kio.Writer{kio.ByteWriter{Writer: os.Stdout}},
+	}.Execute()
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	// Output:
+	// apiVersion: example.com/v1
+	// kind: Foo
+	// metadata:
+	//   name: instance
+	//   annotations:
+	//     foo: bar
+	// ---
+	// apiVersion: example.com/v1
+	// kind: Bar
+	// metadata:
+	//   name: instance
+	//   annotations:
+	//     foo: bar
+}

api/filters/filtersutil/filtersutil.go

@@ -0,0 +1,21 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package filtersutil
+
+import (
+	"sort"
+)
+
+// SortedMapKeys returns a sorted slice of keys to the given map.
+// Writing this function never gets old.
+func SortedMapKeys(m map[string]string) []string {
+	keys := make([]string, len(m))
+	i := 0
+	for k := range m {
+		keys[i] = k
+		i++
+	}
+	sort.Strings(keys)
+	return keys
+}

api/filters/filtersutil/filtersutil_test.go

@@ -0,0 +1,34 @@
+package filtersutil_test
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"sigs.k8s.io/kustomize/api/filters/filtersutil"
+)
+
+func TestSortedKeys(t *testing.T) {
+	testCases := map[string]struct {
+		input    map[string]string
+		expected []string
+	}{
+		"empty": {
+			input:    map[string]string{},
+			expected: []string{}},
+		"one": {
+			input:    map[string]string{"a": "aaa"},
+			expected: []string{"a"}},
+		"three": {
+			input:    map[string]string{"c": "ccc", "b": "bbb", "a": "aaa"},
+			expected: []string{"a", "b", "c"}},
+	}
+	for tn, tc := range testCases {
+		t.Run(tn, func(t *testing.T) {
+			if !assert.Equal(t,
+				filtersutil.SortedMapKeys(tc.input),
+				tc.expected) {
+				t.FailNow()
+			}
+		})
+	}
+}

api/filters/fsslice/doc.go

@@ -0,0 +1,6 @@
+// Copyright 2020 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+// Package fsslice contains a yaml.Filter to modify a resource using an
+// FsSlice to identify fields to be updated within the resource.
+package fsslice

api/filters/fsslice/example_test.go

@@ -0,0 +1,62 @@
+// Copyright 2020 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package fsslice_test
+
+import (
+	"bytes"
+	"log"
+	"os"
+
+	"sigs.k8s.io/kustomize/api/filters/fsslice"
+	"sigs.k8s.io/kustomize/api/types"
+	"sigs.k8s.io/kustomize/kyaml/kio"
+	"sigs.k8s.io/kustomize/kyaml/yaml"
+)
+
+func ExampleFilter() {
+	in := &kio.ByteReader{
+		Reader: bytes.NewBufferString(`
+apiVersion: example.com/v1
+kind: Foo
+metadata:
+  name: instance
+---
+apiVersion: example.com/v1
+kind: Bar
+metadata:
+  name: instance
+`),
+	}
+	fltr := fsslice.Filter{
+		CreateKind: yaml.ScalarNode,
+		SetValue:   fsslice.SetScalar("green"),
+		FsSlice: []types.FieldSpec{
+			{Path: "a/b", CreateIfNotPresent: true},
+		},
+	}
+
+	err := kio.Pipeline{
+		Inputs:  []kio.Reader{in},
+		Filters: []kio.Filter{kio.FilterAll(fltr)},
+		Outputs: []kio.Writer{kio.ByteWriter{Writer: os.Stdout}},
+	}.Execute()
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	// Output:
+	// apiVersion: example.com/v1
+	// kind: Foo
+	// metadata:
+	//   name: instance
+	// a:
+	//   b: green
+	// ---
+	// apiVersion: example.com/v1
+	// kind: Bar
+	// metadata:
+	//   name: instance
+	// a:
+	//   b: green
+}

api/filters/fsslice/fieldspec_filter.go

@@ -0,0 +1,140 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package fsslice
+
+import (
+	"strings"
+
+	"sigs.k8s.io/kustomize/api/types"
+	"sigs.k8s.io/kustomize/kyaml/errors"
+	"sigs.k8s.io/kustomize/kyaml/yaml"
+)
+
+// fieldSpecFilter applies a single fieldSpec to a single object
+// fieldSpecFilter stores internal state and should not be reused
+type fieldSpecFilter struct {
+	// FieldSpec contains the path to the value to set.
+	FieldSpec types.FieldSpec `yaml:"fieldSpec"`
+
+	// Set the field using this function
+	SetValue SetFn
+
+	// CreateKind defines the type of node to create if the field is not found
+	CreateKind yaml.Kind
+
+	// path keeps internal state about the current path
+	path []string
+}
+
+func (fltr fieldSpecFilter) Filter(obj *yaml.RNode) (*yaml.RNode, error) {
+	// check if the FieldSpec applies to the object
+	if match, err := isMatchGVK(fltr.FieldSpec, obj); !match || err != nil {
+		return obj, errors.Wrap(err)
+	}
+	fltr.path = strings.Split(fltr.FieldSpec.Path, "/")
+	if err := fltr.filter(obj); err != nil {
+		s, _ := obj.String()
+		return nil, errors.WrapPrefixf(err,
+			"obj %v at path %v", s, fltr.FieldSpec.Path)
+	}
+	return obj, nil
+}
+
+func (fltr fieldSpecFilter) filter(obj *yaml.RNode) error {
+	if len(fltr.path) == 0 {
+		// found the field -- set its value
+		return fltr.SetValue(obj)
+	}
+	switch obj.YNode().Kind {
+	case yaml.SequenceNode:
+		return fltr.seq(obj)
+	case yaml.MappingNode:
+		return fltr.field(obj)
+	}
+	// not found -- this might be an error since the type doesn't match
+
+	return errors.Errorf("unsupported yaml node")
+}
+
+// field calls filter on the field matching the next path element
+func (fltr fieldSpecFilter) field(obj *yaml.RNode) error {
+	fieldName, isSeq := isSequenceField(fltr.path[0])
+
+	// lookup the field matching the next path element
+	var lookupField yaml.Filter
+	switch {
+	case !fltr.FieldSpec.CreateIfNotPresent || fltr.CreateKind == 0 || isSeq:
+		// dont' create the field if we don't find it
+		lookupField = yaml.Lookup(fieldName)
+	case len(fltr.path) <= 1:
+		// create the field if it is missing: use the provided node kind
+		lookupField = yaml.LookupCreate(fltr.CreateKind, fieldName)
+	default:
+		// create the field if it is missing: must be a mapping node
+		lookupField = yaml.LookupCreate(yaml.MappingNode, fieldName)
+	}
+
+	// locate (or maybe create) the field
+	field, err := obj.Pipe(lookupField)
+	if err != nil || field == nil {
+		return errors.WrapPrefixf(err, "fieldName: %s", fieldName)
+	}
+
+	// copy the current fltr and change the path on the copy
+	var next = fltr
+	// call filter for the next path element on the matching field
+	next.path = fltr.path[1:]
+	return next.filter(field)
+}
+
+// seq calls filter on all sequence elements
+func (fltr fieldSpecFilter) seq(obj *yaml.RNode) error {
+	if err := obj.VisitElements(func(node *yaml.RNode) error {
+		// recurse on each element -- re-allocating a fieldSpecFilter is
+		// not strictly required, but is more consistent with field
+		// and less likely to have side effects
+		// keep the entire path -- it does not contain parts for sequences
+		return fltr.filter(node)
+	}); err != nil {
+		return errors.WrapPrefixf(err,
+			"visit traversal on path: %v", fltr.path)
+	}
+
+	return nil
+}
+
+// isSequenceField returns true if the path element is for a sequence field.
+// isSequence also returns the path element with the '[]' suffix trimmed
+func isSequenceField(name string) (string, bool) {
+	isSeq := strings.HasSuffix(name, "[]")
+	name = strings.TrimSuffix(name, "[]")
+	return name, isSeq
+}
+
+// isMatchGVK returns true if the fs.GVK matches the obj GVK.
+func isMatchGVK(fs types.FieldSpec, obj *yaml.RNode) (bool, error) {
+	meta, err := obj.GetMeta()
+	if err != nil {
+		return false, err
+	}
+	if fs.Kind != "" && fs.Kind != meta.Kind {
+		// kind doesn't match
+		return false, err
+	}
+
+	// parse the group and version from the apiVersion field
+	group, version := parseGV(meta.APIVersion)
+
+	if fs.Group != "" && fs.Group != group {
+		// group doesn't match
+		return false, nil
+	}
+
+	if fs.Version != "" && fs.Version != version {
+		// version doesn't match
+		return false, nil
+	}
+
+	return true, nil
+}

api/filters/fsslice/fsslice.go

@@ -0,0 +1,58 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package fsslice
+
+import (
+	"sigs.k8s.io/kustomize/api/types"
+	"sigs.k8s.io/kustomize/kyaml/yaml"
+)
+
+// SetFn sets a value
+type SetFn func(*yaml.RNode) error
+
+// SetScalar returns a SetFn to set a scalar value
+func SetScalar(value string) SetFn {
+	return func(node *yaml.RNode) error {
+		return node.PipeE(yaml.FieldSetter{StringValue: value})
+	}
+}
+
+// SetEntry returns a SetFn to set an entry in a map
+func SetEntry(key, value string) SetFn {
+	return func(node *yaml.RNode) error {
+		return node.PipeE(yaml.FieldSetter{
+			Name: key, StringValue: value})
+	}
+}
+
+var _ yaml.Filter = Filter{}
+
+// Filter uses an FsSlice to modify fields on a single object
+type Filter struct {
+	// FieldSpecList list of FieldSpecs to set
+	FsSlice types.FsSlice `yaml:"fsSlice"`
+
+	// SetValue is called on each field that matches one of the FieldSpecs
+	SetValue SetFn
+
+	// CreateKind is used to create fields that do not exist
+	CreateKind yaml.Kind
+}
+
+func (fltr Filter) Filter(obj *yaml.RNode) (*yaml.RNode, error) {
+	for i := range fltr.FsSlice {
+		// apply this FieldSpec
+		// create a new filter for each iteration because they
+		// store internal state about the field paths
+		_, err := (&fieldSpecFilter{
+			FieldSpec:  fltr.FsSlice[i],
+			SetValue:   fltr.SetValue,
+			CreateKind: fltr.CreateKind,
+		}).Filter(obj)
+		if err != nil {
+			return nil, err
+		}
+	}
+	return obj, nil
+}

api/filters/fsslice/fsslice_test.go

@@ -0,0 +1,379 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package fsslice_test
+
+import (
+	"bytes"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"sigs.k8s.io/kustomize/api/filters/fsslice"
+	"sigs.k8s.io/kustomize/kyaml/kio"
+	"sigs.k8s.io/kustomize/kyaml/yaml"
+)
+
+type TestCase struct {
+	name     string
+	input    string
+	expected string
+	filter   fsslice.Filter
+	fsSlice  string
+	error    string
+}
+
+var tests = []TestCase{
+	{
+		name: "update",
+		fsSlice: `
+- path: a/b
+  group: foo
+  kind: Bar
+`,
+		input: `
+apiVersion: foo/v1beta1
+kind: Bar
+a:
+  b: c
+`,
+		expected: `
+apiVersion: foo/v1beta1
+kind: Bar
+a:
+  b: e
+`,
+		filter: fsslice.Filter{
+			SetValue: fsslice.SetScalar("e"),
+		},
+	},
+
+	{
+		name: "update-kind-not-match",
+		fsSlice: `
+- path: a/b
+  group: foo
+  kind: Bar1
+`,
+		input: `
+apiVersion: foo/v1beta1
+kind: Bar2
+a:
+  b: c
+`,
+		expected: `
+apiVersion: foo/v1beta1
+kind: Bar2
+a:
+  b: c
+`,
+		filter: fsslice.Filter{
+			SetValue: fsslice.SetScalar("e"),
+		},
+	},
+
+	{
+		name: "update-group-not-match",
+		fsSlice: `
+- path: a/b
+  group: foo1
+  kind: Bar
+`,
+		input: `
+apiVersion: foo2/v1beta1
+kind: Bar
+a:
+  b: c
+`,
+		expected: `
+apiVersion: foo2/v1beta1
+kind: Bar
+a:
+  b: c
+`,
+		filter: fsslice.Filter{
+			SetValue: fsslice.SetScalar("e"),
+		},
+	},
+
+	{
+		name: "update-version-not-match",
+		fsSlice: `
+- path: a/b
+  group: foo
+  version: v1beta1
+  kind: Bar
+`,
+		input: `
+apiVersion: foo/v1beta2
+kind: Bar
+a:
+  b: c
+`,
+		expected: `
+apiVersion: foo/v1beta2
+kind: Bar
+a:
+  b: c
+`,
+		filter: fsslice.Filter{
+			SetValue: fsslice.SetScalar("e"),
+		},
+	},
+
+	{
+		name: "bad-version",
+		fsSlice: `
+- path: a/b
+  group: foo
+  version: v1beta1
+  kind: Bar
+`,
+		input: `
+apiVersion: foo/v1beta2/something
+kind: Bar
+a:
+  b: c
+`,
+		expected: `
+apiVersion: foo/v1beta2/something
+kind: Bar
+a:
+  b: c
+`,
+		filter: fsslice.Filter{
+			SetValue: fsslice.SetScalar("e"),
+		},
+	},
+
+	{
+		name: "bad-meta",
+		fsSlice: `
+- path: a/b
+  group: foo
+  version: v1beta1
+  kind: Bar
+`,
+		input: `
+a:
+  b: c
+`,
+		filter: fsslice.Filter{
+			SetValue: fsslice.SetScalar("e"),
+		},
+		error: "missing Resource metadata",
+	},
+
+	{
+		name: "miss-match-type",
+		fsSlice: `
+- path: a/b/c
+  kind: Bar
+`,
+		input: `
+kind: Bar
+a:
+  b: a
+`,
+		error: "obj kind: Bar\na:\n  b: a\n at path a/b/c: unsupported yaml node",
+		filter: fsslice.Filter{
+			SetValue: fsslice.SetScalar("e"),
+		},
+	},
+
+	{
+		name: "add",
+		fsSlice: `
+- path: a/b/c/d
+  group: foo
+  create: true
+  kind: Bar
+`,
+		input: `
+apiVersion: foo/v1beta1
+kind: Bar
+a: {}
+`,
+		expected: `
+apiVersion: foo/v1beta1
+kind: Bar
+a: {b: {c: {d: e}}}
+`,
+		filter: fsslice.Filter{
+			SetValue:   fsslice.SetScalar("e"),
+			CreateKind: yaml.ScalarNode,
+		},
+	},
+
+	{
+		name: "update-in-sequence",
+		fsSlice: `
+- path: a/b[]/c/d
+  group: foo
+  kind: Bar
+`,
+		input: `
+apiVersion: foo/v1beta1
+kind: Bar
+a:
+  b:
+  - c:
+      d: a
+`,
+		expected: `
+apiVersion: foo/v1beta1
+kind: Bar
+a:
+  b:
+  - c:
+      d: e
+`,
+		filter: fsslice.Filter{
+			SetValue: fsslice.SetScalar("e"),
+		},
+	},
+
+	// Don't create a sequence
+	{
+		name: "empty-sequence-no-create",
+		fsSlice: `
+- path: a/b[]/c/d
+  group: foo
+  create: true
+  kind: Bar
+`,
+		input: `
+apiVersion: foo/v1beta1
+kind: Bar
+a: {}
+`,
+		expected: `
+apiVersion: foo/v1beta1
+kind: Bar
+a: {}
+`,
+		filter: fsslice.Filter{
+			SetValue:   fsslice.SetScalar("e"),
+			CreateKind: yaml.ScalarNode,
+		},
+	},
+
+	// Create a new field for an element in a sequence
+	{
+		name: "empty-sequence-create",
+		fsSlice: `
+- path: a/b[]/c/d
+  group: foo
+  create: true
+  kind: Bar
+`,
+		input: `
+apiVersion: foo/v1beta1
+kind: Bar
+a:
+  b:
+  - c: {}
+`,
+		expected: `
+apiVersion: foo/v1beta1
+kind: Bar
+a:
+  b:
+  - c: {d: e}
+`,
+		filter: fsslice.Filter{
+			SetValue:   fsslice.SetScalar("e"),
+			CreateKind: yaml.ScalarNode,
+		},
+	},
+
+	{
+		name: "group v1",
+		fsSlice: `
+- path: a/b
+  group: v1
+  create: true
+  kind: Bar
+`,
+		input: `
+apiVersion: v1
+kind: Bar
+`,
+		expected: `
+apiVersion: v1
+kind: Bar
+`,
+		filter: fsslice.Filter{
+			SetValue:   fsslice.SetScalar("e"),
+			CreateKind: yaml.ScalarNode,
+		},
+	},
+
+	{
+		name: "version v1",
+		fsSlice: `
+- path: a/b
+  version: v1
+  create: true
+  kind: Bar
+`,
+		input: `
+apiVersion: v1
+kind: Bar
+`,
+		expected: `
+apiVersion: v1
+kind: Bar
+a:
+  b: e
+`,
+		filter: fsslice.Filter{
+			SetValue:   fsslice.SetScalar("e"),
+			CreateKind: yaml.ScalarNode,
+		},
+	},
+}
+
+func TestFilter_Filter(t *testing.T) {
+	for i := range tests {
+		test := tests[i]
+		t.Run(test.name, func(t *testing.T) {
+			err := yaml.Unmarshal([]byte(test.fsSlice), &test.filter.FsSlice)
+			if !assert.NoError(t, err) {
+				t.FailNow()
+			}
+
+			out := &bytes.Buffer{}
+			rw := &kio.ByteReadWriter{
+				Reader:                bytes.NewBufferString(test.input),
+				Writer:                out,
+				OmitReaderAnnotations: true,
+			}
+
+			// run the filter
+			err = kio.Pipeline{
+				Inputs:  []kio.Reader{rw},
+				Filters: []kio.Filter{kio.FilterAll(test.filter)},
+				Outputs: []kio.Writer{rw},
+			}.Execute()
+			if test.error != "" {
+				if !assert.EqualError(t, err, test.error) {
+					t.FailNow()
+				}
+				// stop rest of test
+				return
+			}
+
+			if !assert.NoError(t, err) {
+				t.FailNow()
+			}
+
+			// check results
+			if !assert.Equal(t,
+				strings.TrimSpace(test.expected),
+				strings.TrimSpace(out.String())) {
+				t.FailNow()
+			}
+		})
+	}
+}

api/filters/fsslice/gvk.go

@@ -0,0 +1,49 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+package fsslice
+
+import (
+	"strings"
+
+	"sigs.k8s.io/kustomize/api/resid"
+	"sigs.k8s.io/kustomize/kyaml/yaml"
+)
+
+// Return true for 'v' followed by a 1 or 2, and don't look at rest.
+// I.e. 'v1', 'v1beta1', 'v2', would return true.
+func looksLikeACoreApiVersion(s string) bool {
+	if len(s) < 2 {
+		return false
+	}
+	if s[0:1] != "v" {
+		return false
+	}
+	return s[1:2] == "1" || s[1:2] == "2"
+}
+
+// parseGV parses apiVersion field into group and version.
+func parseGV(apiVersion string) (group, version string) {
+	// parse the group and version from the apiVersion field
+	parts := strings.SplitN(apiVersion, "/", 2)
+	group = parts[0]
+	if len(parts) > 1 {
+		version = parts[1]
+	}
+	// Special case the original "apiVersion" of what
+	// we now call the "core" (empty) group.
+	if version == "" && looksLikeACoreApiVersion(group) {
+		version = group
+		group = ""
+	}
+	return
+}
+
+// GetGVK parses the metadata into a GVK
+func GetGVK(meta yaml.ResourceMeta) resid.Gvk {
+	group, version := parseGV(meta.APIVersion)
+	return resid.Gvk{
+		Group:   group,
+		Version: version,
+		Kind:    meta.Kind,
+	}
+}

api/filters/fsslice/gvk_test.go

@@ -0,0 +1,156 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+package fsslice
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"sigs.k8s.io/kustomize/api/resid"
+	"sigs.k8s.io/kustomize/kyaml/yaml"
+)
+
+func TestParseGV(t *testing.T) {
+	testCases := map[string]struct {
+		input           string
+		expectedGroup   string
+		expectedVersion string
+	}{
+		"empty": {
+			input:           "",
+			expectedGroup:   "",
+			expectedVersion: "",
+		},
+		"certSigning": {
+			input:           "certificates.k8s.io/v1beta1",
+			expectedGroup:   "certificates.k8s.io",
+			expectedVersion: "v1beta1",
+		},
+		"extensions": {
+			input:           "extensions/v1beta1",
+			expectedGroup:   "extensions",
+			expectedVersion: "v1beta1",
+		},
+		"normal": {
+			input:           "apps/v1",
+			expectedGroup:   "apps",
+			expectedVersion: "v1",
+		},
+		"justApps": {
+			input:           "apps",
+			expectedGroup:   "apps",
+			expectedVersion: "",
+		},
+		"coreV1": {
+			input:           "v1",
+			expectedGroup:   "",
+			expectedVersion: "v1",
+		},
+		"coreV2": {
+			input:           "v2",
+			expectedGroup:   "",
+			expectedVersion: "v2",
+		},
+		"coreV2Beta1": {
+			input:           "v2beta1",
+			expectedGroup:   "",
+			expectedVersion: "v2beta1",
+		},
+	}
+
+	for tn, tc := range testCases {
+		t.Run(tn, func(t *testing.T) {
+			group, version := parseGV(tc.input)
+			if !assert.Equal(t, tc.expectedGroup, group) {
+				t.FailNow()
+			}
+			if !assert.Equal(t, tc.expectedVersion, version) {
+				t.FailNow()
+			}
+		})
+	}
+}
+
+func TestGetGVK(t *testing.T) {
+	testCases := map[string]struct {
+		input      string
+		expected   resid.Gvk
+		parseError string
+		metaError  string
+	}{
+		"empty": {
+			input: `
+`,
+			parseError: "EOF",
+		},
+		"junk": {
+			input: `
+congress: effective
+`,
+			metaError: "missing Resource metadata",
+		},
+		"normal": {
+			input: `
+apiVersion: apps/v1
+kind: Deployment
+`,
+			expected: resid.Gvk{Group: "apps", Version: "v1", Kind: "Deployment"},
+		},
+		"apiVersionOnlyWithSlash": {
+			input: `
+apiVersion: apps/v1
+`,
+			expected: resid.Gvk{Group: "apps", Version: "v1", Kind: ""},
+		},
+		"apiVersionOnlyNoSlash1": {
+			input: `
+apiVersion: apps
+`,
+			expected: resid.Gvk{Group: "apps", Version: "", Kind: ""},
+		},
+		"apiVersionOnlyNoSlash2": {
+			input: `
+apiVersion: v1
+`,
+			expected: resid.Gvk{Group: "", Version: "v1", Kind: ""},
+		},
+	}
+
+	for tn, tc := range testCases {
+		t.Run(tn, func(t *testing.T) {
+			obj, err := yaml.Parse(tc.input)
+			if len(tc.parseError) != 0 {
+				if err == nil {
+					t.Error("expected parse error")
+					return
+				}
+				if !strings.Contains(err.Error(), tc.parseError) {
+					t.Errorf("expected parse err '%s', got '%v'", tc.parseError, err)
+				}
+				return
+			}
+			if !assert.NoError(t, err) {
+				t.FailNow()
+			}
+			meta, err := obj.GetMeta()
+			if len(tc.metaError) != 0 {
+				if err == nil {
+					t.Error("expected meta error")
+					return
+				}
+				if !strings.Contains(err.Error(), tc.metaError) {
+					t.Errorf("expected meta err '%s', got '%v'", tc.metaError, err)
+				}
+				return
+			}
+			if !assert.NoError(t, err) {
+				t.FailNow()
+			}
+			gvk := GetGVK(meta)
+			if !assert.Equal(t, tc.expected, gvk) {
+				t.FailNow()
+			}
+		})
+	}
+}

api/filters/imagetag/doc.go

@@ -0,0 +1,12 @@
+// Copyright 2020 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+// Package imagetag contains two kio.Filter implementations to cover the
+// functionality of the kustomize imagetag transformer.
+//
+// Filter updates fields based on a FieldSpec and an ImageTag.
+//
+// LegacyFilter doesn't use a FieldSpec, and instead only updates image
+// references if the field is name image and it is underneath a field called
+// either containers or initContainers.
+package imagetag

api/filters/imagetag/example_test.go

@@ -0,0 +1,126 @@
+// Copyright 2020 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package imagetag
+
+import (
+	"bytes"
+	"log"
+	"os"
+
+	"sigs.k8s.io/kustomize/api/types"
+	"sigs.k8s.io/kustomize/kyaml/kio"
+)
+
+func ExampleFilter() {
+	err := kio.Pipeline{
+		Inputs: []kio.Reader{&kio.ByteReader{Reader: bytes.NewBufferString(`
+apiVersion: example.com/v1
+kind: Foo
+metadata:
+  name: instance
+spec:
+  containers:
+  - name: FooBar
+    image: nginx
+---
+apiVersion: example.com/v1
+kind: Bar
+metadata:
+  name: instance
+spec:
+  containers:
+  - name: BarFoo
+    image: nginx:1.2.1
+`)}},
+		Filters: []kio.Filter{Filter{
+			ImageTag: types.Image{
+				Name:    "nginx",
+				NewName: "apache",
+				Digest:  "12345",
+			},
+			FsSlice: []types.FieldSpec{
+				{
+					Path: "spec/containers[]/image",
+				},
+			},
+		}},
+		Outputs: []kio.Writer{kio.ByteWriter{Writer: os.Stdout}},
+	}.Execute()
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	// Output:
+	// apiVersion: example.com/v1
+	// kind: Foo
+	// metadata:
+	//   name: instance
+	// spec:
+	//   containers:
+	//   - name: FooBar
+	//     image: apache@12345
+	// ---
+	// apiVersion: example.com/v1
+	// kind: Bar
+	// metadata:
+	//   name: instance
+	// spec:
+	//   containers:
+	//   - name: BarFoo
+	//     image: apache@12345
+}
+
+func ExampleLegacyFilter() {
+	err := kio.Pipeline{
+		Inputs: []kio.Reader{&kio.ByteReader{Reader: bytes.NewBufferString(`
+apiVersion: example.com/v1
+kind: Foo
+metadata:
+  name: instance
+spec:
+  containers:
+  - name: FooBar
+    image: nginx
+---
+apiVersion: example.com/v1
+kind: Bar
+metadata:
+  name: instance
+spec:
+  containers:
+  - name: BarFoo
+    image: nginx:1.2.1
+`)}},
+		Filters: []kio.Filter{LegacyFilter{
+			ImageTag: types.Image{
+				Name:    "nginx",
+				NewName: "apache",
+				Digest:  "12345",
+			},
+		}},
+		Outputs: []kio.Writer{kio.ByteWriter{Writer: os.Stdout}},
+	}.Execute()
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	// Output:
+	// apiVersion: example.com/v1
+	// kind: Foo
+	// metadata:
+	//   name: instance
+	// spec:
+	//   containers:
+	//   - name: FooBar
+	//     image: apache@12345
+	// ---
+	// apiVersion: example.com/v1
+	// kind: Bar
+	// metadata:
+	//   name: instance
+	// spec:
+	//   containers:
+	//   - name: BarFoo
+	//     image: apache@12345
+}

api/filters/imagetag/imagetag.go

@@ -0,0 +1,44 @@
+// Copyright 2020 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package imagetag
+
+import (
+	"sigs.k8s.io/kustomize/api/filters/fsslice"
+	"sigs.k8s.io/kustomize/api/types"
+	"sigs.k8s.io/kustomize/kyaml/kio"
+	"sigs.k8s.io/kustomize/kyaml/yaml"
+)
+
+type Filter struct {
+	// imageTag is the tag we want to apply to the inputs
+	ImageTag types.Image `json:"imageTag,omitempty" yaml:"imageTag,omitempty"`
+
+	// FsSlice contains the FieldSpecs to locate the namespace field
+	FsSlice types.FsSlice `json:"fieldSpecs,omitempty" yaml:"fieldSpecs,omitempty"`
+}
+
+var _ kio.Filter = Filter{}
+
+func (f Filter) Filter(nodes []*yaml.RNode) ([]*yaml.RNode, error) {
+	_, err := kio.FilterAll(yaml.FilterFunc(f.filter)).Filter(nodes)
+	return nodes, err
+}
+
+func (f Filter) filter(node *yaml.RNode) (*yaml.RNode, error) {
+	if err := node.PipeE(fsslice.Filter{
+		FsSlice:  f.FsSlice,
+		SetValue: updateImageTagFn(f.ImageTag),
+	}); err != nil {
+		return nil, err
+	}
+	return node, nil
+}
+
+func updateImageTagFn(imageTag types.Image) fsslice.SetFn {
+	return func(node *yaml.RNode) error {
+		return node.PipeE(imageTagUpdater{
+			ImageTag: imageTag,
+		})
+	}
+}

api/filters/imagetag/imagetag_test.go

@@ -0,0 +1,101 @@
+// Copyright 2020 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package imagetag
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	filtertest "sigs.k8s.io/kustomize/api/testutils/filtertest"
+	"sigs.k8s.io/kustomize/api/types"
+)
+
+func TestImageTagUpdater_Filter(t *testing.T) {
+	testCases := map[string]struct {
+		input          string
+		expectedOutput string
+		filter         Filter
+		fsSlice        types.FsSlice
+	}{
+		"update with digest": {
+			input: `
+apiVersion: example.com/v1
+kind: Foo
+metadata:
+  name: instance
+spec:
+  image: nginx:1.2.1
+`,
+			expectedOutput: `
+apiVersion: example.com/v1
+kind: Foo
+metadata:
+  name: instance
+spec:
+  image: apache@12345
+`,
+			filter: Filter{
+				ImageTag: types.Image{
+					Name:    "nginx",
+					NewName: "apache",
+					Digest:  "12345",
+				},
+			},
+			fsSlice: []types.FieldSpec{
+				{
+					Path: "spec/image",
+				},
+			},
+		},
+		"multiple matches in sequence": {
+			input: `
+apiVersion: example.com/v1
+kind: Foo
+metadata:
+  name: instance
+spec:
+  containers:
+  - image: nginx:1.2.1
+  - image: not_nginx@54321
+  - image: nginx:1.2.1
+`,
+			expectedOutput: `
+apiVersion: example.com/v1
+kind: Foo
+metadata:
+  name: instance
+spec:
+  containers:
+  - image: apache:3.2.1
+  - image: not_nginx@54321
+  - image: apache:3.2.1
+`,
+			filter: Filter{
+				ImageTag: types.Image{
+					Name:    "nginx",
+					NewName: "apache",
+					NewTag:  "3.2.1",
+				},
+			},
+			fsSlice: []types.FieldSpec{
+				{
+					Path: "spec/containers/image",
+				},
+			},
+		},
+	}
+
+	for tn, tc := range testCases {
+		t.Run(tn, func(t *testing.T) {
+			filter := tc.filter
+			filter.FsSlice = tc.fsSlice
+			if !assert.Equal(t,
+				strings.TrimSpace(tc.expectedOutput),
+				strings.TrimSpace(filtertest.RunFilter(t, tc.input, filter))) {
+				t.FailNow()
+			}
+		})
+	}
+}

api/filters/imagetag/legacy.go

@@ -0,0 +1,113 @@
+// Copyright 2020 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package imagetag
+
+import (
+	"sigs.k8s.io/kustomize/api/types"
+	"sigs.k8s.io/kustomize/kyaml/kio"
+	"sigs.k8s.io/kustomize/kyaml/yaml"
+)
+
+// LegacyFilter is an implementation of the kio.Filter interface
+// that scans through the provided kyaml data structure and updates
+// any values of any image fields that is inside a sequence under
+// a field called either containers or initContainers. The field is only
+// update if it has a value that matches and image reference and the name
+// of the image is a match with the provided ImageTag.
+type LegacyFilter struct {
+	ImageTag types.Image `json:"imageTag,omitempty" yaml:"imageTag,omitempty"`
+}
+
+var _ kio.Filter = LegacyFilter{}
+
+func (lf LegacyFilter) Filter(nodes []*yaml.RNode) ([]*yaml.RNode, error) {
+	return kio.FilterAll(yaml.FilterFunc(lf.filter)).Filter(nodes)
+}
+
+func (lf LegacyFilter) filter(node *yaml.RNode) (*yaml.RNode, error) {
+	meta, err := node.GetMeta()
+	if err != nil {
+		return nil, err
+	}
+
+	// We do not make any changes if the type of the resource
+	// is CustomResourceDefinition.
+	if meta.Kind == `CustomResourceDefinition` {
+		return node, nil
+	}
+
+	fff := findFieldsFilter{
+		fields:        []string{"containers", "initContainers"},
+		fieldCallback: checkImageTagsFn(lf.ImageTag),
+	}
+	if err := node.PipeE(fff); err != nil {
+		return nil, err
+	}
+	return node, nil
+}
+
+type fieldCallback func(node *yaml.RNode) error
+
+// findFieldsFilter is an implementation of the kio.Filter
+// interface. It will walk the data structure and look for fields
+// that matches the provided list of field names. For each match,
+// the value of the field will be passed in as a parameter to the
+// provided fieldCallback.
+// TODO: move this to kyaml/filterutils
+type findFieldsFilter struct {
+	fields []string
+
+	fieldCallback fieldCallback
+}
+
+func (f findFieldsFilter) Filter(obj *yaml.RNode) (*yaml.RNode, error) {
+	return obj, f.walk(obj)
+}
+
+func (f findFieldsFilter) walk(node *yaml.RNode) error {
+	switch node.YNode().Kind {
+	case yaml.MappingNode:
+		return node.VisitFields(func(n *yaml.MapNode) error {
+			err := f.walk(n.Value)
+			if err != nil {
+				return err
+			}
+			key := n.Key.YNode().Value
+			if contains(f.fields, key) {
+				return f.fieldCallback(n.Value)
+			}
+			return nil
+		})
+	case yaml.SequenceNode:
+		return node.VisitElements(func(n *yaml.RNode) error {
+			return f.walk(n)
+		})
+	}
+	return nil
+}
+
+func contains(slice []string, str string) bool {
+	for _, s := range slice {
+		if s == str {
+			return true
+		}
+	}
+	return false
+}
+
+func checkImageTagsFn(imageTag types.Image) fieldCallback {
+	return func(node *yaml.RNode) error {
+		if node.YNode().Kind != yaml.SequenceNode {
+			return nil
+		}
+
+		return node.VisitElements(func(n *yaml.RNode) error {
+			// Look up any fields on the provided node that is named
+			// image.
+			return n.PipeE(yaml.Get("image"), imageTagUpdater{
+				ImageTag: imageTag,
+			})
+		})
+	}
+}

api/filters/imagetag/legacy_test.go

@@ -0,0 +1,136 @@
+// Copyright 2020 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package imagetag
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	filtertest "sigs.k8s.io/kustomize/api/testutils/filtertest"
+	"sigs.k8s.io/kustomize/api/types"
+)
+
+func TestLegacyImageTag_Filter(t *testing.T) {
+	testCases := map[string]struct {
+		input          string
+		expectedOutput string
+		filter         LegacyFilter
+	}{
+		"updates multiple images inside containers": {
+			input: `
+apiVersion: example.com/v1
+kind: Foo
+metadata:
+  name: instance
+spec:
+  containers:
+  - image: nginx:1.2.1
+  - image: nginx:2.1.2
+`,
+			expectedOutput: `
+apiVersion: example.com/v1
+kind: Foo
+metadata:
+  name: instance
+spec:
+  containers:
+  - image: apache@12345
+  - image: apache@12345
+`,
+			filter: LegacyFilter{
+				ImageTag: types.Image{
+					Name:    "nginx",
+					NewName: "apache",
+					Digest:  "12345",
+				},
+			},
+		},
+		"updates inside both containers and initContainers": {
+			input: `
+apiVersion: example.com/v1
+kind: Foo
+metadata:
+  name: instance
+spec:
+  containers:
+  - image: nginx:1.2.1
+  - image: tomcat:1.2.3
+  initContainers:
+  - image: nginx:1.2.1
+  - image: apache:1.2.3
+`,
+			expectedOutput: `
+apiVersion: example.com/v1
+kind: Foo
+metadata:
+  name: instance
+spec:
+  containers:
+  - image: apache:3.2.1
+  - image: tomcat:1.2.3
+  initContainers:
+  - image: apache:3.2.1
+  - image: apache:1.2.3
+`,
+			filter: LegacyFilter{
+				ImageTag: types.Image{
+					Name:    "nginx",
+					NewName: "apache",
+					NewTag:  "3.2.1",
+				},
+			},
+		},
+		"updates on multiple depths": {
+			input: `
+apiVersion: example.com/v1
+kind: Foo
+metadata:
+  name: instance
+spec:
+  containers:
+  - image: nginx:1.2.1
+  - image: tomcat:1.2.3
+  template:
+    spec: 
+      initContainers:
+      - image: nginx:1.2.1
+      - image: apache:1.2.3
+`,
+			expectedOutput: `
+apiVersion: example.com/v1
+kind: Foo
+metadata:
+  name: instance
+spec:
+  containers:
+  - image: apache:3.2.1
+  - image: tomcat:1.2.3
+  template:
+    spec:
+      initContainers:
+      - image: apache:3.2.1
+      - image: apache:1.2.3
+`,
+			filter: LegacyFilter{
+				ImageTag: types.Image{
+					Name:    "nginx",
+					NewName: "apache",
+					NewTag:  "3.2.1",
+				},
+			},
+		},
+	}
+
+	for tn, tc := range testCases {
+		t.Run(tn, func(t *testing.T) {
+			filter := tc.filter
+			if !assert.Equal(t,
+				strings.TrimSpace(tc.expectedOutput),
+				strings.TrimSpace(filtertest.RunFilter(t, tc.input, filter))) {
+				t.FailNow()
+			}
+		})
+	}
+}

api/filters/imagetag/updater.go

@@ -0,0 +1,43 @@
+// Copyright 2020 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package imagetag
+
+import (
+	"sigs.k8s.io/kustomize/api/image"
+	"sigs.k8s.io/kustomize/api/types"
+	"sigs.k8s.io/kustomize/kyaml/yaml"
+)
+
+// imageTagUpdater is an implementation of the kio.Filter interface
+// that will update the value of the yaml node based on the provided
+// ImageTag if the current value matches the format of an image reference.
+type imageTagUpdater struct {
+	Kind     string      `yaml:"kind,omitempty"`
+	ImageTag types.Image `yaml:"imageTag,omitempty"`
+}
+
+func (u imageTagUpdater) Filter(rn *yaml.RNode) (*yaml.RNode, error) {
+	if err := yaml.ErrorIfInvalid(rn, yaml.ScalarNode); err != nil {
+		return nil, err
+	}
+
+	value := rn.YNode().Value
+
+	if !image.IsImageMatched(value, u.ImageTag.Name) {
+		return rn, nil
+	}
+
+	name, tag := image.Split(value)
+	if u.ImageTag.NewName != "" {
+		name = u.ImageTag.NewName
+	}
+	if u.ImageTag.NewTag != "" {
+		tag = ":" + u.ImageTag.NewTag
+	}
+	if u.ImageTag.Digest != "" {
+		tag = "@" + u.ImageTag.Digest
+	}
+
+	return rn.Pipe(yaml.FieldSetter{StringValue: name + tag})
+}

api/filters/labels/doc.go

@@ -0,0 +1,6 @@
+// Copyright 2020 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+// Package labels contains a kio.Filter implementation of the kustomize
+// labels transformer.
+package labels

api/filters/labels/example_test.go

@@ -0,0 +1,55 @@
+// Copyright 2020 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package labels
+
+import (
+	"bytes"
+	"log"
+	"os"
+
+	"sigs.k8s.io/kustomize/api/internal/plugins/builtinconfig"
+	"sigs.k8s.io/kustomize/kyaml/kio"
+)
+
+func ExampleFilter() {
+	fss := builtinconfig.MakeDefaultConfig().CommonLabels
+	err := kio.Pipeline{
+		Inputs: []kio.Reader{&kio.ByteReader{Reader: bytes.NewBufferString(`
+apiVersion: example.com/v1
+kind: Foo
+metadata:
+  name: instance
+---
+apiVersion: example.com/v1
+kind: Bar
+metadata:
+  name: instance
+`)}},
+		Filters: []kio.Filter{Filter{
+			Labels: map[string]string{
+				"foo": "bar",
+			},
+			FsSlice: fss,
+		}},
+		Outputs: []kio.Writer{kio.ByteWriter{Writer: os.Stdout}},
+	}.Execute()
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	// Output:
+	// apiVersion: example.com/v1
+	// kind: Foo
+	// metadata:
+	//   name: instance
+	//   labels:
+	//     foo: bar
+	// ---
+	// apiVersion: example.com/v1
+	// kind: Bar
+	// metadata:
+	//   name: instance
+	//   labels:
+	//     foo: bar
+}

api/filters/labels/labels.go

@@ -0,0 +1,43 @@
+// Copyright 2020 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package labels
+
+import (
+	"sigs.k8s.io/kustomize/api/filters/filtersutil"
+	"sigs.k8s.io/kustomize/api/filters/fsslice"
+	"sigs.k8s.io/kustomize/api/types"
+	"sigs.k8s.io/kustomize/kyaml/kio"
+	"sigs.k8s.io/kustomize/kyaml/yaml"
+)
+
+type labelMap map[string]string
+
+// Filter sets labels.
+type Filter struct {
+	// Labels is the set of labels to apply to the inputs
+	Labels labelMap `yaml:"labels,omitempty"`
+
+	// FsSlice identifies the label fields.
+	FsSlice types.FsSlice
+}
+
+var _ kio.Filter = Filter{}
+
+func (f Filter) Filter(nodes []*yaml.RNode) ([]*yaml.RNode, error) {
+	keys := filtersutil.SortedMapKeys(f.Labels)
+	_, err := kio.FilterAll(yaml.FilterFunc(
+		func(node *yaml.RNode) (*yaml.RNode, error) {
+			for _, k := range keys {
+				if err := node.PipeE(fsslice.Filter{
+					FsSlice:    f.FsSlice,
+					SetValue:   fsslice.SetEntry(k, f.Labels[k]),
+					CreateKind: yaml.MappingNode, // Labels are MappingNodes.
+				}); err != nil {
+					return nil, err
+				}
+			}
+			return node, nil
+		})).Filter(nodes)
+	return nodes, err
+}

api/filters/labels/labels_test.go

@@ -0,0 +1,305 @@
+// Copyright 2020 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package labels
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"sigs.k8s.io/kustomize/api/resid"
+	filtertest_test "sigs.k8s.io/kustomize/api/testutils/filtertest"
+	"sigs.k8s.io/kustomize/api/types"
+)
+
+func TestLabels_Filter(t *testing.T) {
+	testCases := map[string]struct {
+		input          string
+		expectedOutput string
+		filter         Filter
+	}{
+		"add": {
+			input: `
+apiVersion: example.com/v1
+kind: Foo
+metadata:
+  name: instance
+  labels:
+    hero: batman
+    fiend: riddler
+`,
+			expectedOutput: `
+apiVersion: example.com/v1
+kind: Foo
+metadata:
+  name: instance
+  labels:
+    hero: batman
+    fiend: riddler
+    auto: ford
+    bean: cannellini
+    clown: emmett kelley
+    dragon: smaug
+`,
+			filter: Filter{
+				Labels: labelMap{
+					"clown":  "emmett kelley",
+					"auto":   "ford",
+					"dragon": "smaug",
+					"bean":   "cannellini",
+				},
+				FsSlice: []types.FieldSpec{
+					{
+						Path:               "metadata/labels",
+						CreateIfNotPresent: true,
+					},
+				},
+			},
+		},
+		"update": {
+			input: `
+apiVersion: example.com/v1
+kind: Foo
+metadata:
+  name: instance
+  labels:
+    hero: batman
+    fiend: riddler
+`,
+			expectedOutput: `
+apiVersion: example.com/v1
+kind: Foo
+metadata:
+  name: instance
+  labels:
+    hero: superman
+    fiend: luthor
+    bean: cannellini
+    clown: emmett kelley
+`,
+			filter: Filter{
+				Labels: labelMap{
+					"clown": "emmett kelley",
+					"hero":  "superman",
+					"fiend": "luthor",
+					"bean":  "cannellini",
+				}, FsSlice: []types.FieldSpec{
+					{
+						Path:               "metadata/labels",
+						CreateIfNotPresent: true,
+					},
+				},
+			},
+		},
+
+		"data-fieldspecs": {
+			input: `
+apiVersion: example.com/v1
+kind: Foo
+metadata:
+  name: instance
+---
+apiVersion: example.com/v1
+kind: Bar
+metadata:
+  name: instance
+`,
+			expectedOutput: `
+apiVersion: example.com/v1
+kind: Foo
+metadata:
+  name: instance
+  labels:
+    sleater: kinney
+a:
+  b:
+    sleater: kinney
+---
+apiVersion: example.com/v1
+kind: Bar
+metadata:
+  name: instance
+  labels:
+    sleater: kinney
+a:
+  b:
+    sleater: kinney
+`,
+			filter: Filter{
+				Labels: labelMap{
+					"sleater": "kinney",
+				},
+				FsSlice: []types.FieldSpec{
+					{
+						Path:               "metadata/labels",
+						CreateIfNotPresent: true,
+					},
+					{
+						Path:               "a/b",
+						CreateIfNotPresent: true,
+					},
+				},
+			},
+		},
+
+		"fieldSpecWithKind": {
+			input: `
+apiVersion: example.com/v1
+kind: Foo
+metadata:
+  name: instance
+---
+apiVersion: example.com/v2
+kind: Bar
+metadata:
+  name: instance
+`,
+			expectedOutput: `
+apiVersion: example.com/v1
+kind: Foo
+metadata:
+  name: instance
+  labels:
+    cheese: cheddar
+---
+apiVersion: example.com/v2
+kind: Bar
+metadata:
+  name: instance
+  labels:
+    cheese: cheddar
+a:
+  b:
+    cheese: cheddar
+`,
+			filter: Filter{
+				Labels: labelMap{
+					"cheese": "cheddar",
+				},
+				FsSlice: []types.FieldSpec{
+					{
+						Path:               "metadata/labels",
+						CreateIfNotPresent: true,
+					},
+					{
+						Gvk: resid.Gvk{
+							Kind: "Bar",
+						},
+						Path:               "a/b",
+						CreateIfNotPresent: true,
+					},
+				},
+			},
+		},
+
+		"fieldSpecWithVersion": {
+			input: `
+apiVersion: example.com/v1
+kind: Foo
+metadata:
+  name: instance
+---
+apiVersion: example.com/v2
+kind: Bar
+metadata:
+  name: instance
+`,
+			expectedOutput: `
+apiVersion: example.com/v1
+kind: Foo
+metadata:
+  name: instance
+  labels:
+    cheese: cheddar
+a:
+  b:
+    cheese: cheddar
+---
+apiVersion: example.com/v2
+kind: Bar
+metadata:
+  name: instance
+  labels:
+    cheese: cheddar
+`,
+			filter: Filter{
+				Labels: labelMap{
+					"cheese": "cheddar",
+				},
+				FsSlice: []types.FieldSpec{
+					{
+						Path:               "metadata/labels",
+						CreateIfNotPresent: true,
+					},
+					{
+						Gvk: resid.Gvk{
+							Version: "v1",
+						},
+						Path:               "a/b",
+						CreateIfNotPresent: true,
+					},
+				},
+			},
+		},
+		"fieldSpecWithVersionInConfigButNoGroupInData": {
+			input: `
+apiVersion: v1
+kind: Foo
+metadata:
+  name: instance
+---
+apiVersion: v2
+kind: Bar
+metadata:
+  name: instance
+`,
+			expectedOutput: `
+apiVersion: v1
+kind: Foo
+metadata:
+  name: instance
+  labels:
+    cheese: cheddar
+a:
+  b:
+    cheese: cheddar
+---
+apiVersion: v2
+kind: Bar
+metadata:
+  name: instance
+  labels:
+    cheese: cheddar
+`,
+			filter: Filter{
+				Labels: labelMap{
+					"cheese": "cheddar",
+				},
+				FsSlice: []types.FieldSpec{
+					{
+						Path:               "metadata/labels",
+						CreateIfNotPresent: true,
+					},
+					{
+						Gvk: resid.Gvk{
+							Version: "v1",
+						},
+						Path:               "a/b",
+						CreateIfNotPresent: true,
+					},
+				},
+			},
+		},
+	}
+
+	for tn, tc := range testCases {
+		t.Run(tn, func(t *testing.T) {
+			if !assert.Equal(t,
+				strings.TrimSpace(tc.expectedOutput),
+				strings.TrimSpace(filtertest_test.RunFilter(t, tc.input, tc.filter))) {
+				t.FailNow()
+			}
+		})
+	}
+}

api/filters/namespace/doc.go

@@ -0,0 +1,9 @@
+// Copyright 2020 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+// Package namespace contains a kio.Filter implementation of the kustomize
+// namespace transformer.
+//
+// Special cases for known Kubernetes resources have been hardcoded in addition
+// to those defined by the FsSlice.
+package namespace

api/filters/namespace/example_test.go

@@ -0,0 +1,50 @@
+// Copyright 2020 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package namespace_test
+
+import (
+	"bytes"
+	"log"
+	"os"
+
+	"sigs.k8s.io/kustomize/api/filters/namespace"
+	"sigs.k8s.io/kustomize/api/internal/plugins/builtinconfig"
+	"sigs.k8s.io/kustomize/kyaml/kio"
+)
+
+func ExampleFilter() {
+	fss := builtinconfig.MakeDefaultConfig().NameSpace
+	err := kio.Pipeline{
+		Inputs: []kio.Reader{&kio.ByteReader{Reader: bytes.NewBufferString(`
+apiVersion: example.com/v1
+kind: Foo
+metadata:
+  name: instance
+---
+apiVersion: example.com/v1
+kind: Bar
+metadata:
+  name: instance
+  namespace: bar
+`)}},
+		Filters: []kio.Filter{namespace.Filter{Namespace: "app", FsSlice: fss}},
+		Outputs: []kio.Writer{kio.ByteWriter{Writer: os.Stdout}},
+	}.Execute()
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	// Output:
+	// apiVersion: example.com/v1
+	// kind: Foo
+	// metadata:
+	//   name: instance
+	//   namespace: app
+	// ---
+	// apiVersion: example.com/v1
+	// kind: Bar
+	// metadata:
+	//   name: instance
+	//   namespace: app
+}

api/filters/namespace/namespace.go

@@ -0,0 +1,166 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package namespace
+
+import (
+	"sigs.k8s.io/kustomize/api/filters/fsslice"
+	"sigs.k8s.io/kustomize/api/types"
+	"sigs.k8s.io/kustomize/kyaml/kio"
+	"sigs.k8s.io/kustomize/kyaml/yaml"
+)
+
+type Filter struct {
+	// Namespace is the namespace to apply to the inputs
+	Namespace string `yaml:"namespace,omitempty"`
+
+	// FsSlice contains the FieldSpecs to locate the namespace field
+	FsSlice types.FsSlice `json:"fieldSpecs,omitempty" yaml:"fieldSpecs,omitempty"`
+}
+
+var _ kio.Filter = Filter{}
+
+func (ns Filter) Filter(nodes []*yaml.RNode) ([]*yaml.RNode, error) {
+	return kio.FilterAll(yaml.FilterFunc(ns.run)).Filter(nodes)
+}
+
+// Run runs the filter on a single node rather than a slice
+func (ns Filter) run(node *yaml.RNode) (*yaml.RNode, error) {
+	// hacks for hardcoded types -- :(
+	if err := ns.hacks(node); err != nil {
+		return nil, err
+	}
+
+	// Remove the fieldspecs that are for hardcoded fields.  The fieldspecs
+	// exist for backwards compatibility with other implementations
+	// of this transformation.
+	// This implementation of the namespace transformation
+	// Does not use the fieldspecs for implementing cases which
+	// require hardcoded logic.
+	ns.FsSlice = ns.removeFieldSpecsForHacks(ns.FsSlice)
+
+	// transformations based on data -- :)
+	err := node.PipeE(fsslice.Filter{
+		FsSlice:    ns.FsSlice,
+		SetValue:   fsslice.SetScalar(ns.Namespace),
+		CreateKind: yaml.ScalarNode, // Namespace is a ScalarNode
+	})
+	return node, err
+}
+
+// hacks applies the namespace transforms that are hardcoded rather
+// than specified through FieldSpecs.
+func (ns Filter) hacks(obj *yaml.RNode) error {
+	meta, err := obj.GetMeta()
+	if err != nil {
+		return err
+	}
+
+	if err := ns.metaNamespaceHack(obj, meta); err != nil {
+		return err
+	}
+
+	return ns.roleBindingHack(obj, meta)
+}
+
+// metaNamespaceHack is a hack for implementing the namespace transform
+// for the metadata.namespace field on namespace scoped resources.
+// namespace scoped resources are determined by NOT being present
+// in a blacklist of cluster-scoped resource types (by apiVersion and kind).
+//
+// This hack should be updated to allow individual resources to specify
+// if they are cluster scoped through either an annotation on the resources,
+// or through inlined OpenAPI on the resource as a YAML comment.
+func (ns Filter) metaNamespaceHack(obj *yaml.RNode, meta yaml.ResourceMeta) error {
+	gvk := fsslice.GetGVK(meta)
+	if !gvk.IsNamespaceableKind() {
+		return nil
+	}
+	f := fsslice.Filter{
+		FsSlice: []types.FieldSpec{
+			{Path: types.MetadataNamespacePath, CreateIfNotPresent: true},
+		},
+		SetValue:   fsslice.SetScalar(ns.Namespace),
+		CreateKind: yaml.ScalarNode, // Namespace is a ScalarNode
+	}
+	_, err := f.Filter(obj)
+	return err
+}
+
+// roleBindingHack is a hack for implementing the namespace transform
+// for RoleBinding and ClusterRoleBinding resource types.
+// RoleBinding and ClusterRoleBinding have namespace set on
+// elements of the "subjects" field if and only if the subject elements
+// "name" is "default".  Otherwise the namespace is not set.
+//
+// Example:
+//
+// kind: RoleBinding
+// subjects:
+// - name: "default" # this will have the namespace set
+//   ...
+// - name: "something-else" # this will not have the namespace set
+//   ...
+func (ns Filter) roleBindingHack(obj *yaml.RNode, meta yaml.ResourceMeta) error {
+	if meta.Kind != roleBindingKind && meta.Kind != clusterRoleBindingKind {
+		return nil
+	}
+
+	// Lookup the namespace field on all elements.
+	// We should change the fieldspec so this isn't necessary.
+	obj, err := obj.Pipe(yaml.Lookup(subjectsField))
+	if err != nil || yaml.IsEmpty(obj) {
+		return err
+	}
+
+	// add the namespace to each "subject" with name: default
+	err = obj.VisitElements(func(o *yaml.RNode) error {
+		// copied from kunstruct based kustomize NamespaceTransformer plugin
+		// The only case we need to force the namespace
+		// if for the "service account". "default" is
+		// kind of hardcoded here for right now.
+		name, err := o.Pipe(
+			yaml.Lookup("name"), yaml.Match("default"),
+		)
+		if err != nil || yaml.IsEmpty(name) {
+			return err
+		}
+
+		// set the namespace for the default account
+		v := yaml.NewScalarRNode(ns.Namespace)
+		return o.PipeE(
+			yaml.LookupCreate(yaml.ScalarNode, "namespace"),
+			yaml.FieldSetter{Value: v},
+		)
+	})
+
+	return err
+}
+
+// removeFieldSpecsForHacks removes from the list fieldspecs that
+// have hardcoded implementations
+func (ns Filter) removeFieldSpecsForHacks(fs types.FsSlice) types.FsSlice {
+	var val types.FsSlice
+	for i := range fs {
+		// implemented by metaNamespaceHack
+		if fs[i].Path == types.MetadataNamespacePath {
+			continue
+		}
+		// implemented by roleBindingHack
+		if fs[i].Kind == roleBindingKind && fs[i].Path == subjectsField {
+			continue
+		}
+		// implemented by roleBindingHack
+		if fs[i].Kind == clusterRoleBindingKind && fs[i].Path == subjectsField {
+			continue
+		}
+		val = append(val, fs[i])
+	}
+	return val
+}
+
+const (
+	subjectsField          = "subjects"
+	roleBindingKind        = "RoleBinding"
+	clusterRoleBindingKind = "ClusterRoleBinding"
+)

api/filters/namespace/namespace_test.go

@@ -0,0 +1,280 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package namespace_test
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"sigs.k8s.io/kustomize/api/filters/namespace"
+	"sigs.k8s.io/kustomize/api/internal/plugins/builtinconfig"
+	filtertest_test "sigs.k8s.io/kustomize/api/testutils/filtertest"
+	"sigs.k8s.io/kustomize/api/types"
+)
+
+var tests = []TestCase{
+	{
+		name: "add",
+		input: `
+apiVersion: example.com/v1
+kind: Foo
+metadata:
+  name: instance
+---
+apiVersion: example.com/v1
+kind: Bar
+metadata:
+  name: instance
+`,
+		expected: `
+apiVersion: example.com/v1
+kind: Foo
+metadata:
+  name: instance
+  namespace: foo
+---
+apiVersion: example.com/v1
+kind: Bar
+metadata:
+  name: instance
+  namespace: foo
+`,
+		filter: namespace.Filter{Namespace: "foo"},
+	},
+
+	{
+		name: "add-recurse",
+		input: `
+apiVersion: example.com/v1
+kind: Foo
+---
+apiVersion: example.com/v1
+kind: Bar
+`,
+		expected: `
+apiVersion: example.com/v1
+kind: Foo
+metadata:
+  namespace: foo
+---
+apiVersion: example.com/v1
+kind: Bar
+metadata:
+  namespace: foo
+`,
+		filter: namespace.Filter{Namespace: "foo"},
+	},
+
+	{
+		name: "update",
+		input: `
+apiVersion: example.com/v1
+kind: Foo
+metadata:
+  name: instance
+  # update this namespace
+  namespace: bar
+---
+apiVersion: example.com/v1
+kind: Bar
+metadata:
+  name: instance
+  namespace: bar
+`,
+		expected: `
+apiVersion: example.com/v1
+kind: Foo
+metadata:
+  name: instance
+  # update this namespace
+  namespace: foo
+---
+apiVersion: example.com/v1
+kind: Bar
+metadata:
+  name: instance
+  namespace: foo
+`,
+		filter: namespace.Filter{Namespace: "foo"},
+	},
+
+	{
+		name: "update-rolebinding",
+		input: `
+apiVersion: example.com/v1
+kind: RoleBinding
+subjects:
+- name: default
+---
+apiVersion: example.com/v1
+kind: RoleBinding
+subjects:
+- name: default
+  namespace: foo
+---
+apiVersion: example.com/v1
+kind: RoleBinding
+subjects:
+- name: something
+---
+apiVersion: example.com/v1
+kind: RoleBinding
+subjects:
+- name: something
+  namespace: foo
+`,
+		expected: `
+apiVersion: example.com/v1
+kind: RoleBinding
+subjects:
+- name: default
+  namespace: bar
+metadata:
+  namespace: bar
+---
+apiVersion: example.com/v1
+kind: RoleBinding
+subjects:
+- name: default
+  namespace: bar
+metadata:
+  namespace: bar
+---
+apiVersion: example.com/v1
+kind: RoleBinding
+subjects:
+- name: something
+metadata:
+  namespace: bar
+---
+apiVersion: example.com/v1
+kind: RoleBinding
+subjects:
+- name: something
+  namespace: foo
+metadata:
+  namespace: bar
+`,
+		filter: namespace.Filter{Namespace: "bar"},
+	},
+
+	{
+		name: "update-clusterrolebinding",
+		input: `
+apiVersion: example.com/v1
+kind: ClusterRoleBinding
+subjects:
+- name: default
+---
+apiVersion: example.com/v1
+kind: ClusterRoleBinding
+subjects:
+- name: default
+  namespace: foo
+---
+apiVersion: example.com/v1
+kind: ClusterRoleBinding
+subjects:
+- name: something
+---
+apiVersion: example.com/v1
+kind: ClusterRoleBinding
+subjects:
+- name: something
+  namespace: foo
+`,
+		expected: `
+apiVersion: example.com/v1
+kind: ClusterRoleBinding
+subjects:
+- name: default
+  namespace: bar
+---
+apiVersion: example.com/v1
+kind: ClusterRoleBinding
+subjects:
+- name: default
+  namespace: bar
+---
+apiVersion: example.com/v1
+kind: ClusterRoleBinding
+subjects:
+- name: something
+---
+apiVersion: example.com/v1
+kind: ClusterRoleBinding
+subjects:
+- name: something
+  namespace: foo
+`,
+		filter: namespace.Filter{Namespace: "bar"},
+	},
+
+	{
+		name: "data-fieldspecs",
+		input: `
+apiVersion: example.com/v1
+kind: Foo
+metadata:
+  name: instance
+---
+apiVersion: example.com/v1
+kind: Bar
+metadata:
+  name: instance
+`,
+		expected: `
+apiVersion: example.com/v1
+kind: Foo
+metadata:
+  name: instance
+  namespace: foo
+a:
+  b:
+    c: foo
+---
+apiVersion: example.com/v1
+kind: Bar
+metadata:
+  name: instance
+  namespace: foo
+a:
+  b:
+    c: foo
+`,
+		filter: namespace.Filter{Namespace: "foo"},
+		fsslice: []types.FieldSpec{
+			{
+				Path:               "a/b/c",
+				CreateIfNotPresent: true,
+			},
+		},
+	},
+}
+
+type TestCase struct {
+	name     string
+	input    string
+	expected string
+	filter   namespace.Filter
+	fsslice  types.FsSlice
+}
+
+var config = builtinconfig.MakeDefaultConfig()
+
+func TestNamespace_Filter(t *testing.T) {
+	for i := range tests {
+		test := tests[i]
+		t.Run(test.name, func(t *testing.T) {
+			test.filter.FsSlice = append(config.NameSpace, test.fsslice...)
+			if !assert.Equal(t,
+				strings.TrimSpace(test.expected),
+				strings.TrimSpace(
+					filtertest_test.RunFilter(t, test.input, test.filter))) {
+				t.FailNow()
+			}
+		})
+	}
+}

api/filters/patchjson6902/doc.go

@@ -0,0 +1,6 @@
+// Copyright 2020 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+// Package namespace contains a kio.Filter implementation of the kustomize
+// patchjson6902 transformer
+package patchjson6902

api/filters/patchjson6902/example_test.go

@@ -0,0 +1,55 @@
+// Copyright 2020 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package patchjson6902
+
+import (
+	"bytes"
+	"log"
+	"os"
+
+	"sigs.k8s.io/kustomize/kyaml/kio"
+)
+
+func ExampleFilter() {
+	err := kio.Pipeline{
+		Inputs: []kio.Reader{&kio.ByteReader{Reader: bytes.NewBufferString(`
+apiVersion: example.com/v1
+kind: Foo
+metadata:
+  name: instance
+---
+apiVersion: example.com/v1
+kind: Bar
+metadata:
+  name: instance
+  namespace: bar
+`)}},
+		Filters: []kio.Filter{
+			Filter{
+				Patch: `
+- op: replace
+  path: /metadata/namespace
+  value: "ns"
+`,
+			},
+		},
+		Outputs: []kio.Writer{kio.ByteWriter{Writer: os.Stdout}},
+	}.Execute()
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	// Output:
+	// apiVersion: example.com/v1
+	// kind: Foo
+	// metadata:
+	//   name: instance
+	//   namespace: ns
+	// ---
+	// apiVersion: example.com/v1
+	// kind: Bar
+	// metadata:
+	//   name: instance
+	//   namespace: ns
+}

api/filters/patchjson6902/patchjson6902.go

@@ -0,0 +1,65 @@
+// Copyright 2020 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package patchjson6902
+
+import (
+	"strings"
+
+	jsonpatch "github.com/evanphx/json-patch"
+	"sigs.k8s.io/kustomize/kyaml/kio"
+	"sigs.k8s.io/kustomize/kyaml/yaml"
+	k8syaml "sigs.k8s.io/yaml"
+)
+
+type Filter struct {
+	Patch string
+
+	decodedPatch jsonpatch.Patch
+}
+
+var _ kio.Filter = Filter{}
+
+func (pf Filter) Filter(nodes []*yaml.RNode) ([]*yaml.RNode, error) {
+	decodedPatch, err := pf.decodePatch()
+	if err != nil {
+		return nil, err
+	}
+	pf.decodedPatch = decodedPatch
+	return kio.FilterAll(yaml.FilterFunc(pf.run)).Filter(nodes)
+}
+
+func (pf Filter) decodePatch() (jsonpatch.Patch, error) {
+	patch := pf.Patch
+	// If the patch doesn't look like a JSON6902 patch, we
+	// try to parse it to json.
+	if !strings.HasPrefix(pf.Patch, "[") {
+		p, err := k8syaml.YAMLToJSON([]byte(patch))
+		if err != nil {
+			return nil, err
+		}
+		patch = string(p)
+	}
+	decodedPatch, err := jsonpatch.DecodePatch([]byte(patch))
+	if err != nil {
+		return nil, err
+	}
+	return decodedPatch, nil
+}
+
+func (pf Filter) run(node *yaml.RNode) (*yaml.RNode, error) {
+	// We don't actually use the kyaml library for manipulating the
+	// yaml here. We just marshal it to json and rely on the
+	// jsonpatch library to take care of applying the patch.
+	// This means ordering might not be preserved with this filter.
+	b, err := node.MarshalJSON()
+	if err != nil {
+		return nil, err
+	}
+	res, err := pf.decodedPatch.Apply(b)
+	if err != nil {
+		return nil, err
+	}
+	err = node.UnmarshalJSON(res)
+	return node, err
+}

api/filters/patchjson6902/patchjson6902_test.go

@@ -0,0 +1,173 @@
+// Copyright 2020 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package patchjson6902
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	filtertest "sigs.k8s.io/kustomize/api/testutils/filtertest"
+)
+
+const input = `
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: myDeploy
+spec:
+  replica: 2
+  template:
+    metadata:
+      labels:
+        old-label: old-value
+    spec:
+      containers:
+      - image: nginx
+        name: nginx
+`
+
+func TestSomething(t *testing.T) {
+	testCases := []struct {
+		testName       string
+		input          string
+		filter         Filter
+		expectedOutput string
+	}{
+		{
+			testName: "single operation, json",
+			input:    input,
+			filter: Filter{
+				Patch: `[
+{"op": "replace", "path": "/spec/replica", "value": 5}
+]`,
+			},
+			expectedOutput: `
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: myDeploy
+spec:
+  replica: 5
+  template:
+    metadata:
+      labels:
+        old-label: old-value
+    spec:
+      containers:
+      - image: nginx
+        name: nginx
+`,
+		},
+		{
+			testName: "multiple operations, json",
+			input:    input,
+			filter: Filter{
+				Patch: `[
+{"op": "replace", "path": "/spec/template/spec/containers/0/name", "value": "my-nginx"},
+{"op": "add", "path": "/spec/replica", "value": 999},
+{"op": "add", "path": "/spec/template/spec/containers/0/command", "value": ["arg1", "arg2", "arg3"]}
+]`,
+			},
+			expectedOutput: `
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: myDeploy
+spec:
+  replica: 999
+  template:
+    metadata:
+      labels:
+        old-label: old-value
+    spec:
+      containers:
+      - command:
+        - arg1
+        - arg2
+        - arg3
+        image: nginx
+        name: my-nginx
+`,
+		},
+		{
+			testName: "single operation, yaml",
+			input:    input,
+			filter: Filter{
+				Patch: `
+- op: replace
+  path: /spec/replica
+  value: 5
+`,
+			},
+			expectedOutput: `
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: myDeploy
+spec:
+  replica: 5
+  template:
+    metadata:
+      labels:
+        old-label: old-value
+    spec:
+      containers:
+      - image: nginx
+        name: nginx
+`,
+		},
+		{
+			testName: "multiple operations, yaml",
+			input:    input,
+			filter: Filter{
+				Patch: `
+- op: replace
+  path: /spec/template/spec/containers/0/name
+  value: my-nginx
+- op: add
+  path: /spec/replica
+  value: 999
+- op: add
+  path: /spec/template/spec/containers/0/command
+  value:
+  - arg1
+  - arg2
+  - arg3
+`,
+			},
+			expectedOutput: `
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: myDeploy
+spec:
+  replica: 999
+  template:
+    metadata:
+      labels:
+        old-label: old-value
+    spec:
+      containers:
+      - command:
+        - arg1
+        - arg2
+        - arg3
+        image: nginx
+        name: my-nginx
+`,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.testName, func(t *testing.T) {
+			if !assert.Equal(t,
+				strings.TrimSpace(tc.expectedOutput),
+				strings.TrimSpace(
+					filtertest.RunFilter(t, tc.input, tc.filter))) {
+				t.FailNow()
+			}
+		})
+	}
+}

api/filters/patchstrategicmerge/doc.go

@@ -0,0 +1,6 @@
+// Copyright 2020 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+// Package patchstrategicmerge contains a kio.Filter implementation of the
+// kustomize strategic merge patch transformer.
+package patchstrategicmerge

api/filters/patchstrategicmerge/example_test.go

@@ -0,0 +1,49 @@
+// Copyright 2020 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package patchstrategicmerge
+
+import (
+	"bytes"
+	"log"
+	"os"
+
+	"sigs.k8s.io/kustomize/kyaml/kio"
+	"sigs.k8s.io/kustomize/kyaml/yaml"
+)
+
+func ExampleFilter() {
+	err := kio.Pipeline{
+		Inputs: []kio.Reader{&kio.ByteReader{Reader: bytes.NewBufferString(`
+apiVersion: example.com/v1
+kind: Foo
+metadata:
+  name: instance
+spec:
+  replicas: 3
+`)}},
+		Filters: []kio.Filter{Filter{
+			Patch: yaml.MustParse(`
+spec:
+  template:
+    containers:
+    - image: nginx
+`),
+		}},
+		Outputs: []kio.Writer{kio.ByteWriter{Writer: os.Stdout}},
+	}.Execute()
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	// Output:
+	// apiVersion: example.com/v1
+	// kind: Foo
+	// metadata:
+	//   name: instance
+	// spec:
+	//   replicas: 3
+	//   template:
+	//     containers:
+	//     - image: nginx
+}

api/filters/patchstrategicmerge/patchstrategicmerge.go

@@ -0,0 +1,21 @@
+package patchstrategicmerge
+
+import (
+	"sigs.k8s.io/kustomize/kyaml/kio"
+	"sigs.k8s.io/kustomize/kyaml/yaml"
+	"sigs.k8s.io/kustomize/kyaml/yaml/merge2"
+)
+
+type Filter struct {
+	Patch *yaml.RNode
+}
+
+var _ kio.Filter = Filter{}
+
+func (pf Filter) Filter(nodes []*yaml.RNode) ([]*yaml.RNode, error) {
+	return kio.FilterAll(yaml.FilterFunc(pf.run)).Filter(nodes)
+}
+
+func (pf Filter) run(node *yaml.RNode) (*yaml.RNode, error) {
+	return merge2.Merge(pf.Patch, node)
+}

api/filters/patchstrategicmerge/patchstrategicmerge_test.go

@@ -0,0 +1,82 @@
+package patchstrategicmerge
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	filtertest "sigs.k8s.io/kustomize/api/testutils/filtertest"
+	"sigs.k8s.io/kustomize/kyaml/yaml"
+)
+
+func TestFilter(t *testing.T) {
+	testCases := map[string]struct {
+		input    string
+		patch    *yaml.RNode
+		expected string
+	}{
+		"simple patch": {
+			input: `
+apiVersion: apps/v1
+metadata:
+  name: myDeploy
+kind: Deployment
+`,
+			patch: yaml.MustParse(`
+metadata:
+  name: yourDeploy
+`),
+			expected: `
+apiVersion: apps/v1
+metadata:
+  name: yourDeploy
+kind: Deployment
+`,
+		},
+		"nested patch": {
+			input: `
+apiVersion: apps/v1
+metadata:
+  name: myDeploy
+kind: Deployment
+spec:
+  containers:
+  - name: nginx
+    args:
+    - abc
+`,
+			patch: yaml.MustParse(`
+spec:
+  containers:
+  - name: nginx
+    args:
+    - def
+`),
+			expected: `
+apiVersion: apps/v1
+metadata:
+  name: myDeploy
+kind: Deployment
+spec:
+  containers:
+  - name: nginx
+    args:
+    - def
+`,
+		},
+	}
+
+	for tn, tc := range testCases {
+		t.Run(tn, func(t *testing.T) {
+			f := Filter{
+				Patch: tc.patch,
+			}
+			if !assert.Equal(t,
+				strings.TrimSpace(tc.expected),
+				strings.TrimSpace(
+					filtertest.RunFilter(t, tc.input, f))) {
+				t.FailNow()
+			}
+		})
+	}
+}

api/filters/prefixsuffix/doc.go

@@ -0,0 +1,6 @@
+// Copyright 2020 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+// Package prefixsuffix contains a kio.Filter implementation of the kustomize
+// PrefixSuffixTransformer.
+package prefixsuffix

api/filters/prefixsuffix/example_test.go

@@ -0,0 +1,47 @@
+// Copyright 2020 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package prefixsuffix_test
+
+import (
+	"bytes"
+	"log"
+	"os"
+
+	"sigs.k8s.io/kustomize/api/filters/prefixsuffix"
+	"sigs.k8s.io/kustomize/api/internal/plugins/builtinconfig"
+	"sigs.k8s.io/kustomize/kyaml/kio"
+)
+
+func ExampleFilter() {
+	fss := builtinconfig.MakeDefaultConfig().NamePrefix
+	err := kio.Pipeline{
+		Inputs: []kio.Reader{&kio.ByteReader{Reader: bytes.NewBufferString(`
+apiVersion: example.com/v1
+kind: Foo
+metadata:
+  name: instance
+---
+apiVersion: example.com/v1
+kind: Bar
+metadata:
+  name: instance
+`)}},
+		Filters: []kio.Filter{prefixsuffix.Filter{Prefix: "baz-", FsSlice: fss}},
+		Outputs: []kio.Writer{kio.ByteWriter{Writer: os.Stdout}},
+	}.Execute()
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	// Output:
+	// apiVersion: example.com/v1
+	// kind: Foo
+	// metadata:
+	//   name: baz-instance
+	// ---
+	// apiVersion: example.com/v1
+	// kind: Bar
+	// metadata:
+	//   name: baz-instance
+}

api/filters/prefixsuffix/prefixsuffix.go

@@ -0,0 +1,43 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package prefixsuffix
+
+import (
+	"fmt"
+
+	"sigs.k8s.io/kustomize/api/filters/fsslice"
+	"sigs.k8s.io/kustomize/api/types"
+	"sigs.k8s.io/kustomize/kyaml/kio"
+	"sigs.k8s.io/kustomize/kyaml/yaml"
+)
+
+// Filter applies resource name prefix's and suffix's using the fieldSpecs
+type Filter struct {
+	Prefix string `json:"prefix,omitempty" yaml:"prefix,omitempty"`
+	Suffix string `json:"suffix,omitempty" yaml:"suffix,omitempty"`
+
+	FsSlice types.FsSlice `json:"fieldSpecs,omitempty" yaml:"fieldSpecs,omitempty"`
+}
+
+var _ kio.Filter = Filter{}
+
+func (ns Filter) Filter(nodes []*yaml.RNode) ([]*yaml.RNode, error) {
+	return kio.FilterAll(yaml.FilterFunc(ns.run)).Filter(nodes)
+}
+
+// Run runs the filter on a single node rather than a slice
+func (ns Filter) run(node *yaml.RNode) (*yaml.RNode, error) {
+	// transformations based on data -- :)
+	err := node.PipeE(fsslice.Filter{
+		FsSlice:    ns.FsSlice,
+		SetValue:   ns.set,
+		CreateKind: yaml.ScalarNode, // Name is a ScalarNode
+	})
+	return node, err
+}
+
+func (ns Filter) set(node *yaml.RNode) error {
+	return fsslice.SetScalar(fmt.Sprintf(
+		"%s%s%s", ns.Prefix, node.YNode().Value, ns.Suffix))(node)
+}

api/filters/prefixsuffix/prefixsuffix_test.go

@@ -0,0 +1,167 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package prefixsuffix_test
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"sigs.k8s.io/kustomize/api/filters/prefixsuffix"
+	"sigs.k8s.io/kustomize/api/internal/plugins/builtinconfig"
+	filtertest_test "sigs.k8s.io/kustomize/api/testutils/filtertest"
+	"sigs.k8s.io/kustomize/api/types"
+)
+
+var tests = []TestCase{
+	{
+		name: "prefix",
+		input: `
+apiVersion: example.com/v1
+kind: Foo
+metadata:
+  name: instance
+---
+apiVersion: example.com/v1
+kind: Bar
+metadata:
+  name: instance
+`,
+		expected: `
+apiVersion: example.com/v1
+kind: Foo
+metadata:
+  name: foo-instance
+---
+apiVersion: example.com/v1
+kind: Bar
+metadata:
+  name: foo-instance
+`,
+		filter: prefixsuffix.Filter{Prefix: "foo-"},
+	},
+
+	{
+		name: "suffix",
+		input: `
+apiVersion: example.com/v1
+kind: Foo
+metadata:
+  name: instance
+---
+apiVersion: example.com/v1
+kind: Bar
+metadata:
+  name: instance
+`,
+		expected: `
+apiVersion: example.com/v1
+kind: Foo
+metadata:
+  name: instance-foo
+---
+apiVersion: example.com/v1
+kind: Bar
+metadata:
+  name: instance-foo
+`,
+		filter: prefixsuffix.Filter{Suffix: "-foo"},
+	},
+
+	{
+		name: "prefix-suffix",
+		input: `
+apiVersion: example.com/v1
+kind: Foo
+metadata:
+  name: instance
+---
+apiVersion: example.com/v1
+kind: Bar
+metadata:
+  name: instance
+`,
+		expected: `
+apiVersion: example.com/v1
+kind: Foo
+metadata:
+  name: bar-instance-foo
+---
+apiVersion: example.com/v1
+kind: Bar
+metadata:
+  name: bar-instance-foo
+`,
+		filter: prefixsuffix.Filter{Prefix: "bar-", Suffix: "-foo"},
+	},
+
+	{
+		name: "data-fieldspecs",
+		input: `
+apiVersion: example.com/v1
+kind: Foo
+metadata:
+  name: instance
+a:
+  b:
+    c: d
+---
+apiVersion: example.com/v1
+kind: Bar
+metadata:
+  name: instance
+a:
+  b:
+    c: d
+`,
+		expected: `
+apiVersion: example.com/v1
+kind: Foo
+metadata:
+  name: foo-instance
+a:
+  b:
+    c: foo-d
+---
+apiVersion: example.com/v1
+kind: Bar
+metadata:
+  name: foo-instance
+a:
+  b:
+    c: foo-d
+`,
+		filter: prefixsuffix.Filter{Prefix: "foo-"},
+		fsslice: []types.FieldSpec{
+			{
+				Path: "a/b/c",
+			},
+		},
+	},
+}
+
+type TestCase struct {
+	name     string
+	input    string
+	expected string
+	filter   prefixsuffix.Filter
+	fsslice  types.FsSlice
+}
+
+var config = builtinconfig.MakeDefaultConfig()
+
+func TestFilter(t *testing.T) {
+	for i := range tests {
+		test := tests[i]
+		t.Run(test.name, func(t *testing.T) {
+			test.filter.FsSlice = append(config.NamePrefix, test.fsslice...)
+			if !assert.Equal(t,
+				strings.TrimSpace(test.expected),
+				strings.TrimSpace(
+					filtertest_test.RunFilter(t, test.input, test.filter))) {
+				t.FailNow()
+			}
+		})
+	}
+}

api/filters/replicacount/doc.go

@@ -0,0 +1,6 @@
+// Copyright 2020 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+// Package replicacount contains a kio.Filter implementation of the kustomize
+// ReplicaCountTransformer.
+package replicacount

api/filters/replicacount/example_test.go

@@ -0,0 +1,64 @@
+package replicacount
+
+import (
+	"bytes"
+	"log"
+	"os"
+
+	"sigs.k8s.io/kustomize/api/types"
+	"sigs.k8s.io/kustomize/kyaml/kio"
+)
+
+func ExampleFilter() {
+	err := kio.Pipeline{
+		Inputs: []kio.Reader{&kio.ByteReader{Reader: bytes.NewBufferString(`
+apiVersion: example.com/v1
+kind: Foo
+metadata:
+  name: instance
+spec:
+  template:
+    replicas: 5
+---
+apiVersion: example.com/v1
+kind: Bar
+metadata:
+  name: instance
+spec:
+  template:
+    replicas: 5
+`)}},
+		Filters: []kio.Filter{Filter{
+			Replica: types.Replica{
+				Count: 42,
+				Name:  "instance",
+			},
+			FsSlice: types.FsSlice{
+				{
+					Path: "spec/template/replicas",
+				},
+			},
+		}},
+		Outputs: []kio.Writer{kio.ByteWriter{Writer: os.Stdout}},
+	}.Execute()
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	// Output:
+	// apiVersion: example.com/v1
+	// kind: Foo
+	// metadata:
+	//   name: instance
+	// spec:
+	//   template:
+	//     replicas: 42
+	// ---
+	// apiVersion: example.com/v1
+	// kind: Bar
+	// metadata:
+	//   name: instance
+	// spec:
+	//   template:
+	//     replicas: 42
+}

api/filters/replicacount/replicacount.go

@@ -0,0 +1,48 @@
+package replicacount
+
+import (
+	"strconv"
+
+	"sigs.k8s.io/kustomize/api/filters/fsslice"
+	"sigs.k8s.io/kustomize/api/types"
+	"sigs.k8s.io/kustomize/kyaml/kio"
+	"sigs.k8s.io/kustomize/kyaml/yaml"
+)
+
+// Filter updates/sets replicas fields using the fieldSpecs
+type Filter struct {
+	Replica types.Replica `json:"replica,omitempty" yaml:"replica,omitempty"`
+
+	// FsSlice contains the FieldSpecs to locate the namespace field
+	FsSlice types.FsSlice `json:"fieldSpecs,omitempty" yaml:"fieldSpecs,omitempty"`
+}
+
+var _ kio.Filter = Filter{}
+
+func (rc Filter) Filter(nodes []*yaml.RNode) ([]*yaml.RNode, error) {
+	return kio.FilterAll(yaml.FilterFunc(rc.run)).Filter(nodes)
+}
+
+// run processes each node individually.
+func (rc Filter) run(node *yaml.RNode) (*yaml.RNode, error) {
+	meta, err := node.GetMeta()
+	if err != nil {
+		return nil, err
+	}
+
+	// only update resources where the name matches the Replica name.
+	if meta.Name != rc.Replica.Name {
+		return node, nil
+	}
+
+	err = node.PipeE(fsslice.Filter{
+		FsSlice:    rc.FsSlice,
+		SetValue:   rc.set,
+		CreateKind: yaml.ScalarNode, // replicas is a ScalarNode
+	})
+	return node, err
+}
+
+func (rc Filter) set(node *yaml.RNode) error {
+	return fsslice.SetScalar(strconv.FormatInt(rc.Replica.Count, 10))(node)
+}

api/filters/replicacount/replicacount_test.go

@@ -0,0 +1,164 @@
+package replicacount
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"sigs.k8s.io/kustomize/api/internal/plugins/builtinconfig"
+	filtertest_test "sigs.k8s.io/kustomize/api/testutils/filtertest"
+	"sigs.k8s.io/kustomize/api/types"
+)
+
+func TestFilter(t *testing.T) {
+	var config = builtinconfig.MakeDefaultConfig()
+
+	testCases := map[string]struct {
+		input    string
+		expected string
+		filter   Filter
+		fsslice  types.FsSlice
+	}{
+		"update field": {
+			input: `
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: dep
+spec:
+  replicas: 5
+`,
+			expected: `
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: dep
+spec:
+  replicas: 42
+`,
+			filter: Filter{
+				Replica: types.Replica{
+					Name:  "dep",
+					Count: 42,
+				},
+			},
+			fsslice: types.FsSlice{
+				{
+					Path: "spec/replicas",
+				},
+			},
+		},
+		"add field": {
+			input: `
+apiVersion: custom/v1
+kind: Custom
+metadata:
+  name: cus
+spec:
+  template:
+    other: something
+`,
+			expected: `
+apiVersion: custom/v1
+kind: Custom
+metadata:
+  name: cus
+spec:
+  template:
+    other: something
+    replicas: 42
+`,
+			filter: Filter{
+				Replica: types.Replica{
+					Name:  "cus",
+					Count: 42,
+				},
+			},
+			fsslice: types.FsSlice{
+				{
+					Path:               "spec/template/replicas",
+					CreateIfNotPresent: true,
+				},
+			},
+		},
+		"no update if CreateIfNotPresent is false": {
+			input: `
+apiVersion: custom/v1
+kind: Custom
+metadata:
+  name: cus
+spec:
+  template:
+    other: something
+`,
+			expected: `
+apiVersion: custom/v1
+kind: Custom
+metadata:
+  name: cus
+spec:
+  template:
+    other: something
+`,
+			filter: Filter{
+				Replica: types.Replica{
+					Name:  "cus",
+					Count: 42,
+				},
+			},
+			fsslice: types.FsSlice{
+				{
+					Path: "spec/template/replicas",
+				},
+			},
+		},
+		"update multiple fields": {
+			input: `
+apiVersion: custom/v1
+kind: Custom
+metadata:
+  name: cus
+spec:
+  replicas: 5
+  template:
+    replicas: 5
+`,
+			expected: `
+apiVersion: custom/v1
+kind: Custom
+metadata:
+  name: cus
+spec:
+  replicas: 42
+  template:
+    replicas: 42
+`,
+			filter: Filter{
+				Replica: types.Replica{
+					Name:  "cus",
+					Count: 42,
+				},
+			},
+			fsslice: types.FsSlice{
+				{
+					Path: "spec/template/replicas",
+				},
+				{
+					Path: "spec/replicas",
+				},
+			},
+		},
+	}
+
+	for tn, tc := range testCases {
+		t.Run(tn, func(t *testing.T) {
+			tc.filter.FsSlice = append(config.Replicas, tc.fsslice...)
+			if !assert.Equal(t,
+				strings.TrimSpace(tc.expected),
+				strings.TrimSpace(
+					filtertest_test.RunFilter(t, tc.input, tc.filter))) {
+				t.FailNow()
+			}
+		})
+	}
+}

api/filters/valueadd/valueadd.go

@@ -0,0 +1,124 @@
+// Copyright 2020 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package valueadd
+
+import (
+	"strings"
+
+	"sigs.k8s.io/kustomize/api/filesys"
+	"sigs.k8s.io/kustomize/kyaml/kio"
+	"sigs.k8s.io/kustomize/kyaml/yaml"
+)
+
+// An 'Add' operation aspiring to IETF RFC 6902 JSON.
+//
+// The filter tries to add a value to a node at a particular field path.
+//
+// Kinds of target fields:
+//
+// - Non-existent target field.
+//
+//   The field will be added and the value inserted.
+//
+// - Existing field, scalar or map.
+//
+//   E.g. 'spec/template/spec/containers/[name:nginx]/image'
+//
+//   This behaves like an IETF RFC 6902 Replace operation would;
+//   the existing value is replaced without complaint, even though
+//   this  is an Add operation.  In contrast, a Replace operation
+//   must fail (report an error) if the field doesn't exist.
+//
+// - Existing field, list (array)
+//   Not supported yet.
+//   TODO: Honor fields with RFC-6902-style array indices
+//   TODO: like 'spec/template/spec/containers/2'
+//   TODO: Modify kyaml/yaml/PathGetter to allow this.
+//   The value will be inserted into the array at the given position,
+//   shifting other contents. To instead replace an array entry, use
+//   an implementation of an IETF RFC 6902 Replace operation.
+//
+// For the common case of a filepath in the field value, and a desire
+// to add the value to the filepath (rather than replace the filepath),
+// use a non-zero value of FilePathPosition (see below).
+type Filter struct {
+	// Value is the value to add.
+	//
+	// Empty values are disallowed, i.e. this filter isn't intended
+	// for use in erasing or removing fields. For that, use a filter
+	// more aligned with the IETF RFC 6902 JSON Remove operation.
+	//
+	// At the time of writing, Value's value should be a simple string,
+	// not a JSON document.  This particular filter focuses on easing
+	// injection of a single-sourced cloud project and/or cluster name
+	// into various fields, especially namespace and various filepath
+	// specifications.
+	Value string
+
+	// FieldPath is a JSON-style path to the field intended to hold the value.
+	FieldPath string
+
+	// FilePathPosition is a filepath field index.
+	//
+	// Call the value of this field _i_.
+	//
+	//   If _i_ is zero, negative or unspecified, this field has no effect.
+	//
+	//   If _i_ is > 0, then it's assumed that
+	//   - 'Value' is a string that can work as a directory or file name,
+	//   - the field value intended for replacement holds a filepath.
+	//
+	// The filepath is split into a string slice, the value is inserted
+	// at position [i-1], shifting the rest of the path to the right.
+	// A value of i==1 puts the new value at the start of the path.
+	// This change never converts an absolute path to a relative path,
+	// meaning adding a new field at position i==1 will preserve a
+	// leading slash. E.g. if Value == 'PEACH'
+	//
+	//                  OLD : NEW                    : FilePathPosition
+	//      --------------------------------------------------------
+	//              {empty} : PEACH                  : irrelevant
+	//                    / : /PEACH                 : irrelevant
+	//                  pie : PEACH/pie              : 1 (or less to prefix)
+	//                 /pie : /PEACH/pie             : 1 (or less to prefix)
+	//                  raw : raw/PEACH              : 2 (or more to postfix)
+	//                 /raw : /raw/PEACH             : 2 (or more to postfix)
+	//      a/nice/warm/pie : a/nice/warm/PEACH/pie  : 4
+	//     /a/nice/warm/pie : /a/nice/warm/PEACH/pie : 4
+	//
+	// For robustness (liberal input, conservative output) FilePathPosition
+	// values that that are too large to index the split filepath result in a
+	// postfix rather than an error.  So use 1 to prefix, 9999 to postfix.
+	FilePathPosition int `json:"filePathPosition,omitempty" yaml:"filePathPosition,omitempty"`
+}
+
+var _ kio.Filter = Filter{}
+
+func (f Filter) Filter(nodes []*yaml.RNode) ([]*yaml.RNode, error) {
+	_, err := kio.FilterAll(yaml.FilterFunc(
+		func(node *yaml.RNode) (*yaml.RNode, error) {
+			fields := strings.Split(f.FieldPath, "/")
+			// TODO: support SequenceNode.
+			// Presumably here one could look for array indices (digits) at
+			// the end of the field path (as described in IETF RFC 6902 JSON),
+			// and if found, take it as a signal that this should be a
+			// SequenceNode instead of a ScalarNode, and insert the value
+			// into the proper slot, shifting every over.
+			n, err := node.Pipe(yaml.LookupCreate(yaml.ScalarNode, fields...))
+			if err != nil {
+				return node, err
+			}
+			// TODO: allow more kinds
+			if err := yaml.ErrorIfInvalid(n, yaml.ScalarNode); err != nil {
+				return nil, err
+			}
+			newValue := f.Value
+			if f.FilePathPosition > 0 {
+				newValue = filesys.InsertPathPart(
+					n.YNode().Value, f.FilePathPosition-1, newValue)
+			}
+			return n.Pipe(yaml.FieldSetter{StringValue: newValue})
+		})).Filter(nodes)
+	return nodes, err
+}

api/filters/valueadd/valueadd_test.go

@@ -0,0 +1,109 @@
+// Copyright 2020 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package valueadd
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	filtertest_test "sigs.k8s.io/kustomize/api/testutils/filtertest"
+)
+
+const someResource = `
+kind: SomeKind
+spec:
+  resourceRef:
+    external: projects/whatever
+`
+
+func TestValueAddFilter(t *testing.T) {
+	testCases := map[string]struct {
+		input          string
+		expectedOutput string
+		filter         Filter
+	}{
+		"simpleAdd": {
+			input: `
+kind: SomeKind
+`,
+			expectedOutput: `
+kind: SomeKind
+spec:
+  resourceRef:
+    external: valueAdded
+`,
+			filter: Filter{
+				Value:     "valueAdded",
+				FieldPath: "spec/resourceRef/external",
+			},
+		},
+		"replaceExisting": {
+			input: someResource,
+			expectedOutput: `
+kind: SomeKind
+spec:
+  resourceRef:
+    external: valueAdded
+`,
+			filter: Filter{
+				Value:     "valueAdded",
+				FieldPath: "spec/resourceRef/external",
+			},
+		},
+		"prefixExisting": {
+			input: someResource,
+			expectedOutput: `
+kind: SomeKind
+spec:
+  resourceRef:
+    external: valueAdded/projects/whatever
+`,
+			filter: Filter{
+				Value:            "valueAdded",
+				FieldPath:        "spec/resourceRef/external",
+				FilePathPosition: 1,
+			},
+		},
+		"postfixExisting": {
+			input: someResource,
+			expectedOutput: `
+kind: SomeKind
+spec:
+  resourceRef:
+    external: projects/whatever/valueAdded
+`,
+			filter: Filter{
+				Value:            "valueAdded",
+				FieldPath:        "spec/resourceRef/external",
+				FilePathPosition: 99,
+			},
+		},
+		"placeInMiddleOfExisting": {
+			input: someResource,
+			expectedOutput: `
+kind: SomeKind
+spec:
+  resourceRef:
+    external: projects/valueAdded/whatever
+`,
+			filter: Filter{
+				Value:            "valueAdded",
+				FieldPath:        "spec/resourceRef/external",
+				FilePathPosition: 2,
+			},
+		},
+	}
+
+	for tn, tc := range testCases {
+		t.Run(tn, func(t *testing.T) {
+			filter := tc.filter
+			if !assert.Equal(t,
+				strings.TrimSpace(tc.expectedOutput),
+				strings.TrimSpace(filtertest_test.RunFilter(t, tc.input, filter))) {
+				t.FailNow()
+			}
+		})
+	}
+}

api/go.mod

@@ -4,16 +4,18 @@ go 1.13
 
 require (
 	github.com/evanphx/json-patch v4.5.0+incompatible
-	github.com/go-openapi/spec v0.19.4
+	github.com/go-openapi/spec v0.19.5
 	github.com/golangci/golangci-lint v1.21.0
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510
-	github.com/monopole/mdrip v1.0.1
 	github.com/pkg/errors v0.8.1
+	github.com/stretchr/testify v1.4.0
+	github.com/yujunz/go-getter v1.4.1-lite
 	golang.org/x/tools v0.0.0-20191010075000-0337d82405ff
-	gopkg.in/yaml.v2 v2.2.4
+	gopkg.in/yaml.v2 v2.2.8
 	k8s.io/api v0.17.0
 	k8s.io/apimachinery v0.17.0
 	k8s.io/client-go v0.17.0
 	k8s.io/kube-openapi v0.0.0-20191107075043-30be4d16710a
-	sigs.k8s.io/yaml v1.1.0
+	sigs.k8s.io/kustomize/kyaml v0.1.10
+	sigs.k8s.io/yaml v1.2.0
 )

api/go.sum

@@ -1,15 +1,12 @@
 cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
+github.com/360EntSecGroup-Skylar/excelize v1.4.1/go.mod h1:vnax29X2usfl7HHkBrX5EvSCJcmH3dT9luvxzu8iGAE=
 github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI=
-github.com/Azure/go-autorest/autorest v0.9.2/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI=
 github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0=
-github.com/Azure/go-autorest/autorest/adal v0.8.0/go.mod h1:Z6vX6WXXuyieHAXwMj0S6HY6e6wcHn37qQMBQlvY3lc=
 github.com/Azure/go-autorest/autorest/date v0.1.0/go.mod h1:plvfp3oPSKwf2DNjlBjWF/7vwR+cUD/ELuzDCXwHUVA=
-github.com/Azure/go-autorest/autorest/date v0.2.0/go.mod h1:vcORJHLJEh643/Ioh9+vPmf1Ij9AEBM5FuBIXLmIy0g=
 github.com/Azure/go-autorest/autorest/mocks v0.1.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0=
 github.com/Azure/go-autorest/autorest/mocks v0.2.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0=
-github.com/Azure/go-autorest/autorest/mocks v0.3.0/go.mod h1:a8FDP3DYzQ4RYfVAxAN3SVSiiO77gL2j2ronKKP0syM=
 github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc=
 github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk=
 github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
@@ -18,6 +15,7 @@ github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb0
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
 github.com/OpenPeeDeeP/depguard v1.0.1 h1:VlW4R6jmBIv3/u1JNlawEvJMM4J+dPORPaZasQee8Us=
 github.com/OpenPeeDeeP/depguard v1.0.1/go.mod h1:xsIw86fROiiwelg+jB2uM9PiKihMMmUx/1V+TNhjQvM=
+github.com/PuerkitoBio/goquery v1.5.0/go.mod h1:qD2PgZ9lccMbQlc7eEOjaeRlFQON7xY8kdmcsrnKqMg=
 github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
 github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI=
 github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
@@ -27,12 +25,18 @@ github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdko
 github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
+github.com/andybalholm/cascadia v1.0.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
+github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d h1:xDfNPAt8lFiC1UJrqV3uuy861HCTo708pDMbjHHdCas=
+github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d/go.mod h1:6QX/PXZ00z/TKoufEY6K/a0k6AhaJrQKdFe6OfVXsa4=
 github.com/bombsimon/wsl v1.2.5 h1:9gTOkIwVtoDZywvX802SDHokeX4kW1cKnV8ZTVAPkRs=
 github.com/bombsimon/wsl v1.2.5/go.mod h1:43lEF/i0kpXbLCeDXL9LMT8c92HyBywXb0AsgMHYngM=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
+github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
+github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
+github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
 github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
@@ -41,6 +45,7 @@ github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3Ee
 github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
 github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
 github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE=
+github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
 github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
 github.com/davecgh/go-spew v0.0.0-20151105211317-5215b55f46b2/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -49,14 +54,10 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
 github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
 github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM=
-github.com/docker/spdystream v0.0.0-20181023171402-6480d4af844c/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM=
+github.com/dustmop/soup v1.1.2-0.20190516214245-38228baa104e/go.mod h1:CgNC6SGbT+Xb8wGGvzilttZL1mc5sQ/5KkcxsZttMIk=
 github.com/elazarl/goproxy v0.0.0-20170405201442-c4fc26588b6e/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
-github.com/elazarl/goproxy v0.0.0-20191011121108-aa519ddbe484/go.mod h1:Ro8st/ElPeALwNFlcTpWmkr6IoMFfkjXAvTHpevnDsM=
-github.com/elazarl/goproxy/ext v0.0.0-20190711103511-473e67f1d7d2/go.mod h1:gNh8nYJoAm43RfaxurUnxr+N1PwuFV3ZMl/efxlIlY8=
 github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633 h1:H2pdYOb3KQ1/YsqVWoWNLQO+fusocsw354rqGTZtAgw=
 github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
-github.com/emicklei/go-restful v2.9.6+incompatible h1:tfrHha8zJ01ywiOEC1miGY8st1/igzWB8OmvPgoYX7w=
-github.com/emicklei/go-restful v2.9.6+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
 github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
 github.com/evanphx/json-patch v4.5.0+incompatible h1:ouOWdg56aJriqS0huScTkVXPC5IcNrDCXZ6OoTAWu7M=
 github.com/evanphx/json-patch v4.5.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
@@ -68,6 +69,8 @@ github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2H
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/go-critic/go-critic v0.3.5-0.20190904082202-d79a9f0c64db h1:GYXWx7Vr3+zv833u+8IoXbNnQY0AdXsxAgI0kX7xcwA=
 github.com/go-critic/go-critic v0.3.5-0.20190904082202-d79a9f0c64db/go.mod h1:+sE8vrLDS2M0pZkBk0wy6+nLdKexVDrl/jBqQOTDThA=
+github.com/go-errors/errors v1.0.1 h1:LUHzmkK3GUKUrL/1gfBUxAHzcev3apQlezX/+O7ma6w=
+github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q=
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-lintpack/lintpack v0.5.2 h1:DI5mA3+eKdWeJ40nU4d6Wc26qmdG8RCi/btYq0TuRN0=
 github.com/go-lintpack/lintpack v0.5.2/go.mod h1:NwZuYi2nUHho8XEIZ6SIxihrnPoqBTDqfpXvXAN0sXM=
@@ -83,8 +86,8 @@ github.com/go-openapi/jsonreference v0.0.0-20160704190145-13c6e3589ad9/go.mod h1
 github.com/go-openapi/jsonreference v0.19.2 h1:o20suLFB4Ri0tuzpWtyHlh7E7HnkqTNLq6aR6WVNS1w=
 github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc=
 github.com/go-openapi/spec v0.0.0-20160808142527-6aced65f8501/go.mod h1:J8+jY1nAiCcj+friV/PDoE1/3eeccG9LYBs0tYvLOWc=
-github.com/go-openapi/spec v0.19.4 h1:ixzUSnHTd6hCemgtAJgluaTSGYpLNpJY4mA2DIkdOAo=
-github.com/go-openapi/spec v0.19.4/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo=
+github.com/go-openapi/spec v0.19.5 h1:Xm0Ao53uqnk9QE/LlYV5DEU09UAgpliA85QoT9LzqPw=
+github.com/go-openapi/spec v0.19.5/go.mod h1:Hm2Jr4jv8G1ciIAo+frC/Ft+rR2kQDh8JHKHb3gWUSk=
 github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dpr1UfpPtxFw+EFuQ41HhCWZfha5jSVRG7C7I=
 github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
 github.com/go-openapi/swag v0.19.5 h1:lTz6Ys4CmqqCQmZPBlbQENR1/GucA2bzYTE12Pw4tFY=
@@ -119,13 +122,10 @@ github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7a
 github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
 github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d h1:3PaI8p3seN09VjbTYC/QWlUZdZ1qS1zGjy7LH2Wt07I=
 github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
-github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls=
-github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
-github.com/golang/groupcache v0.0.0-20191027212112-611e8accdfc9/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/protobuf v0.0.0-20161109072736-4bd1920723d7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
@@ -145,15 +145,12 @@ github.com/golangci/goconst v0.0.0-20180610141641-041c5f2b40f3 h1:pe9JHs3cHHDQgO
 github.com/golangci/goconst v0.0.0-20180610141641-041c5f2b40f3/go.mod h1:JXrF4TWy4tXYn62/9x8Wm/K/dm06p8tCKwFRDPZG/1o=
 github.com/golangci/gocyclo v0.0.0-20180528134321-2becd97e67ee h1:J2XAy40+7yz70uaOiMbNnluTg7gyQhtGqLQncQh+4J8=
 github.com/golangci/gocyclo v0.0.0-20180528134321-2becd97e67ee/go.mod h1:ozx7R9SIwqmqf5pRP90DhR2Oay2UIjGuKheCBCNwAYU=
-github.com/golangci/gofmt v0.0.0-20181222123516-0b8337e80d98/go.mod h1:9qCChq59u/eW8im404Q2WWTrnBUQKjpNYKMbU4M7EFU=
 github.com/golangci/gofmt v0.0.0-20190930125516-244bba706f1a h1:iR3fYXUjHCR97qWS8ch1y9zPNsgXThGwjKPrYfqMPks=
 github.com/golangci/gofmt v0.0.0-20190930125516-244bba706f1a/go.mod h1:9qCChq59u/eW8im404Q2WWTrnBUQKjpNYKMbU4M7EFU=
-github.com/golangci/golangci-lint v1.19.1/go.mod h1:2CEc4Fxx3vxDv7g8DyXkHCBF73AOzAymcJAprs2vCps=
 github.com/golangci/golangci-lint v1.21.0 h1:HxAxpR8Z0M8omihvQdsD3PF0qPjlqYqp2vMJzstoKeI=
 github.com/golangci/golangci-lint v1.21.0/go.mod h1:phxpHK52q7SE+5KpPnti4oZTdFCEsn/tKN+nFvCKXfk=
 github.com/golangci/ineffassign v0.0.0-20190609212857-42439a7714cc h1:gLLhTLMk2/SutryVJ6D4VZCU3CUqr8YloG7FPIBWFpI=
 github.com/golangci/ineffassign v0.0.0-20190609212857-42439a7714cc/go.mod h1:e5tpTHCfVze+7EpLEozzMB3eafxo2KT5veNg1k6byQU=
-github.com/golangci/lint-1 v0.0.0-20190420132249-ee948d087217/go.mod h1:66R6K6P6VWk9I95jvqGxkqJxVWGFy9XlDwLwVz1RCFg=
 github.com/golangci/lint-1 v0.0.0-20191013205115-297bf364a8e0 h1:MfyDlzVjl1hoaPzPD4Gpb/QgoRfSBR0jdhwGyAWwMSA=
 github.com/golangci/lint-1 v0.0.0-20191013205115-297bf364a8e0/go.mod h1:66R6K6P6VWk9I95jvqGxkqJxVWGFy9XlDwLwVz1RCFg=
 github.com/golangci/maligned v0.0.0-20180506175553-b1d89398deca h1:kNY3/svz5T29MYHubXix4aDDuE3RWHkPvopM/EDv/MA=
@@ -169,9 +166,8 @@ github.com/golangci/unconvert v0.0.0-20180507085042-28b1c447d1f4/go.mod h1:Izgrg
 github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
-github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg=
-github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/gofuzz v0.0.0-20161122191042-44d81051d367/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI=
 github.com/google/gofuzz v1.0.0 h1:A8PeW59pxE9IoFRqBp37U+mSNaQoZ46F1f0f863XSXw=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
@@ -184,31 +180,22 @@ github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
 github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d h1:7XGaL1e6bYS1yIonGp9761ExpPPV1ui0SAC59Yube9k=
 github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY=
-github.com/googleapis/gnostic v0.3.0 h1:CcQijm0XKekKjP/YCz28LXVSpgguuB+nCxaSjCe09y0=
-github.com/googleapis/gnostic v0.3.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY=
 github.com/gophercloud/gophercloud v0.1.0/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEoIEcSTewFxm1c5g8=
-github.com/gophercloud/gophercloud v0.6.0/go.mod h1:GICNByuaEBibcjmjvI7QvYJSZEbGkcYwAR7EZK2WMqM=
-github.com/gorilla/context v0.0.0-20160226214623-1ea25387ff6f h1:9oNbS1z4rVpbnkHBdPZU4jo9bSmrLpII768arSyMFgk=
-github.com/gorilla/context v0.0.0-20160226214623-1ea25387ff6f/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg=
-github.com/gorilla/mux v1.6.0 h1:UykbtMB/w5No2LmE16gINgLj+r/vbziTgaoERQv6U+0=
-github.com/gorilla/mux v1.6.0/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
-github.com/gorilla/securecookie v0.0.0-20160422134519-667fe4e3466a h1:YH0IojQwndMQdeRWdw1aPT8bkbiWaYR3WD+Zf5e09DU=
-github.com/gorilla/securecookie v0.0.0-20160422134519-667fe4e3466a/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4=
-github.com/gorilla/sessions v0.0.0-20160922145804-ca9ada445741 h1:OuuPl66BpF1q3OEkaPpp+VfzxrBBY62ATGdWqql/XX8=
-github.com/gorilla/sessions v0.0.0-20160922145804-ca9ada445741/go.mod h1:+WVp8kdw6VhyKExm03PAMRn2ZxnPtm58pV0dBVPdhHE=
-github.com/gorilla/websocket v1.2.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
-github.com/gorilla/websocket v1.4.0 h1:WDFjx/TMzVgy9VdMMQi2K2Emtwi2QcUQsztZ/zLaH/Q=
 github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
 github.com/gostaticanalysis/analysisutil v0.0.0-20190318220348-4088753ea4d3 h1:JVnpOZS+qxli+rgVl98ILOXVNbW+kb5wcxeGx8ShUIw=
 github.com/gostaticanalysis/analysisutil v0.0.0-20190318220348-4088753ea4d3/go.mod h1:eEOZF4jCKGi+aprrirO9e7WKB3beBRtWgqGunKl6pKE=
 github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
-github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
 github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
 github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
+github.com/hashicorp/go-cleanhttp v0.5.0 h1:wvCrVc9TjDls6+YGAF2hAifE1E5U1+b4tH6KdvN3Gig=
+github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
+github.com/hashicorp/go-safetemp v1.0.0 h1:2HR189eFNrjHQyENnQMMpCiBAsRxzbTMIgBhEyExpmo=
+github.com/hashicorp/go-safetemp v1.0.0/go.mod h1:oaerMy3BhqiTbVye6QuFhFtIceqFoDHxNAB65b+Rj1I=
+github.com/hashicorp/go-version v1.1.0 h1:bPIoEKD27tNdebFGGxxYwcL4nepeY4j1QP23PFRGzg0=
+github.com/hashicorp/go-version v1.1.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
-github.com/hashicorp/golang-lru v0.5.3/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
 github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
 github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
@@ -236,7 +223,6 @@ github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFB
 github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
-github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA=
 github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw=
 github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
@@ -248,19 +234,20 @@ github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN
 github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e h1:hB2xlXdHp/pmPZq0y3QnmWAArdw9PqbmotexnWx/FU8=
 github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
-github.com/matoous/godox v0.0.0-20190910121045-032ad8106c86/go.mod h1:1BELzlh859Sh1c6+90blK8lbYy0kwQf1bYlBhBysy1s=
 github.com/matoous/godox v0.0.0-20190911065817-5d6d842e92eb h1:RHba4YImhrUVQDHUCe2BNSOz4tVy2yGyXhvYDvxGgeE=
 github.com/matoous/godox v0.0.0-20190911065817-5d6d842e92eb/go.mod h1:1BELzlh859Sh1c6+90blK8lbYy0kwQf1bYlBhBysy1s=
-github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
 github.com/mattn/go-colorable v0.1.4 h1:snbPLB8fVfU9iwbbo30TPtbLRzwWu6aJS6Xh4eaaviA=
 github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
 github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE=
 github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
 github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
+github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
 github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
 github.com/mitchellh/go-ps v0.0.0-20190716172923-621e5597135b/go.mod h1:r1VsdOzOPt1ZSrGZWFoNhsAedKnEd6r9Np1+5blZCWk=
+github.com/mitchellh/go-testing-interface v1.0.0 h1:fzU/JVNcaqHQEcVFAKeR41fkiLdIPrefOvVG1VZ96U0=
+github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI=
 github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE=
 github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
@@ -270,10 +257,7 @@ github.com/modern-go/reflect2 v0.0.0-20180320133207-05fbef0ca5da/go.mod h1:bx2lN
 github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI=
 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
-github.com/monopole/mdrip v1.0.0 h1:RFDBa+tab6mW+gX4Ww2SZDc4kS6p01FwnLtgz64Il+I=
-github.com/monopole/mdrip v1.0.0/go.mod h1:N1/ppRG9CaPeUKAUHZ3dUlfOT81lTpKZLkyhCvTETwM=
-github.com/monopole/mdrip v1.0.1 h1:LzwNWb8ge3+4iBZmxIE6VfUR5KIxhF7DKdf85t8Yvos=
-github.com/monopole/mdrip v1.0.1/go.mod h1:/7E04hlzRG9Jrp6WILZfYYm/REoJWL2l+MlsCO1eH74=
+github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8=
 github.com/mozilla/tls-observatory v0.0.0-20190404164649-a3c1b6cfecfd/go.mod h1:SrKMQvPiws7F7iqYp8/TX+IhxCYhzr6N/1yb8cwHsGk=
 github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
@@ -288,6 +272,7 @@ github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+
 github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
 github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME=
 github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/paulmach/orb v0.1.3/go.mod h1:VFlX/8C+IQ1p6FTRRKzKoOPJnvEtA5G0Veuqwbu//Vk=
 github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
 github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
@@ -306,23 +291,22 @@ github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y8
 github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
 github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
+github.com/qri-io/starlib v0.4.2-0.20200213133954-ff2e8cd5ef8d/go.mod h1:7DPO4domFU579Ga6E61sB9VFNaniPVwJP5C4bBCu3wA=
 github.com/quasilyte/go-consistent v0.0.0-20190521200055-c6f3937de18c/go.mod h1:5STLWrekHfjyYwxBRVRXNOSewLJ3PWfDJd1VyTS21fI=
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
-github.com/rogpeppe/go-charset v0.0.0-20180617210344-2471d30d28b4/go.mod h1:qgYeAmZ5ZIpBWTGllZSQnw97Dj+woV0toclVaRGI8pc=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
-github.com/russross/blackfriday v2.0.0+incompatible h1:cBXrhZNUf9C+La9/YpS+UHpUT8YD6Td9ZMSU9APFcsk=
-github.com/russross/blackfriday v2.0.0+incompatible/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
-github.com/securego/gosec v0.0.0-20190912120752-140048b2a218/go.mod h1:q6oYAujd2qyeU4cJqIri4LBIgdHXGvxWHZ1E29HNFRE=
+github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/securego/gosec v0.0.0-20191002120514-e680875ea14d h1:BzRvVq1EHuIjxpijCEKpAxzKUUMurOQ4sknehIATRh8=
 github.com/securego/gosec v0.0.0-20191002120514-e680875ea14d/go.mod h1:w5+eXa0mYznDkHaMCXA4XYffjlH+cy1oyKbfzJXa2Do=
+github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0=
+github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
 github.com/shirou/gopsutil v0.0.0-20190901111213-e4ec7b275ada/go.mod h1:WWnYX4lzhCH5h/3YBfyVA3VbLYjlMZZAQcW9ojMexNc=
 github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc=
 github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e h1:MZM7FHLqUHYI0Y/mQAt3d2aYa0SiNms/hFqC9qJYolM=
 github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk=
 github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041 h1:llrF3Fs4018ePo4+G/HV/uQUqEI1HMDjCeOf2V6puPc=
 github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041/go.mod h1:N5mDOmsrJOB+vfqUK+7DmDyjhSLIIBnXo9lvZJj3MWQ=
-github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo=
 github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4=
@@ -338,6 +322,8 @@ github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8=
 github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
 github.com/spf13/cobra v0.0.5 h1:f0B+LkLX6DtmRH1isoNA9VTtNUK9K8xYd28JNNfOv/s=
 github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU=
+github.com/spf13/cobra v1.0.0 h1:6m/oheQuQ13N9ks4hubMG6BnvwOeaJrqSPLahSnczz8=
+github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE=
 github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk=
 github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
 github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
@@ -353,18 +339,19 @@ github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48=
 github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
 github.com/stretchr/testify v0.0.0-20151208002404-e3a8ff8ce365/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/testify v1.2.3-0.20181224173747-660f15d67dbb/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
-github.com/timakin/bodyclose v0.0.0-20190721030226-87058b9bfcec/go.mod h1:Qimiffbc6q9tBWlVV6x0P9sat/ao1xEkREYPPj9hphk=
 github.com/timakin/bodyclose v0.0.0-20190930140734-f7f2e9bca95e h1:RumXZ56IrCj4CL+g1b9OL/oH0QnsF976bC8xQFYUD5Q=
 github.com/timakin/bodyclose v0.0.0-20190930140734-f7f2e9bca95e/go.mod h1:Qimiffbc6q9tBWlVV6x0P9sat/ao1xEkREYPPj9hphk=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=
 github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
+github.com/ulikunitz/xz v0.5.5 h1:pFrO0lVpTBXLpYw+pnLj6TbvHuyjXMfjGeCwSqCVwok=
+github.com/ulikunitz/xz v0.5.5/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8=
 github.com/ultraware/funlen v0.0.2 h1:Av96YVBwwNSe4MLR7iI/BIa3VyI7/djnto/pK3Uxbdo=
 github.com/ultraware/funlen v0.0.2/go.mod h1:Dp4UiAus7Wdb9KUZsYWZEWiRzGuM2kXM1lPbfaF6xhA=
-github.com/ultraware/whitespace v0.0.3/go.mod h1:aVMh/gQve5Maj9hQ/hg+F75lr/X5A89uZnzAmWSineA=
 github.com/ultraware/whitespace v0.0.4 h1:If7Va4cM03mpgrNH9k49/VOicWpGoG70XPBFFODYDsg=
 github.com/ultraware/whitespace v0.0.4/go.mod h1:aVMh/gQve5Maj9hQ/hg+F75lr/X5A89uZnzAmWSineA=
 github.com/uudashr/gocognit v0.0.0-20190926065955-1655d0de0517 h1:ChMKTho2hWKpks/nD/FL2KqM1wuVt62oJeiE8+eFpGs=
@@ -374,9 +361,15 @@ github.com/valyala/fasthttp v1.2.0/go.mod h1:4vX61m6KN+xDduDNwXrhIAVZaZaZiQ1luJk
 github.com/valyala/quicktemplate v1.2.0/go.mod h1:EH+4AkTd43SvgIbQHYu59/cJyxDoOVRUAfrukLPuGJ4=
 github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio=
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
+github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca h1:1CFlNzQhALwjS9mBAUkycX616GzgsuYUOCHA5+HSlXI=
+github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg=
 github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
+github.com/yujunz/go-getter v1.4.1-lite h1:FhvNc94AXMZkfqUwfMKhnQEC9phkphSGdPTL7tIdhOM=
+github.com/yujunz/go-getter v1.4.1-lite/go.mod h1:sbmqxXjyLunH1PkF3n7zSlnVeMvmYUuIl9ZVs/7NyCc=
 go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
+go.starlark.net v0.0.0-20190528202925-30ae18b8564f/go.mod h1:c1/X6cHgvdXj6pUlmWKMkuqRnW4K8x2vwt6JAaaircg=
+go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5/go.mod h1:nmDLcffg48OtT/PSW0Hg7FvpRQsQh5OSqIylirxKC7o=
 go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
 go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
@@ -385,9 +378,7 @@ golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnf
 golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
@@ -396,6 +387,7 @@ golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTk
 golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
 golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -410,11 +402,11 @@ golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR
 golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20190909003024-a7b16738d86b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20190923162816-aa69164e4478 h1:l5EDrHhldLYb3ZRHDUhXF7Om7MvYXnkV9/iQNo1lX6g=
 golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20191004110552-13f9640d40b9 h1:rjwSpXsdiK0dV8/Naq3kAw9ymfAeJIyd0upUIElB+lI=
 golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200226121028-0de0cce0169b h1:0mm1VjtFUOIlE1SbDlwjYaDxZVDP2S5ou6y0gSgXHu8=
+golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -435,11 +427,11 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h
 golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190911201528-7ad0cfa0b7b5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190922100055-0a153f010e69 h1:rOhMmluY6kLMhdnrivzec6lLgaVbMHMn2ISQXJeJ5EM=
 golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191002063906-3421d5a6bb1c h1:Vco5b+cuG5NNfORVxZy6bYZQ7rsigisU1WQFkvQ0L5E=
+golang.org/x/sys v0.0.0-20191002063906-3421d5a6bb1c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -447,7 +439,6 @@ golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
@@ -462,12 +453,9 @@ golang.org/x/tools v0.0.0-20190311215038-5c2858a9cfe5/go.mod h1:LCzVGOaR6xXOjkQ3
 golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190322203728-c1a832b0ad89/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190521203540-521d6ed310dd/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
-golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190719005602-e377ae9d6386/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI=
 golang.org/x/tools v0.0.0-20190910044552-dd2b5c81c578/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20190911230505-6bfd74cf029c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20190912215617-3720d1ec3678/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20190930201159-7c411dea38b0/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191010075000-0337d82405ff h1:XdBG6es/oFDr1HwaxkxgVve7NB281QhxgK/i4voubFs=
 golang.org/x/tools v0.0.0-20191010075000-0337d82405ff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
@@ -492,8 +480,6 @@ gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMy
 gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
 gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
 gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
-gopkg.in/russross/blackfriday.v2 v2.0.0 h1:+FlnIV8DSQnT7NZ43hcVKcdJdzZoeCmJj4Ql8gq5keA=
-gopkg.in/russross/blackfriday.v2 v2.0.0/go.mod h1:6sSBNz/GtOm/pJTuh5UmBK2ZHfmnxGbl2NZg1UliSOI=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
 gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
@@ -501,6 +487,12 @@ gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I=
 gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.7 h1:VUgggvou5XRW9mHwD/yXxIYSMtY0zoKQf/v226p2nyo=
+gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
+gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v3 v3.0.0-20191120175047-4206685974f2 h1:XZx7nhd5GMaZpmDaEHFVafUZC7ya0fuo7cSJ3UCKYmM=
+gopkg.in/yaml.v3 v3.0.0-20191120175047-4206685974f2/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.1-2019.2.3 h1:3JgtbtFHMiCmsznwGVTUWbgGov+pVqnlf1dEJTNAXeM=
@@ -518,7 +510,6 @@ k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8=
 k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I=
 k8s.io/kube-openapi v0.0.0-20191107075043-30be4d16710a h1:UcxjrRMyNx/i/y8G7kPvLyy7rfbeuf1PYyBf973pgyU=
 k8s.io/kube-openapi v0.0.0-20191107075043-30be4d16710a/go.mod h1:1TqjTSzOxsLGIKfj0lK8EeCP7K1iUG65v09OM0/WG5E=
-k8s.io/utils v0.0.0-20191030222137-2b95a09bc58d/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew=
 k8s.io/utils v0.0.0-20191114184206-e782cd3c129f/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew=
 mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed h1:WX1yoOaKQfddO/mLzdV4wptyWgoH/6hwLs7QHTixo0I=
 mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed/go.mod h1:Xkxe497xwlCKkIaQYRfC7CSLworTXY9RMqwhhCm+8Nc=
@@ -526,12 +517,12 @@ mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b h1:DxJ5nJdkhDlLok9K6qO+5290kphD
 mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b/go.mod h1:2odslEg/xrtNQqCYg2/jCoyKnw3vv5biOc3JnIcYfL4=
 mvdan.cc/unparam v0.0.0-20190720180237-d51796306d8f h1:Cq7MalBHYACRd6EesksG1Q8EoIAKOsiZviGKbOLIej4=
 mvdan.cc/unparam v0.0.0-20190720180237-d51796306d8f/go.mod h1:4G1h5nDURzA3bwVMZIVpwbkw+04kSxk3rAtzlimaUJw=
-sigs.k8s.io/kustomize/api v0.2.0/go.mod h1:zVtMg179jW1gr74jo9fc2Ac9dLYLTZZThc3DDb9lDW4=
-sigs.k8s.io/kustomize/pluginator/v2 v2.0.0 h1:sES7e09G19Q0VjRp4ATSYKpTXoWaX8WMSHfw6u3G2Ok=
-sigs.k8s.io/kustomize/pluginator/v2 v2.0.0/go.mod h1:zrXhTv8BAKt0egmZX/8AtMOSFUSWM9YuoHvvqz8/eHE=
-sigs.k8s.io/kustomize/pseudo/k8s v0.1.0/go.mod h1:bl/gVJgYYhJZCZdYU2BfnaKYAlqFkgbJEkpl302jEss=
+sigs.k8s.io/kustomize/kyaml v0.1.10 h1:ZZfBnA/kYa9ZxUFIgI9oq3pWKzzA1gGOKgU0Hv/NhVg=
+sigs.k8s.io/kustomize/kyaml v0.1.10/go.mod h1:mPmeBSRy0LTMv6fSrYSoi2yIFNZVouGKDsTekE5kdhs=
 sigs.k8s.io/structured-merge-diff v0.0.0-20190525122527-15d366b2352e/go.mod h1:wWxsB5ozmmv/SG7nM11ayaAW51xMvak/t1r0CSlcokI=
 sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs=
 sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=
+sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q=
+sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc=
 sourcegraph.com/sqs/pbtypes v0.0.0-20180604144634-d3ebe8f20ae4 h1:JPJh2pk3+X4lXAkZIk2RuE/7/FoK9maXw+TNPJhVS/c=
 sourcegraph.com/sqs/pbtypes v0.0.0-20180604144634-d3ebe8f20ae4/go.mod h1:ketZ/q3QxT9HOBeFhu6RdvsftgpsbFHBF5Cas6cDKZ0=

api/ifc/ifc.go

@@ -75,14 +75,8 @@ type KunstructuredFactory interface {
 	SliceFromBytes([]byte) ([]Kunstructured, error)
 	FromMap(m map[string]interface{}) Kunstructured
 	Hasher() KunstructuredHasher
-	MakeConfigMap(
-		kvLdr KvLoader,
-		options *types.GeneratorOptions,
-		args *types.ConfigMapArgs) (Kunstructured, error)
-	MakeSecret(
-		kvLdr KvLoader,
-		options *types.GeneratorOptions,
-		args *types.SecretArgs) (Kunstructured, error)
+	MakeConfigMap(kvLdr KvLoader, args *types.ConfigMapArgs) (Kunstructured, error)
+	MakeSecret(kvLdr KvLoader, args *types.SecretArgs) (Kunstructured, error)
 }
 
 // KunstructuredHasher returns a hash of the argument

api/image/image.go

@@ -0,0 +1,50 @@
+// Copyright 2020 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package image
+
+import (
+	"regexp"
+	"strings"
+)
+
+// IsImageMatched returns true if the value of t is identical to the
+// image name in the full image name and tag as given by s.
+func IsImageMatched(s, t string) bool {
+	// Tag values are limited to [a-zA-Z0-9_.{}-].
+	// Some tools like Bazel rules_k8s allow tag patterns with {} characters.
+	// More info: https://github.com/bazelbuild/rules_k8s/pull/423
+	pattern, _ := regexp.Compile("^" + t + "(@sha256)?(:[a-zA-Z0-9_.{}-]*)?$")
+	return pattern.MatchString(s)
+}
+
+// Split separates and returns the name and tag parts
+// from the image string using either colon `:` or at `@` separators.
+// Note that the returned tag keeps its separator.
+func Split(imageName string) (name string, tag string) {
+	// check if image name contains a domain
+	// if domain is present, ignore domain and check for `:`
+	ic := -1
+	if slashIndex := strings.Index(imageName, "/"); slashIndex < 0 {
+		ic = strings.LastIndex(imageName, ":")
+	} else {
+		lastIc := strings.LastIndex(imageName[slashIndex:], ":")
+		// set ic only if `:` is present
+		if lastIc > 0 {
+			ic = slashIndex + lastIc
+		}
+	}
+	ia := strings.LastIndex(imageName, "@")
+	if ic < 0 && ia < 0 {
+		return imageName, ""
+	}
+
+	i := ic
+	if ia > 0 {
+		i = ia
+	}
+
+	name = imageName[:i]
+	tag = imageName[i:]
+	return
+}

api/image/image_test.go

@@ -0,0 +1,80 @@
+// Copyright 2020 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package image
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestIsImageMatched(t *testing.T) {
+	testCases := []struct {
+		testName  string
+		value     string
+		name      string
+		isMatched bool
+	}{
+		{
+			testName:  "identical",
+			value:     "nginx",
+			name:      "nginx",
+			isMatched: true,
+		},
+		{
+			testName:  "name is match",
+			value:     "nginx:12345",
+			name:      "nginx",
+			isMatched: true,
+		},
+		{
+			testName:  "name is not a match",
+			value:     "apache:12345",
+			name:      "nginx",
+			isMatched: false,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.testName, func(t *testing.T) {
+			assert.Equal(t, tc.isMatched, IsImageMatched(tc.value, tc.name))
+		})
+	}
+}
+
+func TestSplit(t *testing.T) {
+	testCases := []struct {
+		testName string
+		value    string
+		name     string
+		tag      string
+	}{
+		{
+			testName: "no tag",
+			value:    "nginx",
+			name:     "nginx",
+			tag:      "",
+		},
+		{
+			testName: "with tag",
+			value:    "nginx:1.2.3",
+			name:     "nginx",
+			tag:      ":1.2.3",
+		},
+		{
+			testName: "with digest",
+			value:    "nginx@12345",
+			name:     "nginx",
+			tag:      "@12345",
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.testName, func(t *testing.T) {
+			name, tag := Split(tc.value)
+			assert.Equal(t, tc.name, name)
+			assert.Equal(t, tc.tag, tag)
+		})
+	}
+}

api/internal/accumulator/refvartransformer.go

@@ -40,7 +40,11 @@ func (rv *refVarTransformer) replaceVars(in interface{}) (interface{}, error) {
 	case []interface{}:
 		var xs []interface{}
 		for _, a := range in.([]interface{}) {
-			xs = append(xs, expansion2.Expand(a.(string), rv.mappingFunc))
+			x, ok := a.(string)
+			if !ok {
+				return nil, fmt.Errorf("expected array of strings, found %v", in)
+			}
+			xs = append(xs, expansion2.Expand(x, rv.mappingFunc))
 		}
 		return xs, nil
 	case map[string]interface{}:
@@ -49,7 +53,7 @@ func (rv *refVarTransformer) replaceVars(in interface{}) (interface{}, error) {
 		for k, v := range inMap {
 			s, ok := v.(string)
 			if !ok {
-				// This field not contain a $(VAR) since it is not
+				// This field can not contain a $(VAR) since it is not
 				// of string type. For instance .spec.replicas: 3 in
 				// a Deployment object
 				xs[k] = v
@@ -64,7 +68,7 @@ func (rv *refVarTransformer) replaceVars(in interface{}) (interface{}, error) {
 	case interface{}:
 		s, ok := in.(string)
 		if !ok {
-			// This field not contain a $(VAR) since it is not of string type.
+			// This field can not contain a $(VAR) since it is not of string type.
 			return in, nil
 		}
 		// This field can potentially contain a $(VAR) since it is

api/internal/accumulator/refvartransformer_test.go

@@ -29,6 +29,7 @@ func TestRefVarTransformer(t *testing.T) {
 		description string
 		given       given
 		expected    expected
+		errMessage  string
 	}{
 		{
 			description: "var replacement in map[string]",
@@ -111,6 +112,27 @@ func TestRefVarTransformer(t *testing.T) {
 				unused: []string{"BAR"},
 			},
 		},
+		{
+			description: "var replacement panic in map[string]",
+			given: given{
+				varMap: map[string]interface{}{},
+				fs: []types.FieldSpec{
+					{Gvk: resid.Gvk{Version: "v1", Kind: "ConfigMap"}, Path: "data/slice"},
+				},
+				res: resmaptest_test.NewRmBuilder(
+					t, resource.NewFactory(kunstruct.NewKunstructuredFactoryImpl())).
+					Add(map[string]interface{}{
+						"apiVersion": "v1",
+						"kind":       "ConfigMap",
+						"metadata": map[string]interface{}{
+							"name": "cm1",
+						},
+						"data": map[string]interface{}{
+							"slice": []interface{}{5}, // noticeably *not* a []string
+						}}).ResMap(),
+			},
+			errMessage: "expected array of strings, found [5]",
+		},
 	}
 
 	for _, tc := range testCases {
@@ -122,16 +144,23 @@ func TestRefVarTransformer(t *testing.T) {
 			err := tr.Transform(tc.given.res)
 
 			// assert
-			if err != nil {
-				t.Errorf("unexpected error: %v", err)
-			}
+			if tc.errMessage != "" {
+				if err == nil {
+					t.Fatalf("missing expected error %v", tc.errMessage)
+				} else if err.Error() != tc.errMessage {
+					t.Fatalf("actual error doesn't match expected error: \nACTUAL: %v\nEXPECTED: %v", err.Error(), tc.errMessage)
+				}
+			} else {
+				if err != nil {
+					t.Errorf("unexpected error: %v", err)
+				}
 
-			a, e := tc.given.res, tc.expected.res
-			if !reflect.DeepEqual(a, e) {
-				err = e.ErrorIfNotEqualLists(a)
-				t.Fatalf("actual doesn't match expected: \nACTUAL:\n%v\nEXPECTED:\n%v\nERR: %v", a, e, err)
+				a, e := tc.given.res, tc.expected.res
+				if !reflect.DeepEqual(a, e) {
+					err = e.ErrorIfNotEqualLists(a)
+					t.Fatalf("actual doesn't match expected: \nACTUAL:\n%v\nEXPECTED:\n%v\nERR: %v", a, e, err)
+				}
 			}
-
 		})
 	}
 }

api/internal/crawl/README.md

@@ -209,7 +209,7 @@ but they are functional.
 
 ## Technical details
 
-### Overall design and imlpementation
+### Overall design and implementations
 
 There are a few components that are all running together in order to get
 the overall application to work smoothly. This section will provide a brief

api/internal/crawl/backend/search_backend.go

@@ -44,7 +44,7 @@ type kustomizeSearch struct {
 // /register: not implemented, but meant as an endpoint for adding new
 // kustomization files to the corpus.
 func NewKustomizeSearch(ctx context.Context) (*kustomizeSearch, error) {
-	idx, err := index.NewKustomizeIndex(ctx)
+	idx, err := index.NewKustomizeIndex(ctx, "kustomize")
 	if err != nil {
 		return nil, err
 	}

api/internal/crawl/cmd/crawler/crawler.go

@@ -2,11 +2,15 @@ package main
 
 import (
 	"context"
+	"flag"
 	"fmt"
+	"log"
 	"net/http"
 	"os"
 	"time"
 
+	"sigs.k8s.io/kustomize/api/internal/crawl/utils"
+
 	"sigs.k8s.io/kustomize/api/internal/crawl/crawler"
 	"sigs.k8s.io/kustomize/api/internal/crawl/crawler/github"
 	"sigs.k8s.io/kustomize/api/internal/crawl/doc"
@@ -23,71 +27,189 @@ const (
 	retryCount           = 3
 )
 
+type CrawlMode int
+
+const (
+	CrawlUnknown CrawlMode = iota
+	// Crawl all the kustomization files in all the repositories of a Github user
+	CrawlUser
+	// Crawl all the kustomization files in a Github repo
+	CrawlRepo
+	// Crawl all the documents in the index
+	CrawlIndex
+	// Crawl all the kustomization files on Github
+	CrawlGithub
+	// Crawl all the documents in the index and crawling all the kustomization files on Github
+	CrawlIndexAndGithub
+)
+
+func NewCrawlMode(s string) CrawlMode {
+	switch s {
+	case "github-user":
+		return CrawlUser
+	case "github-repo":
+		return CrawlRepo
+	case "index+github":
+		return CrawlIndexAndGithub
+	case "index":
+		return CrawlIndex
+	case "github":
+		return CrawlGithub
+	default:
+		return CrawlUnknown
+	}
+}
+
 func main() {
+	indexNamePtr := flag.String(
+		"index", "kustomize", "The name of the ElasticSearch index.")
+	modePtr := flag.String("mode", "index+github",
+		`The crawling mode, which can be one of [github-user, github-repo, index, github, index+github].
+  * github-user: crawl all the kustomization files in all the repositories of a Github user (--github-user must be specified for this mode).
+  * github-repo: crawl all the kustomization files in a Github repository (--github-repo must be specified for this mode).
+  * index: crawl all the documents in the index.
+  * gihub: crawl all the kustomization files on Github.
+  * index+github: crawl all the documents in the index and crawling all the kustomization files on Github.`)
+	githubUserPtr := flag.String("github-user", "",
+		"A github user name (e.g., kubernetes-sigs). This flag is required for the `github-user` mode.")
+	githubRepoPtr := flag.String("github-repo", "",
+		"A github repository name (e.g., kubernetes-sigs/kustomize). This flag is required for the `github-repo` mode.")
+	flag.Parse()
+
 	githubToken := os.Getenv(githubAccessTokenVar)
 	if githubToken == "" {
-		fmt.Printf("Must set the variable '%s' to make github requests.\n",
+		log.Printf("Must set the variable '%s' to make github requests.\n",
 			githubAccessTokenVar)
 		return
 	}
 
 	ctx := context.Background()
-	idx, err := index.NewKustomizeIndex(ctx)
+	idx, err := index.NewKustomizeIndex(ctx, *indexNamePtr)
 	if err != nil {
-		fmt.Printf("Could not create an index: %v\n", err)
+		log.Printf("Could not create an index: %v\n", err)
 		return
 	}
 
 	cacheURL := os.Getenv(redisCacheURL)
-
-	query := []byte(`{ "query":{ "match_all":{} } }`)
-	it := idx.IterateQuery(query, 10000, 60*time.Second)
-	docs := make(crawler.CrawlSeed, 0)
-	for it.Next() {
-		for _, hit := range it.Value().Hits.Hits {
-			docs = append(docs, hit.Document.Copy())
-		}
-	}
-
-	if err := it.Err(); err != nil {
-		fmt.Printf("Error iterating: %v\n", err)
-	}
-
 	cache, err := redis.DialURL(cacheURL)
 	clientCache := &http.Client{}
 	if err != nil {
-		fmt.Printf("Error: redis could not make a connection: %v\n", err)
+		log.Printf("Error: redis could not make a connection: %v\n", err)
 	} else {
 		clientCache = httpclient.NewClient(cache)
 	}
 
-	ghCrawler := github.NewCrawler(githubToken, retryCount, clientCache,
-		github.QueryWith(
-			github.Filename("kustomization.yaml"),
-			github.Filename("kustomization.yml")),
-	)
+	// docConverter takes in a plain document and processes it for the index.
+	docConverter := func(d *doc.Document) (crawler.CrawledDocument, error) {
+		kdoc := doc.KustomizationDocument{
+			Document: *d,
+		}
 
-	crawler.CrawlFromSeed(ctx, docs, []crawler.Crawler{ghCrawler},
-		// Converter takes in a plain document and processes it for the
-		// index.
-		func(d *doc.Document) (crawler.CrawledDocument, error) {
-			kdoc := doc.KustomizationDocument{
-				Document: *d,
-			}
+		err := kdoc.ParseYAML()
+		return &kdoc, err
+	}
 
-			err := kdoc.ParseYAML()
-			return &kdoc, err
-		},
-		// IndexFunc updates the value in the index.
-		func(cdoc crawler.CrawledDocument, crwlr crawler.Crawler) error {
-			switch d := cdoc.(type) {
-			case *doc.KustomizationDocument:
-				fmt.Println("Inserting: ", d.ID(), d)
-				_, err := idx.Put(d.ID(), d)
-				return err
+	// Index updates the value in the index.
+	indexFunc := func(cdoc crawler.CrawledDocument, mode index.Mode) error {
+		switch d := cdoc.(type) {
+		case *doc.KustomizationDocument:
+			switch mode {
+			case index.Delete:
+				log.Printf("Deleting: %v", d)
+				return idx.Delete(d.ID())
 			default:
-				return fmt.Errorf("type %T not supported", d)
+				log.Printf("Inserting: %v", d)
+				return idx.Put(d.ID(), d)
 			}
-		},
-	)
+		default:
+			return fmt.Errorf("type %T not supported", d)
+		}
+	}
+
+	// seen tracks the IDs of all the documents in the index and their corresponding file types.
+	// This helps avoid indexing a given document multiple times.
+	seen := utils.NewSeenMap()
+
+	mode := NewCrawlMode(*modePtr)
+
+	ghCrawlerConstructor := func(user, repo string) crawler.Crawler {
+		if user != "" {
+			return github.NewCrawler(githubToken, retryCount, clientCache,
+				github.QueryWith(
+					github.Filename("kustomization.yaml"),
+					github.Filename("kustomization.yml"),
+					github.Filename("kustomization"),
+					github.User(user)),
+			)
+		} else if repo != "" {
+			return github.NewCrawler(githubToken, retryCount, clientCache,
+				github.QueryWith(
+					github.Filename("kustomization.yaml"),
+					github.Filename("kustomization.yml"),
+					github.Filename("kustomization"),
+					github.Repo(repo)),
+			)
+		} else {
+			return github.NewCrawler(githubToken, retryCount, clientCache,
+				github.QueryWith(
+					github.Filename("kustomization.yaml"),
+					github.Filename("kustomization.yml"),
+					github.Filename("kustomization")),
+			)
+		}
+	}
+
+	seedDocs := make(crawler.CrawlSeed, 0)
+
+	// get all the documents in the index
+	getSeedDocsFunc := func() {
+		query := []byte(`{ "query":{ "match_all":{} } }`)
+		it := idx.IterateQuery(query, 10000, 60*time.Second)
+		for it.Next() {
+			for _, hit := range it.Value().Hits.Hits {
+				seedDocs = append(seedDocs, hit.Document.Document.Copy())
+			}
+		}
+		if err := it.Err(); err != nil {
+			log.Fatalf("getSeedDocsFunc Error iterating: %v\n", err)
+		}
+	}
+
+	switch mode {
+	case CrawlIndexAndGithub:
+		getSeedDocsFunc()
+		crawlers := []crawler.Crawler{ghCrawlerConstructor("", "")}
+		crawler.CrawlFromSeed(ctx, seedDocs, crawlers, docConverter, indexFunc, seen)
+		crawler.CrawlGithub(ctx, crawlers, docConverter, indexFunc, seen)
+	case CrawlIndex:
+		getSeedDocsFunc()
+		crawlers := []crawler.Crawler{ghCrawlerConstructor("", "")}
+		crawler.CrawlFromSeed(ctx, seedDocs, crawlers, docConverter, indexFunc, seen)
+	case CrawlGithub:
+		crawlers := []crawler.Crawler{ghCrawlerConstructor("", "")}
+		// add all the documents in the index into seen.
+		// this greatly reduces the time overhead of CrawlGithub.
+		getSeedDocsFunc()
+		for _, d := range seedDocs {
+			seen[d.ID()] = d.FileType
+		}
+		crawler.CrawlGithub(ctx, crawlers, docConverter, indexFunc, seen)
+	case CrawlUser:
+		if *githubUserPtr == "" {
+			flag.Usage()
+			log.Fatalf("Please specify a github user with the github-user flag!")
+		}
+		crawlers := []crawler.Crawler{ghCrawlerConstructor(*githubUserPtr, "")}
+		crawler.CrawlGithub(ctx, crawlers, docConverter, indexFunc, seen)
+	case CrawlRepo:
+		if *githubRepoPtr == "" {
+			flag.Usage()
+			log.Fatalf("Please specify a github repository with the github-repo flag!")
+		}
+		crawlers := []crawler.Crawler{ghCrawlerConstructor("", *githubRepoPtr)}
+		crawler.CrawlGithub(ctx, crawlers, docConverter, indexFunc, seen)
+	case CrawlUnknown:
+		flag.Usage()
+		log.Fatalf("The --mode flag must be one of [github-user, github-repo, index, github, index+github].")
+	}
 }

api/internal/crawl/cmd/kustomize_stats/Dockerfile

@@ -0,0 +1,14 @@
+FROM golang:1.11 AS build
+
+ARG GO111MODULE=on
+
+WORKDIR /go/src/sigs.k8s.io/kustomize/api/internal/crawl
+COPY . /go/src/sigs.k8s.io/kustomize/api/internal/crawl
+
+RUN go mod download
+RUN CGO_ENABLED=0 go install ./cmd/kustomize_stats
+
+FROM scratch
+COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
+COPY --from=build /go/bin/kustomize_stats /
+ENTRYPOINT ["/kustomize_stats"]

api/internal/crawl/cmd/kustomize_stats/main.go

@@ -0,0 +1,249 @@
+package main
+
+import (
+	"context"
+	"crypto/sha256"
+	"flag"
+	"fmt"
+	"log"
+	"sort"
+	"time"
+
+	"sigs.k8s.io/kustomize/api/internal/crawl/doc"
+
+	"sigs.k8s.io/kustomize/api/internal/crawl/index"
+)
+
+// iterateArr adds each item in arr into countMap.
+func iterateArr(arr []string, countMap map[string]int) {
+	for _, item := range arr {
+		if _, ok := countMap[item]; !ok {
+			countMap[item] = 1
+		} else {
+			countMap[item]++
+		}
+	}
+
+}
+
+// SortMapKeyByValueInt takes a map as its input, sorts its keys according to their values
+// in the map, and outputs the sorted keys as a slice.
+func SortMapKeyByValueInt(m map[string]int) []string {
+	keys := make([]string, 0, len(m))
+	for key := range m {
+		keys = append(keys, key)
+	}
+	// sort keys according to their values in the map m
+	sort.Slice(keys, func(i, j int) bool { return m[keys[i]] > m[keys[j]] })
+	return keys
+}
+
+// SortMapKeyByValue takes a map as its input, sorts its keys according to their values
+// in the map, and outputs the sorted keys as a slice.
+func SortMapKeyByValueLen(m map[string][]string) []string {
+	keys := make([]string, 0, len(m))
+	for key := range m {
+		keys = append(keys, key)
+	}
+	// sort keys according to their values in the map m
+	sort.Slice(keys, func(i, j int) bool { return len(m[keys[i]]) > len(m[keys[j]]) })
+	return keys
+}
+
+func GeneratorOrTransformerStats(docs []*doc.KustomizationDocument) {
+	n := len(docs)
+	if n == 0 {
+		return
+	}
+
+	fileType := docs[0].FileType
+	fmt.Printf("There are totally %d %s files.\n", n, fileType)
+
+	GitRepositorySummary(docs, fileType)
+
+	// key of kindToUrls: a string in the KustomizationDocument.Kinds field
+	// value of kindToUrls: a slice of string urls defining a given kind.
+	kindToUrls := make(map[string][]string)
+
+	for _, d := range docs {
+		url := fmt.Sprintf("%s/blob/%s/%s", d.RepositoryURL, d.DefaultBranch, d.FilePath)
+		for _, kind := range d.Kinds {
+			if _, ok := kindToUrls[kind]; !ok {
+				kindToUrls[kind] = []string{url}
+			} else {
+				kindToUrls[kind] = append(kindToUrls[kind], url)
+			}
+		}
+	}
+	fmt.Printf("There are totally %d kinds of %s\n", len(kindToUrls), fileType)
+	sortedKeys := SortMapKeyByValueLen(kindToUrls)
+	for _, k := range sortedKeys {
+		sort.Strings(kindToUrls[k])
+		fmt.Printf("%s kind %s appears %d times\n", fileType, k, len(kindToUrls[k]))
+		for _, url := range kindToUrls[k] {
+			fmt.Printf("%s\n", url)
+		}
+	}
+}
+
+// GitRepositorySummary counts the distribution of docs:
+// 1) how many git repositories are these docs from?
+// 2) how many docs are from each git repository?
+func GitRepositorySummary(docs []*doc.KustomizationDocument, fileType string) {
+	m := make(map[string]int)
+	for _, d := range docs {
+		if _, ok := m[d.RepositoryURL]; ok {
+			m[d.RepositoryURL]++
+		} else {
+			m[d.RepositoryURL] = 1
+		}
+	}
+	sortedKeys := SortMapKeyByValueInt(m)
+	topN := 10
+	i := 0
+	for _, k := range sortedKeys {
+		if i >= topN {
+			break
+		}
+		fmt.Printf("%d %s are from %s\n", m[k], fileType, k)
+		i++
+	}
+}
+
+func main() {
+	topKindsPtr := flag.Int(
+		"kinds", -1,
+		`the number of kubernetes object kinds to be listed according to their popularities.
+By default, all the kinds will be listed.
+If you only want to list the 10 most popular kinds, set the flag to 10.`)
+	topIdentifiersPtr := flag.Int(
+		"identifiers", -1,
+		`the number of identifiers to be listed according to their popularities.
+By default, all the identifiers will be listed.
+If you only want to list the 10 most popular identifiers, set the flag to 10.`)
+	topKustomizeFeaturesPtr := flag.Int(
+		"kustomize-features", -1,
+		`the number of kustomize features to be listed according to their popularities.
+By default, all the features will be listed.
+If you only want to list the 10 most popular features, set the flag to 10.`)
+	indexNamePtr := flag.String(
+		"index", "kustomize", "The name of the ElasticSearch index.")
+	flag.Parse()
+
+	ctx := context.Background()
+	idx, err := index.NewKustomizeIndex(ctx, *indexNamePtr)
+	if err != nil {
+		log.Fatalf("Could not create an index: %v\n", err)
+	}
+
+	// count tracks the number of documents in the index
+	count := 0
+
+	// kustomizationFilecount tracks the number of kustomization files in the index
+	kustomizationFilecount := 0
+
+	kindsMap := make(map[string]int)
+	identifiersMap := make(map[string]int)
+	kustomizeIdentifiersMap := make(map[string]int)
+
+	// ids tracks the unique IDs of the documents in the index
+	ids := make(map[string]struct{})
+
+	// generatorFiles include all the non-kustomization files whose FileType is generator
+	generatorFiles := make([]*doc.KustomizationDocument, 0)
+
+	// transformersFiles include all the non-kustomization files whose FileType is transformer
+	transformersFiles := make([]*doc.KustomizationDocument, 0)
+
+	checksums := make(map[string]int)
+
+	// get all the documents in the index
+	query := []byte(`{ "query":{ "match_all":{} } }`)
+	it := idx.IterateQuery(query, 10000, 60*time.Second)
+	for it.Next() {
+		for _, hit := range it.Value().Hits.Hits {
+			sum := fmt.Sprintf("%x", sha256.Sum256([]byte(hit.Document.DocumentData)))
+			if _, ok := checksums[sum]; ok {
+				checksums[sum]++
+			} else {
+				checksums[sum] = 1
+			}
+
+			// check whether there is any duplicate IDs in the index
+			if _, ok := ids[hit.ID]; !ok {
+				ids[hit.ID] = struct{}{}
+			} else {
+				log.Printf("Found duplicate ID (%s)\n", hit.ID)
+			}
+
+			count++
+			iterateArr(hit.Document.Kinds, kindsMap)
+			iterateArr(hit.Document.Identifiers, identifiersMap)
+
+			if doc.IsKustomizationFile(hit.Document.FilePath) {
+				kustomizationFilecount++
+				iterateArr(hit.Document.Identifiers, kustomizeIdentifiersMap)
+
+			} else {
+				switch hit.Document.FileType {
+				case "generator":
+					generatorFiles = append(generatorFiles, hit.Document.Copy())
+				case "transformer":
+					transformersFiles = append(transformersFiles, hit.Document.Copy())
+				}
+			}
+		}
+	}
+
+	if err := it.Err(); err != nil {
+		log.Fatalf("Error iterating: %v\n", err)
+	}
+
+	sortedKindsMapKeys := SortMapKeyByValueInt(kindsMap)
+	sortedIdentifiersMapKeys := SortMapKeyByValueInt(identifiersMap)
+	sortedKustomizeIdentifiersMapKeys := SortMapKeyByValueInt(kustomizeIdentifiersMap)
+
+	fmt.Printf(`The count of unique document IDs in the kustomize index: %d
+There are %d documents in the kustomize index.
+%d kinds of kubernetes objects are customized:`, len(ids), count, len(kindsMap))
+	fmt.Printf("\n")
+
+	kindCount := 0
+	for _, key := range sortedKindsMapKeys {
+		if *topKindsPtr < 0 || (*topKindsPtr >= 0 && kindCount < *topKindsPtr) {
+			fmt.Printf("\tkind `%s` is customimzed in %d documents\n", key, kindsMap[key])
+			kindCount++
+		}
+	}
+
+	fmt.Printf("%d kinds of identifiers are found:\n", len(identifiersMap))
+	identifierCount := 0
+	for _, key := range sortedIdentifiersMapKeys {
+		if *topIdentifiersPtr < 0 || (*topIdentifiersPtr >= 0 && identifierCount < *topIdentifiersPtr) {
+			fmt.Printf("\tidentifier `%s` appears in %d documents\n", key, identifiersMap[key])
+			identifierCount++
+		}
+	}
+
+	fmt.Printf(`There are %d kustomization files in the kustomize index.
+%d kinds of kustomize features are found:`, kustomizationFilecount, len(kustomizeIdentifiersMap))
+	fmt.Printf("\n")
+	kustomizeFeatureCount := 0
+	for _, key := range sortedKustomizeIdentifiersMapKeys {
+		if *topKustomizeFeaturesPtr < 0 || (*topKustomizeFeaturesPtr >= 0 && kustomizeFeatureCount < *topKustomizeFeaturesPtr) {
+			fmt.Printf("\tfeature `%s` is used in %d documents\n", key, kustomizeIdentifiersMap[key])
+			kustomizeFeatureCount++
+		}
+	}
+
+	GeneratorOrTransformerStats(generatorFiles)
+	GeneratorOrTransformerStats(transformersFiles)
+
+	fmt.Printf("There are total %d checksums of document contents\n", len(checksums))
+	sortedChecksums := SortMapKeyByValueInt(checksums)
+	sortedChecksums = sortedChecksums[:20]
+	fmt.Printf("The top 20 checksums are:\n")
+	for _, key := range sortedChecksums {
+		fmt.Printf("checksum %s apprears %d\n", key, checksums[key])
+	}
+}

api/internal/crawl/cmd/log-parser/README.md

@@ -0,0 +1,8 @@
+This binary takes as its input a json file including GKE logs (which can be
+[exported](https://cloud.google.com/logging/docs/export/configure_export_v2) into 
+[Cloud Storage](https://cloud.google.com/storage/docs/)), 
+and extracts the `textPayload` field of each log entry.
+
+Here is an log entry example:
+
+{"insertId":"1sxuh4jg5lw6w10","labels":{"compute.googleapis.com/resource_name":"gke-crawler2-default-pool-5e55ea05-gzgv","container.googleapis.com/namespace_name":"default","container.googleapis.com/pod_name":"kustomize-stats-5bczg","container.googleapis.com/stream":"stdout"},"logName":"projects/haiyanmeng-gke-dev/logs/kustomize-stats","receiveTimestamp":"2020-01-06T23:33:07.012831742Z","resource":{"labels":{"cluster_name":"crawler2","container_name":"kustomize-stats","instance_id":"8183086081854184383","namespace_id":"default","pod_id":"kustomize-stats-5bczg","project_id":"haiyanmeng-gke-dev","zone":"us-central1-a"},"type":"container"},"severity":"INFO","textPayload":"The kustomize index already exists\n","timestamp":"2020-01-06T23:32:46.628930547Z"}

api/internal/crawl/cmd/log-parser/kustomize-stats-cmd

@@ -0,0 +1,7 @@
+wget <log-file-url> -O log
+go build .
+./log-parser log >out
+cat out | grep "kind \`" | cut -d\` -f2 | tail -n 50
+cat out | grep "kind \`" | awk '{print $6}' | tail -n 50
+cat out | grep "feature \`" | grep -v "\`resources\`" | grep -v -e "\`apiVersion\`" | grep -v -e "\`apiversion\`" | cut -d\` -f2
+cat out | grep "feature \`" | grep -v "\`resources\`" | grep -v -e "\`apiVersion\`" | grep -v -e "\`apiversion\`" | awk '{print $6}'

api/internal/crawl/cmd/log-parser/main.go

@@ -0,0 +1,49 @@
+package main
+
+import (
+	"bufio"
+	"encoding/json"
+	"fmt"
+	"log"
+	"os"
+)
+
+func main() {
+	if len(os.Args) != 2 {
+		log.Fatalf("The usage of the command is: \n\t%s <log-file.json>", os.Args[0])
+	}
+
+	file, err := os.Open(os.Args[1])
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	closeFile := func(file *os.File) {
+		if err := file.Close(); err != nil {
+			log.Fatal(err)
+		}
+	}
+	defer closeFile(file)
+
+	scanner := bufio.NewScanner(file)
+	for scanner.Scan() {
+		line := scanner.Text()
+
+		var entry interface{}
+		if err := json.Unmarshal([]byte(line), &entry); err != nil {
+			log.Printf("failed to unmarshal a log entry: %s\n", line)
+		}
+
+		m := entry.(map[string]interface{})
+		if payload, ok := m["textPayload"]; ok {
+			// use fmt.Printf here instead of log.Printf to avoid the time and code location info the log package provides
+			fmt.Printf("%s", payload)
+		} else {
+			log.Printf("the log entry does not have the `textPayload` field: %s\n", line)
+		}
+	}
+
+	if err := scanner.Err(); err != nil {
+		log.Fatal(err)
+	}
+}

api/internal/crawl/config/base/kustomization.yaml

@@ -2,5 +2,4 @@ configmapGenerator:
 - name: elasticsearch-config
   literals:
   - es-url="http://esbasic-master:9200"
-  - kustomize-index-name="kustomize"
   - plugin-index-name="plugin"

api/internal/crawl/config/crawler/base/.gitignore

@@ -0,0 +1 @@
+github_api_secret.txt

api/internal/crawl/config/crawler/cronjob/cronjob.yaml

@@ -1,9 +1,10 @@
 apiVersion: batch/v1beta1
 kind: CronJob
 metadata:
-  name: crawler
+  name: crawler-cronjob
 spec:
-  schedule: "5 0 * * */1"
+  # run the cronjob at 00:00 every 7 days
+  schedule: "0 0 */7 * *"
   jobTemplate:
     spec:
       template:
@@ -11,7 +12,9 @@ spec:
           restartPolicy: OnFailure
           containers:
           - name: crawler
-            image: gcr.io/kustomize-search/crawler:latest
+            image: gcr.io/haiyanmeng-gke-dev/crawler:v1
+            command: ["/crawler"]
+            args: ["--mode=index+github", "--github-repo=kubernetes-sigs/kustomize", "--index=kustomize"]
             imagePullPolicy: Always
             env:
             - name: GITHUB_ACCESS_TOKEN

api/internal/crawl/config/crawler/job/README.md

@@ -0,0 +1,53 @@
+The crawler job can run in one of the following mode:
+
+# Crawling all the documents in the index and crawling all the kustomization files on Github
+
+This is the default setting of the crawler job. The `command` and `args` field
+of the container should be:
+
+```
+        command: ["/crawler"]
+```
+
+Or
+
+```
+        command: ["/crawler"]
+        args: ["--mode=index+github"]
+```
+
+# Crawling all the documents in the index
+
+The `command` and `args` field of the container should be:
+
+```
+        command: ["/crawler"]
+        args: ["--mode=index"]
+```
+
+# Crawling all the kustomization files on Github
+
+The `command` and `args` field of the container should be:
+
+```
+        command: ["/crawler"]
+        args: ["--mode=github"]
+```
+
+# Crawling all the kustomization files in a Github repo
+
+The `command` and `args` field of the container should be like:
+
+```
+        command: ["/crawler"]
+        args: ["--mode=github-repo", "--github-repo=kubernetes-sigs/kustomize"]
+```
+
+# Crawling all the kustomization files in all the repositories of a Github user
+
+The `command` and `args` field of the container should be like:
+
+```
+        command: ["/crawler"]
+        args: ["--github-user", "--github-user=kubernetes-sigs"]
+```

api/internal/crawl/config/crawler/job/job.yaml

@@ -8,8 +8,10 @@ spec:
       restartPolicy: OnFailure
       containers:
       - name: crawler
-        image: gcr.io/kustomize-search/crawler:latest
+        image: gcr.io/haiyanmeng-gke-dev/crawler:v1
         imagePullPolicy: Always
+        command: ["/crawler"]
+        args: ["--mode=github-repo", "--github-repo=kubernetes-sigs/kustomize", "--index=kustomize"]
         env:
         - name: GITHUB_ACCESS_TOKEN
           valueFrom:

api/internal/crawl/config/crawler/kustomize_stats/job.yaml

@@ -0,0 +1,20 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: kustomize-stats
+spec:
+  template:
+    spec:
+      restartPolicy: OnFailure
+      containers:
+      - name: kustomize-stats
+        image: gcr.io/haiyanmeng-gke-dev/kustomize_stats:v1
+        imagePullPolicy: Always
+        command: ["/kustomize_stats"]
+        args: ["--index=kustomize", "--kinds=51", "--identifiers=50", "--kustomize-features=50"]
+        env:
+        - name: ELASTICSEARCH_URL
+          valueFrom:
+            configMapKeyRef:
+              name: elasticsearch-config
+              key: es-url

api/internal/crawl/config/crawler/kustomize_stats/kustomization.yaml

@@ -0,0 +1,3 @@
+resources:
+- ../base
+- job.yaml

api/internal/crawl/config/elastic/esbackup.yaml

@@ -0,0 +1,23 @@
+# ESBackup depends on ESCluster, and is depended by ESSnapshot.
+# Creating `esbackup/kustomize-backbup` will create the `kustomize-backup` snapshot repository.
+# Deleting `esbackup/kustomize-backbup` will delete the `kustomize-backup` snapshot repository.
+# Deleting `esbackup/kustomize-backbup` will NOT delete the snapshots in the `kustomize-backup` snapshot repository, instead makes all the snapshots in the repository inaccessible.
+# Deleting `esbackup/kustomize-backbup` will NOT delete the essnapshot objects depending on it, but will cause those essnapshot objects to be reconciled, which update the status of the essnapshot objects to reflect the fact that the esbackup object is missing.
+# If you delete the `kustomize-backup` snapshot repository directly without deleting `esbackup/kustomize-backbup`, the ESBackup object will not recreate the snapshot repository.
+apiVersion: elasticsearch.cloud.google.com/v1alpha1
+kind: ESBackup
+metadata:
+  name: kustomize-backup
+spec:
+  storage:
+    gcs:
+      # the bucket must exist for the snapshot repository to be created successfully.
+      bucket: kustomize-backup
+      # the path does not need to exist.
+      # If the path does not exist, the controller will create the folder in the GCS bucket.
+      # If the path already exists and includes snapshots, these snapshots can be used.
+      path: kustomize
+      secret:
+        name: kustomizesa
+  escluster:
+    name: esbasic

api/internal/crawl/config/elastic/escluster.yaml

@@ -1,3 +1,4 @@
+# ESCluster is depended by ESBackup and ESRestore.
 apiVersion: elasticsearch.cloud.google.com/v1alpha1
 kind: ESCluster
 metadata:
@@ -8,6 +9,13 @@ spec:
     - repository-gcs
     - ingest-user-agent
     - ingest-geoip
+  # To set `gcpserviceaccount`,
+  # First, create and download a GCP service account into a json file, named `sakey.json` following the instruction:
+  #     https://www.elastic.co/guide/en/elasticsearch/plugins/6.5/repository-gcs-usage.html#repository-gcs-using-service-account
+  # Second, create a secret for the service account using the following command:
+  #     $ kubectl create secret generic kustomizesa --from-file=./sakey.json
+  gcpserviceaccount:
+    name: kustomizesa
   config:
     env:
       example: test

api/internal/crawl/config/elastic/esrestore.yaml

@@ -0,0 +1,19 @@
+# ESRestore depends on both ESCluster and ESSnapshot.
+# Creating `esrestore/kustomize-restore` will restore the `kuostmize` index in the `kustomize-snapshot` snapshot to a new index named `kusotmize-restore`.
+# Deleting `esrestore/kustomize-restore` will not delete the restored index.
+# Deleting `esrestore/kustomize-restore` should happen before deleting `essnapshot/kustomize-snapshot`.
+# After the restore is complete, if the `kusotmize-restore` index is deleted manually, the ESRestore object will NOT restore the `kustomize` index to it again.
+# The correct way of using ESRestore is: create a ESRestore object to restore the index; delete the ESRestore object after the restore is complete.
+apiVersion: elasticsearch.cloud.google.com/v1alpha1
+kind: ESRestore
+metadata:
+  name: kustomize-restore
+spec:
+  include_global_state: true
+  ignore_unavailable: true
+  rename_pattern: kustomize
+  rename_replacement: kustomize-restore
+  essnapshot:
+    name: kustomize-snapshot
+  escluster:
+    name: esbasic

api/internal/crawl/config/elastic/essnapshot.yaml

@@ -0,0 +1,23 @@
+# ESSnapshot depends on ESBackup, and is depended by ESRestore.
+# Creating `essnapshot/kustomize-snapshot` will create a snapshot named `kustomize-snapshot` in the `kustomize-backup` snapshot repository.
+# After being created, the `kustomize-snapshot` snapshot will not be automatically updated when the `kuostomize` index is updated.
+# If you delete `essnapshot/kustomize-snapshot` and recreate it, the new snapshot will capture the current status of the `kustomize` index.
+# Deleting `essnapshot/kustomize-snapshot` will delete the snapshot.
+# Deleting `essnapshot/kustomize-snapshot` should happen before deleting `esbackup/kustomize-backup`.
+# If the `kustomize-snapshot` snapshot is deleted directly without deleting `essnapshot/kustomize-snapshot`, the ESSnapshot object will recreate the snapshot.
+# The correct way of using ESSnapshot is: create an ESSnapshot object to create a snapshot, keep the ESSnapshot object until the snapshot is no longer needed.
+# To update the snapshot to capture the latest version of the index, you can either:
+#   1) delete the snapshot, and wait for the ESSnapshot object to recreate the snapshot;
+#   2) delete the ESSnapshot object, and recreate the ESSnapshot object.
+apiVersion: elasticsearch.cloud.google.com/v1alpha1
+kind: ESSnapshot
+metadata:
+  name: kustomize-snapshot
+spec:
+  # indices are optional. If not specified all indices are selected.
+  indices:
+  - kustomize
+  include_global_state: true
+  ignore_unavailable: true
+  esbackup:
+    name: kustomize-backup

api/internal/crawl/crawler/crawler.go

@@ -10,6 +10,10 @@ import (
 	"os"
 	"sync"
 
+	"sigs.k8s.io/kustomize/api/internal/crawl/utils"
+
+	"sigs.k8s.io/kustomize/api/internal/crawl/index"
+
 	_ "github.com/gomodule/redigo/redis"
 
 	"sigs.k8s.io/kustomize/api/internal/crawl/doc"
@@ -27,157 +31,210 @@ type Crawler interface {
 	// Crawl returns when it is done processing. This method does not take
 	// ownership of the channel. The channel is write only, and it
 	// designates where the crawler should forward the documents.
-	Crawl(ctx context.Context, output chan<- CrawledDocument) error
+	Crawl(ctx context.Context, output chan<- CrawledDocument, seen utils.SeenMap) error
 
 	// Get the document data given the FilePath, Repo, and Ref/Tag/Branch.
 	FetchDocument(context.Context, *doc.Document) error
 	// Write to the document what the created time is.
 	SetCreated(context.Context, *doc.Document) error
 
+	SetDefaultBranch(*doc.Document)
+
 	Match(*doc.Document) bool
 }
 
 type CrawledDocument interface {
 	ID() string
 	GetDocument() *doc.Document
-	GetResources() ([]*doc.Document, error)
+	// Get all the Documents directly referred in a Document.
+	// For a Document representing a non-kustomization file, an empty slice will be returned.
+	// For a Document representing a kustomization file:
+	// the `includeResources` parameter determines whether the documents referred in the `resources` field are returned or not;
+	// the `includeTransformers` parameter determines whether the documents referred in the `transformers` field are returned or not;
+	// the `includeGenerators` parameter determines whether the documents referred in the `generators` field are returned or not.
+	GetResources(includeResources, includeTransformers, includeGenerators bool) ([]*doc.Document, error)
 	WasCached() bool
 }
 
 type CrawlSeed []*doc.Document
 
-type IndexFunc func(CrawledDocument, Crawler) error
+type IndexFunc func(CrawledDocument, index.Mode) error
 type Converter func(*doc.Document) (CrawledDocument, error)
 
-// Cleaner, more efficient, and more extensible crawler implementation.
-// The seed must include the ids of each document in the index.
-func CrawlFromSeed(ctx context.Context, seed CrawlSeed,
-	crawlers []Crawler, conv Converter, indx IndexFunc) {
+func logIfErr(err error) {
+	if err == nil {
+		return
+	}
+	logger.Println("error: ", err)
+}
 
-	seen := make(map[string]struct{})
-
-	logIfErr := func(err error) {
-		if err == nil {
-			return
+func findMatch(d *doc.Document, crawlers []Crawler) Crawler {
+	for _, crawl := range crawlers {
+		if crawl.Match(d) {
+			return crawl
 		}
-		logger.Println("error: ", err)
+	}
+	return nil
+}
+
+func addBranches(cdoc CrawledDocument, match Crawler, indx IndexFunc,
+	seen utils.SeenMap, stack *CrawlSeed) {
+
+	seen.Set(cdoc.ID(), cdoc.GetDocument().FileType)
+
+	match.SetDefaultBranch(cdoc.GetDocument())
+
+	// Insert into index
+	if err := indx(cdoc, index.InsertOrUpdate); err != nil {
+		logger.Printf("Failed to insert or update doc(%s): %v",
+			cdoc.GetDocument().Path(), err)
+		return
 	}
 
-	stack := make(CrawlSeed, 0)
-
-	findMatch := func(d *doc.Document) Crawler {
-		for _, crawl := range crawlers {
-			if crawl.Match(d) {
-				return crawl
-			}
-		}
-
-		return nil
+	deps, err := cdoc.GetResources(true, true, true)
+	if err != nil {
+		logger.Println(err)
+		return
 	}
 
-	addBranches := func(cdoc CrawledDocument, match Crawler) {
-		if _, ok := seen[cdoc.ID()]; ok {
-			return
+	for _, dep := range deps {
+		if seen.Seen(dep.ID()) && seen.Value(dep.ID()) == dep.FileType {
+			continue
 		}
+		*stack = append(*stack, dep)
+	}
+}
 
-		seen[cdoc.ID()] = struct{}{}
-		// Insert into index
-		err := indx(cdoc, match)
-		logIfErr(err)
-		if err != nil {
-			return
-		}
+func doCrawl(ctx context.Context, docsPtr *CrawlSeed, crawlers []Crawler, conv Converter, indx IndexFunc,
+	seen utils.SeenMap, stack *CrawlSeed, refreshDoc bool, updateFileType bool) {
 
-		deps, err := cdoc.GetResources()
-		logIfErr(err)
-		if err != nil {
-			return
-		}
-		for _, dep := range deps {
-			if _, ok := seen[dep.ID()]; ok {
+	UpdatedDocCount := 0
+	seenDocCount := 0
+	cachedDocCount := 0
+	findMatchErrCount := 0
+	FetchDocumentErrCount := 0
+	SetCreatedErrCount := 0
+	convErrCount := 0
+	deleteDocCount := 0
+	crawledDocCount := 0
+
+	// During the execution of the for loop, more Documents may be added into (*docsPtr).
+	for len(*docsPtr) > 0 {
+		// get the last Document in (*docPtr), which will be crawled in this iteration.
+		tail := (*docsPtr)[len(*docsPtr)-1]
+
+		// remove the last Document in (*docPtr)
+		*docsPtr = (*docsPtr)[:(len(*docsPtr) - 1)]
+
+		crawledDocCount++
+		logger.Printf("Crawling doc %d: %s", crawledDocCount, tail.Path())
+
+		if seen.Seen(tail.ID()) {
+			if !updateFileType || seen.Value(tail.ID()) == tail.FileType {
+				logger.Printf("this doc has been seen before")
+				seenDocCount++
 				continue
 			}
-			stack = append(stack, dep)
 		}
-	}
 
-	doCrawl := func(docsPtr *CrawlSeed) {
-		n := len(*docsPtr)
-		for i := 0; i < n; i++ {
-			next := (*docsPtr)[i]
-			match := findMatch(next)
-			if match == nil {
-				logIfErr(fmt.Errorf(
-					"%v could not match any crawler", next))
-				continue
-			}
+		if tail.WasCached() {
+			logger.Printf("doc(%s) is cached already", tail.Path())
+			cachedDocCount++
+			continue
+		}
 
-			logger.Println("Crawling ", next.RepositoryURL, next.FilePath)
-			err := match.FetchDocument(ctx, next)
-			logIfErr(err)
-			// If there was no change or there is an error, we don't have
-			// to branch out, since the dependencies are already in the
-			// index, or we cannot find the document.
-			if err != nil || next.WasCached() {
-				if next.WasCached() {
-					logger.Println(next.RepositoryURL, next.FilePath, "is cached already")
+		match := findMatch(tail, crawlers)
+		if match == nil {
+			logIfErr(fmt.Errorf("%v could not match any crawler", tail))
+			findMatchErrCount++
+			continue
+		}
+
+		if tail.User == "" {
+			tail.User = doc.UserName(tail.RepositoryURL)
+		}
+
+		// If the Document represents a kustomization root, FetchDcoument will change
+		// the `filePath` field of the Document by adding `kustomization.yaml` or
+		// `kustomization.yml` or `kustomization` into the the field.
+		// Therefore, it is necessary to add the ID of the Document into seen before
+		// calling FetchDocument. Otherwise, the binary may enter into an infinite loop
+		// if a kustomization file points to its kustmozation root in its `resources` or
+		// `bases` field.
+		seen.Set(tail.ID(), tail.FileType)
+
+		if refreshDoc || tail.DefaultBranch == "" {
+			match.SetDefaultBranch(tail)
+		}
+
+		if refreshDoc || tail.DocumentData == "" {
+			if err := match.FetchDocument(ctx, tail); err != nil {
+				logger.Printf("FetchDocument failed on doc(%s): %v", tail.Path(), err)
+				FetchDocumentErrCount++
+				// delete the document from the index
+				cdoc := &doc.KustomizationDocument{
+					Document: *tail,
 				}
+				seen.Set(cdoc.ID(), tail.FileType)
+				if err := indx(cdoc, index.Delete); err != nil {
+					logger.Printf("Failed to delete doc(%s): %v", cdoc.Path(), err)
+				}
+				deleteDocCount++
 				continue
 			}
-
-			logIfErr(match.SetCreated(ctx, next))
-
-			cdoc, err := conv(next)
-			logIfErr(err)
-
-			addBranches(cdoc, match)
 		}
+
+		if refreshDoc || tail.CreationTime == nil {
+			if err := match.SetCreated(ctx, tail); err != nil {
+				logger.Printf("SetCreated failed on doc(%s): %v", tail.Path(), err)
+				SetCreatedErrCount++
+			}
+		}
+
+		cdoc, err := conv(tail)
+		// If conv returns an error, cdoc can still be added into the index so that
+		// cdoc.Document can be searched.
+		if err != nil {
+			logger.Printf("conv failed on doc(%s): %v", tail.Path(), err)
+			convErrCount++
+		}
+
+		UpdatedDocCount++
+		addBranches(cdoc, match, indx, seen, stack)
 	}
+	logger.Printf("Summary of doCrawl:\n")
+	logger.Printf("\t%d documents were updated\n", UpdatedDocCount)
+	logger.Printf("\t%d documents were seen by the crawler already and skipped\n", seenDocCount)
+	logger.Printf("\t%d documents were cached already and skipped\n", cachedDocCount)
+	logger.Printf("\t%d documents didn't have a matching crawler and skipped\n", findMatchErrCount)
+	logger.Printf("\t%d documents cannot be fetched, %d out of them are deleted\n",
+		FetchDocumentErrCount, deleteDocCount)
+	logger.Printf("\t%d documents cannot update its creation time but still were inserted or updated in the index\n", SetCreatedErrCount)
+	logger.Printf("\t%d documents cannot be converted but still were inserted or updated in the index\n", convErrCount)
+}
+
+// CrawlFromSeed updates all the documents in seed, and crawls all the new
+// documents referred in the seed.
+func CrawlFromSeed(ctx context.Context, seed CrawlSeed, crawlers []Crawler,
+	conv Converter, indx IndexFunc, seen utils.SeenMap) {
+
+	// stack tracks the documents directly referred in other documents.
+	stack := make(CrawlSeed, 0)
 
 	// Exploit seed to update bulk of corpus.
 	logger.Printf("updating %d documents from seed\n", len(seed))
-	doCrawl(&seed)
-	// Traverse any new links added while updating corpus.
+	// each unique document in seed will be crawled once.
+	doCrawl(ctx, &seed, crawlers, conv, indx, seen, &stack, true, false)
+
+	// Traverse any new documents added while updating corpus.
 	logger.Printf("crawling %d new documents found in the seed\n", len(stack))
-	doCrawl(&stack)
-
-	ch := make(chan CrawledDocument, 1<<10)
-	wg := sync.WaitGroup{}
-
-	wg.Add(1)
-	go func() {
-		defer wg.Done()
-		for cdoc := range ch {
-			if _, ok := seen[cdoc.ID()]; ok {
-				continue
-			}
-			match := findMatch(cdoc.GetDocument())
-			if match == nil {
-				logIfErr(fmt.Errorf(
-					"%v could not match any crawler", cdoc))
-				continue
-			}
-			addBranches(cdoc, match)
-		}
-	}()
-
-	// Exploration through APIs.
-	errs := CRunner(ctx, ch, crawlers)
-	if errs != nil {
-		for _, err := range errs {
-			logIfErr(err)
-		}
-	}
-	close(ch)
-	logger.Println("Processing the new documents from the crawlers' exploration.")
-	wg.Wait()
-	// Handle deps of newly discovered documents.
-	logger.Printf("crawling the %d new documents from the crawlers' exploration.",
-		len(stack))
-	doCrawl(&stack)
+	// While crawling each document in stack, the documents directly referred in the document
+	// will be added into stack.
+	// After this statement is done, stack will become empty.
+	doCrawl(ctx, &stack, crawlers, conv, indx, seen, &stack, false, true)
 }
 
-// CRunner is a blocking function and only returns once all of the
+// CrawlGithubRunner is a blocking function and only returns once all of the
 // crawlers are finished with execution.
 //
 // This function uses the output channel to forward kustomization documents
@@ -188,14 +245,14 @@ func CrawlFromSeed(ctx context.Context, seed CrawlSeed,
 // index of the crawler that emitted the error. Although the errors themselves
 // can be nil, the array will always be exactly the size of the crawlers array.
 //
-// CRunner takes in a seed, which represents the documents stored in an
+// CrawlGithubRunner takes in a seed, which represents the documents stored in an
 // index somewhere. The document data is not required to be populated. If there
 // are many documents, this is preferable. The order of iteration over the seed
-// is not guaranteed, but the CRunner does guarantee that every element
+// is not guaranteed, but the CrawlGithub does guarantee that every element
 // from the seed will be processed before any other documents from the
 // crawlers.
-func CRunner(ctx context.Context,
-	output chan<- CrawledDocument, crawlers []Crawler) []error {
+func CrawlGithubRunner(ctx context.Context, output chan<- CrawledDocument,
+	crawlers []Crawler, seen utils.SeenMap) []error {
 
 	errs := make([]error, len(crawlers))
 	wg := sync.WaitGroup{}
@@ -229,10 +286,59 @@ func CRunner(ctx context.Context,
 				}
 			}()
 			defer close(docs)
-			errs[idx] = crawler.Crawl(ctx, docs)
+			errs[idx] = crawler.Crawl(ctx, docs, seen)
 		}(i, crawler, docs) // Copies the index and the crawler
 	}
 
 	wg.Wait()
 	return errs
 }
+
+// CrawlGithub crawls all the kustomization files on Github.
+func CrawlGithub(ctx context.Context, crawlers []Crawler, conv Converter,
+	indx IndexFunc, seen utils.SeenMap) {
+	// stack tracks the documents directly referred in other documents.
+	stack := make(CrawlSeed, 0)
+
+	// ch is channel where all the crawlers sends the crawled documents to.
+	ch := make(chan CrawledDocument, 1<<10)
+
+	wg := sync.WaitGroup{}
+
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		docCount := 0
+		for cdoc := range ch {
+			docCount++
+			logger.Printf("Processing doc %d found on Github", docCount)
+			// all the docs here are kustomization files found by querying Github, and
+			// their `FileType` fields all should be empty.
+			if seen.Seen(cdoc.ID()) {
+				logger.Printf("the doc has been seen before")
+				continue
+			}
+			match := findMatch(cdoc.GetDocument(), crawlers)
+			if match == nil {
+				logIfErr(fmt.Errorf(
+					"%v could not match any crawler", cdoc))
+				continue
+			}
+			addBranches(cdoc, match, indx, seen, &stack)
+		}
+	}()
+
+	logger.Println("processing the documents found from crawling github")
+	if errs := CrawlGithubRunner(ctx, ch, crawlers, seen); errs != nil {
+		for _, err := range errs {
+			logIfErr(err)
+		}
+	}
+	close(ch)
+	wg.Wait()
+
+	// Handle deps of newly discovered documents.
+	logger.Printf("crawling the %d new documents referred by other documents",
+		len(stack))
+	doCrawl(ctx, &stack, crawlers, conv, indx, seen, &stack, false, true)
+}

api/internal/crawl/crawler/crawler_test.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"log"
 	"reflect"
 	"sort"
 	"strings"
@@ -11,6 +12,10 @@ import (
 	"testing"
 	"time"
 
+	"sigs.k8s.io/kustomize/api/internal/crawl/utils"
+
+	"sigs.k8s.io/kustomize/api/internal/crawl/index"
+
 	"sigs.k8s.io/kustomize/api/internal/crawl/doc"
 	"sigs.k8s.io/kustomize/api/konfig"
 )
@@ -32,18 +37,21 @@ func (c testCrawler) Match(d *doc.Document) bool {
 	return d != nil
 }
 
+func (c testCrawler) SetDefaultBranch(d *doc.Document) {}
+
 func (c testCrawler) FetchDocument(_ context.Context, d *doc.Document) error {
 	if i, ok := c.lukp[d.ID()]; ok {
 		d.DocumentData = c.docs[i].DocumentData
 		return nil
 	}
 	for _, suffix := range konfig.RecognizedKustomizationFileNames() {
-		fmt.Println(d.ID(), "/", suffix)
-		i, ok := c.lukp[d.ID()+"/"+suffix]
+		savedFilePath := d.FilePath
+		d.FilePath += "/" + suffix
+		i, ok := c.lukp[d.ID()]
 		if !ok {
+			d.FilePath = savedFilePath
 			continue
 		}
-		d.FilePath += "/" + suffix
 		d.DocumentData = c.docs[i].DocumentData
 		return nil
 	}
@@ -72,7 +80,7 @@ func newCrawler(matchPrefix string, err error,
 
 // Crawl implements the Crawler interface for testing.
 func (c testCrawler) Crawl(_ context.Context,
-	output chan<- CrawledDocument) error {
+	output chan<- CrawledDocument, _ utils.SeenMap) error {
 
 	for i, d := range c.docs {
 		isResource := true
@@ -106,8 +114,8 @@ func (s sortableDocs) Len() int {
 	return len(s)
 }
 
-func TestCrawlerRunner(t *testing.T) {
-	fmt.Println("testing CRunner")
+func TestCrawlGithubRunner(t *testing.T) {
+	log.Println("testing CrawlGithubRunner")
 	tests := []struct {
 		tc   []Crawler
 		errs []error
@@ -178,8 +186,9 @@ func TestCrawlerRunner(t *testing.T) {
 			defer close(output)
 			defer wg.Done()
 
-			errs := CRunner(context.Background(),
-				output, test.tc)
+			seen := utils.NewSeenMap()
+			errs := CrawlGithubRunner(context.Background(),
+				output, test.tc, seen)
 
 			// Check that errors are returned as they should be.
 			if !reflect.DeepEqual(errs, test.errs) {
@@ -212,7 +221,7 @@ func TestCrawlerRunner(t *testing.T) {
 }
 
 func TestCrawlFromSeed(t *testing.T) {
-	fmt.Println("testing CrawlFromSeed")
+	log.Println("testing CrawlFromSeed")
 
 	tests := []struct {
 		seed    CrawlSeed
@@ -302,29 +311,6 @@ resources:
 					RepositoryURL: kustomizeRepo,
 					FilePath:      "examples/seedcrawl2/job.yaml",
 				}},
-				// Visited from the crawler runner.
-				{Document: doc.Document{
-					RepositoryURL: kustomizeRepo,
-					FilePath:      "examples/other/base/kustomization.yaml",
-					DocumentData: `
-resources:
-- ../app
-`,
-				}},
-				// Visited from the crawler runner.
-				{Document: doc.Document{
-					RepositoryURL: kustomizeRepo,
-					FilePath:      "examples/other/app/kustomization.yaml",
-					DocumentData: `
-resources:
-- resource.yaml
-`,
-				}},
-				// Visited from crawling runner imported as resource.
-				{Document: doc.Document{
-					RepositoryURL: kustomizeRepo,
-					FilePath:      "examples/other/app/resource.yaml",
-				}},
 			},
 		},
 	}
@@ -338,10 +324,11 @@ resources:
 					Document: *d,
 				}, nil
 			},
-			func(d CrawledDocument, cr Crawler) error {
+			func(d CrawledDocument, mode index.Mode) error {
 				visited[d.ID()]++
 				return nil
 			},
+			utils.NewSeenMap(),
 		)
 		if lv, lc := len(visited), len(tc.corpus); lv != lc {
 			t.Errorf("error: %d of %d documents visited.", lv, lc)

api/internal/crawl/crawler/github/crawler.go

@@ -16,6 +16,8 @@ import (
 	"strings"
 	"time"
 
+	"sigs.k8s.io/kustomize/api/internal/crawl/utils"
+
 	"sigs.k8s.io/kustomize/api/internal/crawl/crawler"
 	"sigs.k8s.io/kustomize/api/internal/crawl/doc"
 	"sigs.k8s.io/kustomize/api/internal/crawl/httpclient"
@@ -30,6 +32,8 @@ var logger = log.New(os.Stdout, "Github Crawler: ",
 type githubCrawler struct {
 	client GhClient
 	query  Query
+	// branchMap maps github repositories to their default branches
+	branchMap map[string]string
 }
 
 type GhClient struct {
@@ -51,13 +55,60 @@ func NewCrawler(accessToken string, retryCount uint64, client *http.Client,
 			},
 			accessToken: accessToken,
 		},
-		query: query,
+		query:     query,
+		branchMap: map[string]string{},
 	}
 }
 
+func (gc githubCrawler) SetDefaultBranch(d *doc.Document) {
+	url := gc.client.ReposRequest(d.RepositoryFullName())
+	defaultBranch, err := gc.client.GetDefaultBranch(url, d.RepositoryURL, gc.branchMap)
+	if err != nil {
+		logger.Printf(
+			"(error: %v) setting default_branch to master\n", err)
+		defaultBranch = "master"
+	}
+	d.DefaultBranch = defaultBranch
+	gc.branchMap[d.RepositoryURL] = d.DefaultBranch
+}
+
+func (gc githubCrawler) DefaultBranch(repo string) string {
+	return gc.branchMap[repo]
+}
+
 // Implements crawler.Crawler.
-func (gc githubCrawler) Crawl(
-	ctx context.Context, output chan<- crawler.CrawledDocument) error {
+func (gc githubCrawler) Crawl(ctx context.Context,
+	output chan<- crawler.CrawledDocument, seen utils.SeenMap) error {
+
+	ranges := []RangeWithin{
+		RangeWithin{
+		start: uint64(0),
+		end:   githubMaxFileSize,
+	},
+	}
+
+	errs := make(multiError, 0)
+	for len(ranges) > 0 {
+		tailRange := ranges[len(ranges) - 1]
+		ranges = ranges[:(len(ranges) - 1)]
+		reProcessQueryRanges, err := gc.CrawlSingleRange(ctx, output, seen, tailRange.start, tailRange.end)
+		if err != nil {
+			errs = append(errs, err)
+		}
+		ranges = append(ranges, reProcessQueryRanges...)
+	}
+
+	if len(errs) > 0 {
+		return errs
+	}
+	return nil
+}
+
+func (gc githubCrawler) CrawlSingleRange(ctx context.Context,
+	output chan<- crawler.CrawledDocument, seen utils.SeenMap,
+	lowerBound, upperBound uint64) ([]RangeWithin, error) {
+
+	log.Printf("CrawlSingleRange [%d, %d]", lowerBound, upperBound)
 
 	noETagClient := GhClient{
 		RequestConfig: gc.client.RequestConfig,
@@ -66,12 +117,25 @@ func (gc githubCrawler) Crawl(
 		accessToken:   gc.client.accessToken,
 	}
 
+	var reProcessQueryRanges []RangeWithin
+
+	var ranges []string
+	var err error
 	// Since Github returns a max of 1000 results per query, we can use
 	// multiple queries that split the search space into chunks of at most
 	// 1000 files to get all of the data.
-	ranges, err := FindRangesForRepoSearch(newCache(noETagClient, gc.query))
+	for i := 0; i < 5; i++ {
+		ranges, err = FindRangesForRepoSearch(newCache(noETagClient, gc.query),
+			lowerBound, upperBound)
+		if err == nil {
+			logger.Printf("FindRangesForRepoSearch succeeded after %d retries", i)
+			break
+		} else {
+			time.Sleep(time.Minute)
+		}
+	}
 	if err != nil {
-		return fmt.Errorf("could not split %v into ranges, %v\n",
+		return reProcessQueryRanges, fmt.Errorf("could not split %v into ranges, %v\n",
 			gc.query, err)
 	}
 
@@ -79,20 +143,30 @@ func (gc githubCrawler) Crawl(
 
 	// Query each range for files.
 	errs := make(multiError, 0)
+	queryResult := RangeQueryResult{}
 	for _, query := range ranges {
-		err := processQuery(ctx, gc.client, query, output)
+		reProcessQuery, rangeResult, err := processQuery(ctx, gc.client, query, output, seen, gc.branchMap)
 		if err != nil {
 			errs = append(errs, err)
 		}
+		queryResult.Add(rangeResult)
+		if reProcessQuery {
+			reProcessQueryRanges = append(reProcessQueryRanges, RangeSizes(query))
+		}
 	}
 
+	logger.Printf("Summary of Crawl: %s", queryResult.String())
+
 	if len(errs) > 0 {
-		return errs
+		return reProcessQueryRanges, errs
 	}
 
-	return nil
+	return reProcessQueryRanges, nil
 }
 
+// FetchDocument first tries to fetch the document with d.FilePath. If it fails,
+// it will try to add each string in konfig.RecognizedKustomizationFileNames() to
+// d.FilePath, and try to fetch the document again.
 func (gc githubCrawler) FetchDocument(_ context.Context, d *doc.Document) error {
 	repoURL := d.RepositoryURL + "/" + d.FilePath + "?ref=" + d.DefaultBranch
 	repoSpec, err := git.NewRepoSpecFromUrl(repoURL)
@@ -104,9 +178,13 @@ func (gc githubCrawler) FetchDocument(_ context.Context, d *doc.Document) error
 		"/" + repoSpec.Ref + "/" + repoSpec.Path
 
 	handle := func(resp *http.Response, err error, path string) error {
+		if resp == nil {
+			return fmt.Errorf("empty http response (url: %s; path: %s), error: %v",
+				url, path, err)
+		}
 		if err == nil && resp.StatusCode == http.StatusOK {
 			d.IsSame = httpclient.FromCache(resp.Header)
-			defer resp.Body.Close()
+			defer CloseResponseBody(resp)
 			data, err := ioutil.ReadAll(resp.Body)
 			if err != nil {
 				return err
@@ -115,18 +193,18 @@ func (gc githubCrawler) FetchDocument(_ context.Context, d *doc.Document) error
 			d.FilePath = d.FilePath + path
 			return nil
 		}
+
 		return err
 	}
-	resp, err := gc.client.GetRawUserContent(url)
-	if err := handle(resp, err, ""); err == nil {
+	resp, errGetRawUserContent := gc.client.GetRawUserContent(url)
+	if err := handle(resp, errGetRawUserContent, ""); err == nil {
 		return nil
 	}
 
 	for _, file := range konfig.RecognizedKustomizationFileNames() {
-		resp, err = gc.client.GetRawUserContent(url + "/" + file)
-		err := handle(resp, err, "/"+file)
-		if err != nil {
-			continue
+		resp, errGetRawUserContent = gc.client.GetRawUserContent(url + "/" + file)
+		if err = handle(resp, errGetRawUserContent, "/"+file); err == nil {
+			return nil
 		}
 	}
 	return fmt.Errorf("file not found: %s, error: %v", url, err)
@@ -158,10 +236,32 @@ func (gc githubCrawler) Match(d *doc.Document) bool {
 	return strings.Contains(repoSpec.Host, "github.com")
 }
 
+type RangeQueryResult struct {
+	totalDocCnt uint64
+	seenDocCnt  uint64
+	newDocCnt   uint64
+	errorCnt    uint64
+}
+
+func (r *RangeQueryResult) Add(other RangeQueryResult) {
+	r.totalDocCnt += other.totalDocCnt
+	r.newDocCnt += other.newDocCnt
+	r.seenDocCnt += other.seenDocCnt
+	r.errorCnt += other.errorCnt
+}
+
+func (r *RangeQueryResult) String() string {
+	return fmt.Sprintf("got %d files from API. "+
+		"%d have been seen before. %d are new and sent to the output channel."+
+		" %d have kustomizationResultAdapter errors.",
+		r.totalDocCnt, r.seenDocCnt, r.newDocCnt, r.errorCnt)
+}
+
 // processQuery follows all of the pages in a query, and updates/adds the
 // documents from the crawl to the datastore/index.
 func processQuery(ctx context.Context, gcl GhClient, query string,
-	output chan<- crawler.CrawledDocument) error {
+	output chan<- crawler.CrawledDocument, seen utils.SeenMap,
+	branchMap map[string]string) (bool, RangeQueryResult, error) {
 
 	queryPages := make(chan GhResponseInfo)
 
@@ -177,60 +277,86 @@ func processQuery(ctx context.Context, gcl GhClient, query string,
 		close(queryPages)
 	}()
 
+	reProcessQuery := false
+
 	errs := make(multiError, 0)
-	errorCnt := 0
-	totalCnt := 0
+	result := RangeQueryResult{}
+	pageID := 1
 	for page := range queryPages {
 		if page.Error != nil {
 			errs = append(errs, page.Error)
 			continue
 		}
-
+		pageResult := RangeQueryResult{}
 		for _, file := range page.Parsed.Items {
-			k, err := kustomizationResultAdapter(gcl, file)
+			k, err := kustomizationResultAdapter(gcl, file, seen, branchMap)
 			if err != nil {
 				logger.Printf("kustomizationResultAdapter failed: %v", err)
 				errs = append(errs, err)
-				errorCnt++
+				pageResult.errorCnt++
 			}
 			if k != nil {
+				pageResult.newDocCnt++
 				output <- k
+			} else {
+				pageResult.seenDocCnt++
 			}
-			totalCnt++
+			pageResult.totalDocCnt++
 		}
 
-		logger.Printf("got %d files out of %d from API. %d of %d had errors\n",
-			totalCnt, page.Parsed.TotalCount, errorCnt, totalCnt)
+		logger.Printf("processQuery [TotalCount %d - page %d]: %s",
+			page.Parsed.TotalCount, pageID, pageResult.String())
+		result.Add(pageResult)
+
+		pageID++
+
+		if page.Parsed.TotalCount > githubMaxResultsPerQuery {
+			reProcessQuery = true
+		}
 	}
 
-	return errs
+	logger.Printf("Summary of processQuery: %s", result.String())
+
+	return reProcessQuery, result, errs
 }
 
-func kustomizationResultAdapter(gcl GhClient, k GhFileSpec) (
-	crawler.CrawledDocument, error) {
-
-	data, err := gcl.GetFileData(k)
-	if err != nil {
-		return nil, err
-	}
-
+func kustomizationResultAdapter(gcl GhClient, k GhFileSpec, seen utils.SeenMap,
+	branchMap map[string]string) (crawler.CrawledDocument, error) {
 	url := gcl.ReposRequest(k.Repository.FullName)
-	defaultBranch, err := gcl.GetDefaultBranch(url)
+	defaultBranch, err := gcl.GetDefaultBranch(url, k.Repository.URL, branchMap)
 	if err != nil {
 		logger.Printf(
 			"(error: %v) setting default_branch to master\n", err)
 		defaultBranch = "master"
 	}
 
+	// document here is a kustomization file found by querying Github, whose
+	// `FileType` field should be empty.
+	document := doc.Document{
+		FilePath:      k.Path,
+		DefaultBranch: defaultBranch,
+		RepositoryURL: k.Repository.URL,
+		User:          doc.UserName(k.Repository.URL),
+	}
+
+	if seen.Seen(document.ID()) {
+		return nil, nil
+	}
+
+	data, err := gcl.GetFileData(k)
+	if err != nil {
+		return nil, err
+	}
+
 	d := doc.KustomizationDocument{
 		Document: doc.Document{
 			DocumentData:  string(data),
 			FilePath:      k.Path,
 			DefaultBranch: defaultBranch,
 			RepositoryURL: k.Repository.URL,
+			User:          doc.UserName(k.Repository.URL),
 		},
 	}
-	logger.Printf("Set the creationTime field")
 	creationTime, err := gcl.GetFileCreationTime(k)
 	if err != nil {
 		logger.Printf("GetFileCreationTime failed: %v", err)
@@ -253,7 +379,7 @@ func (gcl GhClient) ForwardPaginatedQuery(ctx context.Context, query string,
 	output chan<- GhResponseInfo) error {
 
 	logger.Println("querying: ", query)
-	response := gcl.parseGithubResponse(query)
+	response := gcl.parseGithubResponseWithRetry(query)
 
 	if response.Error != nil {
 		return response.Error
@@ -266,7 +392,7 @@ func (gcl GhClient) ForwardPaginatedQuery(ctx context.Context, query string,
 		case <-ctx.Done():
 			return nil
 		default:
-			response = gcl.parseGithubResponse(response.NextURL)
+			response = gcl.parseGithubResponseWithRetry(response.NextURL)
 			if response.Error != nil {
 				return response.Error
 			}
@@ -294,7 +420,10 @@ func (gcl GhClient) GetFileData(k GhFileSpec) ([]byte, error) {
 		return nil, fmt.Errorf("%+v: could not read '%s' metadata: %v",
 			k, url, err)
 	}
-	resp.Body.Close()
+
+	if err := resp.Body.Close(); err != nil {
+		return nil, err
+	}
 
 	type githubContentRawURL struct {
 		DownloadURL string `json:"download_url,omitempty"`
@@ -313,18 +442,32 @@ func (gcl GhClient) GetFileData(k GhFileSpec) ([]byte, error) {
 			k, rawURL.DownloadURL, err)
 	}
 
-	defer resp.Body.Close()
+	defer CloseResponseBody(resp)
 	data, err = ioutil.ReadAll(resp.Body)
 	return data, err
 }
 
-func (gcl GhClient) GetDefaultBranch(url string) (string, error) {
+func CloseResponseBody(resp *http.Response) {
+	if err := resp.Body.Close(); err != nil {
+		log.Printf("failed to close response body: %v", err)
+	}
+}
+
+// GetDefaultBranch gets the default branch of a github repository.
+// m is a map which maps a github repository to its default branch.
+// If repo is already in m, the default branch for url will be obtained from m;
+// otherwise, a query will be made to github to obtain the default branch.
+func (gcl GhClient) GetDefaultBranch(url, repo string, m map[string]string) (string, error) {
+	if v, ok := m[repo]; ok {
+		return v, nil
+	}
+
 	resp, err := gcl.GetReposData(url)
 	if err != nil {
 		return "", fmt.Errorf(
 			"'%s' could not get default_branch: %v", url, err)
 	}
-	defer resp.Body.Close()
+	defer CloseResponseBody(resp)
 	data, err := ioutil.ReadAll(resp.Body)
 	if err != nil {
 		return "", fmt.Errorf(
@@ -376,7 +519,7 @@ func (gcl GhClient) GetFileCreationTime(
 		}
 	}
 
-	defer resp.Body.Close()
+	defer CloseResponseBody(resp)
 	data, err := ioutil.ReadAll(resp.Body)
 	if err != nil {
 		return defaultTime, fmt.Errorf(
@@ -444,6 +587,8 @@ type githubResponse struct {
 	// This is the number of files that match the query.
 	TotalCount uint64 `json:"total_count,omitempty"`
 
+	IncompleteResults bool `json:"incomplete_results,omitempty"`
+
 	// Github representation of a file.
 	Items []GhFileSpec `json:"items,omitempty"`
 }
@@ -486,6 +631,17 @@ func parseGithubLinkFormat(links string) (string, string) {
 	return next, last
 }
 
+func (gcl GhClient) parseGithubResponseWithRetry(getRequest string) GhResponseInfo {
+	resp := gcl.parseGithubResponse(getRequest)
+	retries := 0
+	for resp.Parsed.IncompleteResults {
+		resp = gcl.parseGithubResponse(getRequest)
+		retries++
+	}
+	log.Printf("The result of query(%s) is complete after %d retries", getRequest, retries)
+	return resp
+}
+
 func (gcl GhClient) parseGithubResponse(getRequest string) GhResponseInfo {
 	resp, err := gcl.SearchGithubAPI(getRequest)
 	requestInfo := GhResponseInfo{
@@ -499,7 +655,7 @@ func (gcl GhClient) parseGithubResponse(getRequest string) GhResponseInfo {
 	}
 
 	var data []byte
-	defer resp.Body.Close()
+	defer CloseResponseBody(resp)
 	data, requestInfo.Error = ioutil.ReadAll(resp.Body)
 	if requestInfo.Error != nil {
 		return requestInfo
@@ -533,7 +689,7 @@ func (gcl GhClient) parseGithubResponse(getRequest string) GhResponseInfo {
 }
 
 // SearchGithubAPI performs a search query and handles rate limitting for
-// the 'code/search?' endpoint as well as timed retries in the case of abuse
+// the 'search/code?' endpoint as well as timed retries in the case of abuse
 // prevention.
 func (gcl GhClient) SearchGithubAPI(query string) (*http.Response, error) {
 	throttleSearchAPI()
@@ -560,7 +716,15 @@ func (gcl GhClient) Do(query string) (*http.Response, error) {
 		return nil, err
 	}
 	req.Header.Add("Authorization", fmt.Sprintf("token %s", gcl.accessToken))
-	return gcl.client.Do(req)
+
+	// gcl.client.Do: a non-2xx status code doesn't cause an error.
+	// See https://golang.org/pkg/net/http/#Client.Do for more info.
+	resp, err := gcl.client.Do(req)
+	if resp != nil && resp.StatusCode != http.StatusOK {
+		err = fmt.Errorf("GhClient.Do(%s) failed with response code: %d",
+			query, resp.StatusCode)
+	}
+	return resp, err
 }
 
 func (gcl GhClient) getWithRetry(
@@ -570,13 +734,10 @@ func (gcl GhClient) getWithRetry(
 
 	retryCount := gcl.retryCount
 
-	for err == nil &&
-		resp.StatusCode == http.StatusForbidden &&
-		retryCount > 0 {
-
+	for resp != nil && resp.StatusCode == http.StatusForbidden && retryCount > 0 {
 		retryTime := resp.Header.Get("Retry-After")
-		i, err := strconv.Atoi(retryTime)
-		if err != nil {
+		i, errAtoi := strconv.Atoi(retryTime)
+		if errAtoi != nil {
 			return resp, fmt.Errorf(
 				"query '%s' forbidden without 'Retry-After'", query)
 		}

api/internal/crawl/crawler/github/queries.go

@@ -90,6 +90,17 @@ func Path(p string) queryField {
 	return queryField{name: "path", value: p}
 }
 
+// Repo takes a repository (i.e., kubernetes-sigs/kustomize) and formats
+// it according to the Github API.
+func Repo(r string) queryField {
+	return queryField{name: "repo", value: r}
+}
+
+// Path takes a github username and formats it according to the Github API.
+func User(u string) queryField {
+	return queryField{name: "user", value: u}
+}
+
 // RequestConfig stores common variables that must be present for the queries.
 // - CodeSearchRequests: ask Github to check the code indices given a query.
 // - ContentsRequests: ask Github where to download a resource given a repo and a
@@ -105,9 +116,11 @@ type RequestConfig struct {
 // the URL method to get the string value of the URL. See request.CopyWith, to
 // understand why the request object is useful.
 func (rc RequestConfig) CodeSearchRequestWith(query Query) request {
-	req := rc.makeRequest("search/code", query)
-	req.vals.Set("sort", "indexed")
-	req.vals.Set("order", "desc")
+	vals := url.Values{
+		"sort":  []string{"indexed"},
+		"order": []string{"desc"},
+	}
+	req := rc.makeRequest("search/code", query, vals)
 	return req
 }
 
@@ -115,23 +128,25 @@ func (rc RequestConfig) CodeSearchRequestWith(query Query) request {
 // query for the Github API to find the dowload information of this filepath.
 func (rc RequestConfig) ContentsRequest(fullRepoName, path string) string {
 	uri := fmt.Sprintf("repos/%s/contents/%s", fullRepoName, path)
-	return rc.makeRequest(uri, Query{}).URL()
+	return rc.makeRequest(uri, Query{}, url.Values{}).URL()
 }
 
 func (rc RequestConfig) ReposRequest(fullRepoName string) string {
 	uri := fmt.Sprintf("repos/%s", fullRepoName)
-	return rc.makeRequest(uri, Query{}).URL()
+	return rc.makeRequest(uri, Query{}, url.Values{}).URL()
 }
 
 // CommitsRequest given the repo name, and a filepath returns a formatted query
 // for the Github API to find the commits that affect this file.
 func (rc RequestConfig) CommitsRequest(fullRepoName, path string) string {
 	uri := fmt.Sprintf("repos/%s/commits", fullRepoName)
-	return rc.makeRequest(uri, Query{Path(path)}).URL()
+	vals := url.Values{
+		"path": []string{path},
+	}
+	return rc.makeRequest(uri, Query{}, vals).URL()
 }
 
-func (rc RequestConfig) makeRequest(path string, query Query) request {
-	vals := url.Values{}
+func (rc RequestConfig) makeRequest(path string, query Query, vals url.Values) request {
 	vals.Set(perPageArg, fmt.Sprint(rc.perPage))
 
 	return request{
@@ -151,7 +166,7 @@ type request struct {
 	query Query
 }
 
-// CopyWith copies the requests and adds the extra query parameters. Usefull
+// CopyWith copies the requests and adds the extra query parameters. It is useful
 // for dynamically adding sizes to a filename only query without modifying it.
 func (r request) CopyWith(queryParams ...queryField) request {
 	cpy := r

api/internal/crawl/crawler/github/queries_test.go

@@ -53,8 +53,11 @@ func TestQueryType(t *testing.T) {
 				Filename("kustomization.yaml"),
 				Keyword("keyword1"),
 				Keyword("keyword2"),
+				Repo("user1/repo1"),
+				User("user1"),
 			),
-			expected: "q=size:24..64+filename:kustomization.yaml+keyword1+keyword2",
+			expected: "q=size:24..64+filename:kustomization.yaml+keyword1+keyword2+" +
+				"repo:user1/repo1+user:user1",
 		},
 	}
 
@@ -98,7 +101,27 @@ func TestGithubSearchQuery(t *testing.T) {
 				"examples/helloWorld/kustomization.yaml?per_page=100",
 
 			expectedCommitsQuery: "https://api.github.com/repos/kubernetes-sigs/kustomize/commits?" +
-				"q=path:examples/helloWorld/kustomization.yaml&per_page=100",
+				"path=examples%2FhelloWorld%2Fkustomization.yaml&per_page=100",
+		},
+		{
+			rc: RequestConfig{
+				perPage: perPage,
+			},
+			codeQuery: Query{
+				Filename("kustomization.yaml"),
+				Filesize(RangeWithin{64, 128}),
+			},
+			fullRepoName: "kubernetes-sigs/kustomize",
+			path:         "examples 1/helloWorld/kustomization.yaml",
+
+			expectedCodeQuery: "https://api.github.com/search/code?" +
+				"q=filename:kustomization.yaml+size:64..128&order=desc&per_page=100&sort=indexed",
+
+			expectedContentsQuery: "https://api.github.com/repos/kubernetes-sigs/kustomize/contents/" +
+				"examples%201/helloWorld/kustomization.yaml?per_page=100",
+
+			expectedCommitsQuery: "https://api.github.com/repos/kubernetes-sigs/kustomize/commits?" +
+				"path=examples+1%2FhelloWorld%2Fkustomization.yaml&per_page=100",
 		},
 	}
 

api/internal/crawl/crawler/github/split_search_ranges.go

@@ -93,13 +93,15 @@ package github
 //   apiCallsPerResult * 10(pages) * 100(resultsPerPage) * totalResults / 1000
 //   = apiCallsPerResult * totalResults.
 //
-// So it could very well take apiCallsPerResult * 50 times longer to acutally
+// So it could very well take apiCallsPerResult * 50 times longer to actually
 // fetch the results (assuming the quotas for the API calls are the same as the
 // search API), than it does to perform these range searches.
 
 import (
 	"fmt"
 	"math/bits"
+	"strconv"
+	"strings"
 )
 
 // Files cannot be more than 2^19 bytes, according to
@@ -112,7 +114,7 @@ const (
 // Interface instead of struct for testing purposes.
 // Not expecting to have multiple implementations.
 type cachedSearch interface {
-	CountResults(uint64) (uint64, error)
+	CountResults(uint64, uint64) (uint64, error)
 	RequestString(filesize rangeFormatter) string
 }
 
@@ -139,7 +141,7 @@ type cachedSearch interface {
 //    problematic). The current cache implementation looks at the
 //    predecessor entry to find out if the current value is monotonic.
 //    This is where the bit trick is used, since each step in the binary
-//    search is adding or ommiting to add a decreasing power of 2 to the query
+//    search is adding or omitting to add a decreasing power of 2 to the query
 //    value, we can remove the least significant set bit to find the
 //    predecessor in constant time. Ultimately since the search is rate
 //    limited, we could also easily afford to compute this in linear time
@@ -161,16 +163,16 @@ func newCache(client GhClient, query Query) githubCachedSearch {
 	}
 }
 
-func (c githubCachedSearch) CountResults(upperBound uint64) (uint64, error) {
+func (c githubCachedSearch) CountResults(lowerBound, upperBound uint64) (uint64, error) {
 	count, cached := c.cache[upperBound]
 	if cached {
 		return count, nil
 	}
 
-	sizeRange := RangeWithin{0, upperBound}
+	sizeRange := RangeWithin{lowerBound, upperBound}
 	rangeRequest := c.RequestString(sizeRange)
 
-	result := c.gcl.parseGithubResponse(rangeRequest)
+	result := c.gcl.parseGithubResponseWithRetry(rangeRequest)
 	if result.Error != nil {
 		return count, result.Error
 	}
@@ -204,7 +206,7 @@ func (c githubCachedSearch) CountResults(upperBound uint64) (uint64, error) {
 			"Retrying query... current lower bound: %d, got: %d\n",
 			c.cache[prev], result.Parsed.TotalCount)
 
-		result = c.gcl.parseGithubResponse(rangeRequest)
+		result = c.gcl.parseGithubResponseWithRetry(rangeRequest)
 		if result.Error != nil {
 			return count, result.Error
 		}
@@ -219,8 +221,8 @@ func (c githubCachedSearch) CountResults(upperBound uint64) (uint64, error) {
 	}
 
 	count = result.Parsed.TotalCount
-	logger.Printf("Caching new query %s, with count %d\n",
-		sizeRange.RangeString(), count)
+	logger.Printf("Caching new query %s, with count %d (incomplete_results: %v)\n",
+		sizeRange.RangeString(), count, result.Parsed.IncompleteResults)
 	c.cache[upperBound] = count
 	return count, nil
 }
@@ -238,16 +240,16 @@ func (c githubCachedSearch) RequestString(filesize rangeFormatter) string {
 // This would mean that the search as it is could not find all files. If queries
 // are sorted by last indexed, and retrieved on regular intervals, it should be
 // sufficient to get most if not all documents.
-func FindRangesForRepoSearch(cache cachedSearch) ([]string, error) {
-	totalFiles, err := cache.CountResults(githubMaxFileSize)
+func FindRangesForRepoSearch(cache cachedSearch, lowerBound, upperBound uint64) ([]string, error) {
+	totalFiles, err := cache.CountResults(lowerBound, upperBound)
 	if err != nil {
 		return nil, err
 	}
-	logger.Println("total files: ", totalFiles)
+	logger.Println("total kustomization files: ", totalFiles)
 
 	if githubMaxResultsPerQuery >= totalFiles {
 		return []string{
-			cache.RequestString(RangeWithin{0, githubMaxFileSize}),
+			cache.RequestString(RangeWithin{lowerBound, upperBound}),
 		}, nil
 	}
 
@@ -275,6 +277,7 @@ func FindRangesForRepoSearch(cache cachedSearch) ([]string, error) {
 	// range.
 	filesAccessible := uint64(0)
 	sizes := make([]uint64, 0)
+	sizes = append(sizes, lowerBound)
 	for filesAccessible < totalFiles {
 		target := filesAccessible + githubMaxResultsPerQuery
 		if target >= totalFiles {
@@ -284,22 +287,22 @@ func FindRangesForRepoSearch(cache cachedSearch) ([]string, error) {
 		logger.Printf("%d accessible files, next target = %d\n",
 			filesAccessible, target)
 
-		cur, err := lowerBoundFileCount(cache, target)
+		size, err := FindFileSize(cache, target, lowerBound, upperBound)
 		if err != nil {
 			return nil, err
 		}
 
 		// If there are more than 1000 files in the next bucket, we must
 		// advance anyway and lose out on some files :(.
-		if l := len(sizes); l > 0 && sizes[l-1] == cur {
-			cur++
+		if l := len(sizes); l > 0 && sizes[l-1] == size {
+			size++
 		}
 
-		nextAccessible, err := cache.CountResults(cur)
+		nextAccessible, err := cache.CountResults(lowerBound, size)
 		if err != nil {
 			return nil, fmt.Errorf(
 				"cache should be populated at %d already, got %v",
-				cur, err)
+				size, err)
 		}
 		if nextAccessible < filesAccessible {
 			return nil, fmt.Errorf(
@@ -309,31 +312,31 @@ func FindRangesForRepoSearch(cache cachedSearch) ([]string, error) {
 
 		filesAccessible = nextAccessible
 		if nextAccessible < totalFiles {
-			sizes = append(sizes, cur)
+			sizes = append(sizes, size)
 		}
 	}
-
+	sizes = append(sizes, upperBound)
 	return formatFilesizeRanges(cache, sizes), nil
 }
 
-// lowerBoundFileCount finds the filesize range from [0, return value] that has
+// FindFileSize finds the filesize range from [lowerBound, return value] that has
 // the largest file count that is smaller than or equal to
 // githubMaxResultsPerQuery. It is important to note that this returned value
 // could already be in a previous range if the next file size has more than 1000
 // results. It is left to the caller to handle this bit of logic and guarantee
 // forward progession in this case.
-func lowerBoundFileCount(
-	cache cachedSearch, targetFileCount uint64) (uint64, error) {
+func FindFileSize(
+	cache cachedSearch, targetFileCount, lowerBound, upperBound uint64) (uint64, error) {
 
 	// Binary search for file sizes that make up the next <=1000 element
 	// chunk.
-	cur := uint64(0)
-	increase := githubMaxFileSize / 2
+	cur := lowerBound
+	increase := (upperBound - lowerBound) / 2
 
 	for increase > 0 {
 		mid := cur + increase
 
-		count, err := cache.CountResults(mid)
+		count, err := cache.CountResults(lowerBound, mid)
 		if err != nil {
 			return count, err
 		}
@@ -353,26 +356,24 @@ func lowerBoundFileCount(
 }
 
 func formatFilesizeRanges(cache cachedSearch, sizes []uint64) []string {
-	ranges := make([]string, 0, len(sizes)+1)
-
-	if len(sizes) > 0 {
-		ranges = append(ranges, cache.RequestString(
-			RangeLessThan{sizes[0] + 1},
-		))
+	n := len(sizes)
+	if n < 2 {
+		return []string{}
 	}
 
-	for i := 0; i < len(sizes)-1; i += 1 {
-		ranges = append(ranges, cache.RequestString(
-			RangeWithin{sizes[i] + 1, sizes[i+1]},
-		))
-
-		if i != len(sizes)-2 {
-			continue
-		}
-		ranges = append(ranges, cache.RequestString(
-			RangeGreaterThan{sizes[i+1]},
-		))
+	ranges := make([]string, 0, n-1)
+	ranges = append(ranges, cache.RequestString(RangeWithin{sizes[0], sizes[1]}))
+	for i := 1; i < n-1; i++ {
+		ranges = append(ranges, cache.RequestString(RangeWithin{sizes[i] + 1, sizes[i+1]}))
 	}
-
 	return ranges
 }
+
+func RangeSizes(s string) RangeWithin {
+	start := strings.Index(s, "+size:") + len("+size:")
+	end := strings.Index(s, "&")
+	ranges := strings.Split(s[start:end], "..")
+	lowerBound, _ := strconv.ParseUint(ranges[0], 10, 64)
+	upperBound, _ := strconv.ParseUint(ranges[1], 10, 64)
+	return RangeWithin{lowerBound, upperBound}
+}

api/internal/crawl/crawler/github/split_search_ranges_test.go

@@ -2,6 +2,7 @@ package github
 
 import (
 	"fmt"
+	"log"
 	"reflect"
 	"testing"
 )
@@ -10,8 +11,8 @@ type testCachedSearch struct {
 	cache map[uint64]uint64
 }
 
-func (c testCachedSearch) CountResults(upperBound uint64) (uint64, error) {
-	fmt.Printf("CountResults(%05x)\n", upperBound)
+func (c testCachedSearch) CountResults(lowerBound, upperBound uint64) (uint64, error) {
+	log.Printf("CountResults(%05x)\n", upperBound)
 	count, ok := c.cache[upperBound]
 	if !ok {
 		return count, fmt.Errorf("cache not set at %x", upperBound)
@@ -72,19 +73,29 @@ func TestRangeSplitting(t *testing.T) {
 		},
 	}
 
-	requests, err := FindRangesForRepoSearch(cache)
+	requests, err := FindRangesForRepoSearch(cache, 0, 524288)
 	if err != nil {
 		t.Errorf("Error while finding ranges: %v", err)
 	}
 	expected := []string{
-		"<107",      // cache.RequestString(RangeLessThan{0x6b}),
-		"107..128",  // cache.RequestString(RangeWithin{0x6b, 0x80}),
-		"129..256",  // cache.RequestString(RangeWithin{0x81, 0x100}),
-		"257..4095", // cache.RequestString(RangeWithin{0x101, 0xfff}),
-		">4095",     // cache.RequestString(RangeGreaterThan{0xfff}),
+		"0..106",       // cache.RequestString(RangeWithin{0x00, 0x6a}),
+		"107..128",     // cache.RequestString(RangeWithin{0x6b, 0x80}),
+		"129..256",     // cache.RequestString(RangeWithin{0x81, 0x100}),
+		"257..4095",    // cache.RequestString(RangeWithin{0x101, 0xfff}),
+		"4096..524288", // cache.RequestString(RangeWithin{0x1000, 0x80000}),
 	}
 
 	if !reflect.DeepEqual(requests, expected) {
 		t.Errorf("Expected requests (%v) to equal (%v)", requests, expected)
 	}
 }
+
+func TestRangeSizes(t *testing.T) {
+	s := "https://api.github.com/search/code?q=filename:kustomization.yaml+filename:kustomization.yml" +
+		"+filename:kustomization+size:2365..10000&order=desc&per_page=100&sort=indexed"
+	returnedResult := RangeSizes(s)
+	expectedResult := RangeWithin{uint64(2365), uint64(10000)}
+	if !reflect.DeepEqual(returnedResult, expectedResult) {
+		t.Errorf("RangeSizes expected (%v), got (%v)",expectedResult, returnedResult)
+	}
+}

api/internal/crawl/doc/doc.go

@@ -2,6 +2,9 @@ package doc
 
 import (
 	"fmt"
+	"log"
+	"path/filepath"
+	"sort"
 	"strings"
 
 	"sigs.k8s.io/kustomize/api/k8sdeps/kunstruct"
@@ -43,21 +46,36 @@ type KustomizationDocument struct {
 
 type set map[string]struct{}
 
+func (doc *KustomizationDocument) Copy() *KustomizationDocument {
+	return &KustomizationDocument{
+		Document:    *(doc.Document.Copy()),
+		Kinds:       doc.Kinds,
+		Identifiers: doc.Identifiers,
+		Values:      doc.Values,
+	}
+}
+
 func (doc *KustomizationDocument) String() string {
 	return fmt.Sprintf("%s %s %s %v %v %v len(identifiers):%v  len(values):%v",
 		doc.RepositoryURL, doc.FilePath, doc.DefaultBranch, doc.CreationTime,
-		doc.IsSame,	doc.Kinds, len(doc.Identifiers), len(doc.Values))
+		doc.IsSame, doc.Kinds, len(doc.Identifiers), len(doc.Values))
+}
+
+// IsKustomizationFile determines whether a file path is a kustomization file
+func IsKustomizationFile(path string) bool {
+	basename := filepath.Base(path)
+	for _, name := range konfig.RecognizedKustomizationFileNames() {
+		if basename == name {
+			return true
+		}
+	}
+	return false
 }
 
 // Implements the CrawlerDocument interface.
-func (doc *KustomizationDocument) GetResources() ([]*Document, error) {
-	isResource := true
-	for _, suffix := range konfig.RecognizedKustomizationFileNames() {
-		if strings.HasSuffix(doc.FilePath, "/"+suffix) {
-			isResource = false
-		}
-	}
-	if isResource {
+func (doc *KustomizationDocument) GetResources(
+	includeResources, includeTransformers, includeGenerators bool) ([]*Document, error) {
+	if !IsKustomizationFile(doc.FilePath) {
 		return []*Document{}, nil
 	}
 
@@ -75,19 +93,46 @@ func (doc *KustomizationDocument) GetResources() ([]*Document, error) {
 	}
 	k.FixKustomizationPostUnmarshalling()
 
-	res := make([]*Document, 0, len(k.Resources))
-	for _, r := range k.Resources {
-		next, err := doc.Document.FromRelativePath(r)
-		if err != nil {
-			fmt.Printf("GetResources error: %v\n", err)
-			continue
-		}
-		res = append(res, &next)
+	res := make([]*Document, 0)
+
+	if includeResources {
+		resourceDocs := doc.CollectDocuments(k.Resources, "resource")
+		res = append(res, resourceDocs...)
+	}
+
+	if includeGenerators {
+		generatorDocs := doc.CollectDocuments(k.Generators, "generator")
+		res = append(res, generatorDocs...)
+	}
+
+	if includeTransformers {
+		transformerDocs := doc.CollectDocuments(k.Transformers, "transformer")
+		res = append(res, transformerDocs...)
 	}
 
 	return res, nil
 }
 
+// CollectDocuments construct a Document for each path in paths, and return
+// a slice of Document pointers.
+func (doc *KustomizationDocument) CollectDocuments(
+	paths []string, fileType string) []*Document {
+	docs := make([]*Document, 0, len(paths))
+	for _, r := range paths {
+		if strings.TrimSpace(r) == "" {
+			continue
+		}
+		next, err := doc.Document.FromRelativePath(r)
+		if err != nil {
+			log.Printf("CollectDocuments error: %v\n", err)
+			continue
+		}
+		next.FileType = fileType
+		docs = append(docs, &next)
+	}
+	return docs
+}
+
 func (doc *KustomizationDocument) readBytes() ([]map[string]interface{}, error) {
 	data := []byte(doc.DocumentData)
 
@@ -116,6 +161,8 @@ func (doc *KustomizationDocument) readBytes() ([]map[string]interface{}, error)
 	return configs, nil
 }
 
+// ParseYAML parses doc.Document and sets the following fields of doc:
+// Kinds, Values, Identifiers.
 func (doc *KustomizationDocument) ParseYAML() error {
 	doc.Identifiers = make([]string, 0)
 	doc.Values = make([]string, 0)
@@ -159,6 +206,13 @@ func (doc *KustomizationDocument) ParseYAML() error {
 		doc.Identifiers = append(doc.Identifiers, key)
 	}
 
+	// Without sorting these fields, every time when the string order in these fields changes,
+	// the document in the index will be updated.
+	// Sorting these fields are necessary to avoid a document being updated unnecessarily.
+	sort.Strings(doc.Kinds)
+	sort.Strings(doc.Values)
+	sort.Strings(doc.Identifiers)
+
 	return nil
 }
 

api/internal/crawl/doc/doc_test.go

@@ -189,11 +189,13 @@ metadata:
 	}
 }
 
+type TestStructForGetResources struct {
+	doc       KustomizationDocument
+	resources []*Document
+}
+
 func TestGetResources(t *testing.T) {
-	tests := []struct {
-		doc       KustomizationDocument
-		resources []*Document
-	}{
+	tests := []TestStructForGetResources{
 		{
 			doc: KustomizationDocument{
 				Document: Document{
@@ -213,19 +215,27 @@ resources:
 				{
 					RepositoryURL: "sigs.k8s.io/kustomize",
 					FilePath:      "some/path/to/base",
+					FileType:      "resource",
+					User:          "sigs.k8s.io",
 				},
 				{
 					RepositoryURL: "sigs.k8s.io/kustomize",
 					FilePath:      "some/path/to/otherbase",
+					FileType:      "resource",
+					User:          "sigs.k8s.io",
 				},
 				{
 					RepositoryURL: "sigs.k8s.io/kustomize",
 					FilePath:      "some/path/to/kdir/file.yaml",
+					FileType:      "resource",
+					User:          "sigs.k8s.io",
 				},
 				{
 					RepositoryURL: "https://github.com/kubernetes-sigs/kustomize",
 					FilePath:      "examples/helloWorld",
 					DefaultBranch: "v3.1.0",
+					FileType:      "resource",
+					User:          "kubernetes-sigs",
 				},
 			},
 		},
@@ -248,9 +258,12 @@ resources:
 			resources: []*Document{},
 		},
 	}
+	runTest(t, tests, true, false, false)
+}
 
+func runTest(t *testing.T, tests []TestStructForGetResources, includeResources, includeTransformers, includeGenerators bool) {
 	for _, test := range tests {
-		res, err := test.doc.GetResources()
+		res, err := test.doc.GetResources(includeResources, includeTransformers, includeGenerators)
 		if err != nil {
 			t.Errorf("Unexpected error: %v\n", err)
 			continue
@@ -284,3 +297,83 @@ resources:
 		}
 	}
 }
+
+func TestGetResourcesAndGenerators(t *testing.T) {
+	tests := []TestStructForGetResources{
+		{
+			doc: KustomizationDocument{
+				Document: Document{
+					RepositoryURL: "sigs.k8s.io/kustomize",
+					FilePath:      "some/path/to/kdir/kustomization.yaml",
+					DocumentData: `
+resources:
+- file.yaml
+
+generators:
+- gen.yaml
+
+transformers:
+- tr.yaml
+`},
+			},
+			resources: []*Document{
+				{
+					RepositoryURL: "sigs.k8s.io/kustomize",
+					FilePath:      "some/path/to/kdir/gen.yaml",
+					FileType:      "generator",
+					User:          "sigs.k8s.io",
+				},
+				{
+					RepositoryURL: "sigs.k8s.io/kustomize",
+					FilePath:      "some/path/to/kdir/file.yaml",
+					FileType:      "resource",
+					User:          "sigs.k8s.io",
+				},
+			},
+		},
+	}
+	runTest(t, tests, true, false, true)
+}
+
+func TestGetResourcesAndGeneratorsAndTransformers(t *testing.T) {
+	tests := []TestStructForGetResources{
+		{
+			doc: KustomizationDocument{
+				Document: Document{
+					RepositoryURL: "sigs.k8s.io/kustomize",
+					FilePath:      "some/path/to/kdir/kustomization.yaml",
+					DocumentData: `
+resources:
+- file.yaml
+
+generators:
+- gen.yaml
+
+transformers:
+- tr.yaml
+`},
+			},
+			resources: []*Document{
+				{
+					RepositoryURL: "sigs.k8s.io/kustomize",
+					FilePath:      "some/path/to/kdir/tr.yaml",
+					FileType:      "transformer",
+					User:          "sigs.k8s.io",
+				},
+				{
+					RepositoryURL: "sigs.k8s.io/kustomize",
+					FilePath:      "some/path/to/kdir/gen.yaml",
+					FileType:      "generator",
+					User:          "sigs.k8s.io",
+				},
+				{
+					RepositoryURL: "sigs.k8s.io/kustomize",
+					FilePath:      "some/path/to/kdir/file.yaml",
+					FileType:      "resource",
+					User:          "sigs.k8s.io",
+				},
+			},
+		},
+	}
+	runTest(t, tests, true, true, true)
+}