Add lint-kustomize to prow-presubmit-check

To do tool installs.

.gitignore

@@ -4,7 +4,9 @@
 *.dll
 *.so
 *.dylib
-kustomize
+
+.idea
+*.iml
 
 # Test binary, build with `go test -c`
 *.test

.golangci-kustomize.yml

@@ -0,0 +1,50 @@
+run:
+  deadline: 5m
+
+linters:
+  disable-all: true
+  enable:
+    - bodyclose
+    - deadcode
+    - depguard
+    # - dogsled
+    - dupl
+    # - errcheck
+    # - funlen
+    # - gochecknoinits
+    - goconst
+    # - gocritic
+    - gocyclo
+    - gofmt
+    - goimports
+    - golint
+    - gosec
+    - gosimple
+    - govet
+    - ineffassign
+    - interfacer
+    - lll
+    - misspell
+    - nakedret
+    # - scopelint
+    - staticcheck
+    - structcheck
+    # stylecheck demands that acronyms not be treated as words
+    # in camelCase, so JsonOp become JSONOp, etc.  Yuck.
+    # - stylecheck
+    - typecheck
+    - unconvert
+    - unparam
+    - unused
+    - varcheck
+    # - whitespace
+
+linters-settings:
+  dupl:
+    threshold: 400
+  lll:
+    line-length: 170
+  gocyclo:
+    min-complexity: 15
+  golint:
+    min-confidence: 0.85

.golangci.yml

@@ -1,30 +0,0 @@
-run:
-  deadline: 5m
-
-linters:
-  disable-all: true
-  enable:
-    - dupl
-    - goconst
-    - gocyclo
-    - gofmt
-    - golint
-    - govet
-    - ineffassign
-    - interfacer
-    - lll
-    - misspell
-    - nakedret
-    - structcheck
-    - unparam
-    - varcheck
-
-linters-settings:
-  dupl:
-    threshold: 400 
-  lll:   
-    line-length: 170
-  gocyclo:
-    min-complexity: 15
-  golint:
-    min-confidence: 0.85

.travis.yml

@@ -1,34 +1,42 @@
-language: go
+os:
+  - linux
+  - osx
+# TODO: Speed up the slowness of the osx travis runs
+# Maybe cache brew installs?
+#
+# TODO: Uncomment when some gets the tests running on Windows.
+#  - windows
 
-go:
-  - 1.11.x
-
-go_import_path: sigs.k8s.io/kustomize
-
-# Maybe, maybe not.
-# sudo: false
+addons:
+  apt:
+    packages:
+    - tree
+  homebrew:
+    packages:
+    - tree
+    update: true
 
 # Only clone the most recent commit.
 git:
   depth: 1
 
-env:
-- GOLANGCI_RELEASE="v1.10.2"
+language: go
+
+go:
+  - "1.13"
+
+go_import_path: sigs.k8s.io/kustomize
 
 before_install:
-  - source ./bin/consider-early-travis-exit.sh
-  - sudo apt-get install tree
-  - curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | sh -s -- -b $GOPATH/bin ${GOLANGCI_RELEASE}
-  - go get -u github.com/monopole/mdrip
+  - source ./travis/consider-early-travis-exit.sh
 
-# Install must be set to prevent default `go get` to run.
-# The dependencies have already been vendored by `dep` so
-# we don't need to fetch them.
-install:
-  -
+# Skip the install process; let pre-commit.sh do it.
+install: true
 
 script:
-  - ./bin/pre-commit.sh
+  - make verify-kustomize
+  - ./travis/kyaml-pre-commit.sh
+  - ./travis/check-go-mod.sh
 
 # TBD. Suppressing for now.
 notifications:

CONTRIBUTING.md

@@ -6,11 +6,15 @@ _As contributors and maintainers of this project, and in the interest of fosteri
 
 ## Getting Started
 
+Dev guides:
+
+- [Mac](docs/macDevGuide.md)
+
 We have full documentation on how to get started contributing here:
 
 - [Contributor License Agreement](https://git.k8s.io/community/CLA.md) Kubernetes projects require that you sign a Contributor License Agreement (CLA) before we can accept your pull requests
 - [Kubernetes Contributor Guide](http://git.k8s.io/community/contributors/guide) - Main contributor documentation, or you can just jump directly to the [contributing section](http://git.k8s.io/community/contributors/guide#contributing)
-- [Contributor Cheat Sheet](https://git.k8s.io/community/contributors/guide/contributor-cheatsheet.md) - Common resources for existing developers
+- [Contributor Cheat Sheet](https://git.k8s.io/community/contributors/guide/contributor-cheatsheet/README.md) - Common resources for existing developers
 
 ## Mentorship
 

Gopkg.lock

@@ -1,388 +0,0 @@
-# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'.
-
-
-[[projects]]
-  digest = "1:d8ebbd207f3d3266d4423ce4860c9f3794956306ded6c7ba312ecc69cdfbf04c"
-  name = "github.com/PuerkitoBio/purell"
-  packages = ["."]
-  pruneopts = "NUT"
-  revision = "0bcb03f4b4d0a9428594752bd2a3b9aa0a9d4bd4"
-  version = "v1.1.0"
-
-[[projects]]
-  branch = "master"
-  digest = "1:8098cd40cd09879efbf12e33bcd51ead4a66006ac802cd563a66c4f3373b9727"
-  name = "github.com/PuerkitoBio/urlesc"
-  packages = ["."]
-  pruneopts = "NUT"
-  revision = "de5bf2ad457846296e2031421a34e2568e304e35"
-
-[[projects]]
-  digest = "1:a2c1d0e43bd3baaa071d1b9ed72c27d78169b2b269f71c105ac4ba34b1be4a39"
-  name = "github.com/davecgh/go-spew"
-  packages = ["spew"]
-  pruneopts = "NUT"
-  revision = "346938d642f2ec3594ed81d874461961cd0faa76"
-  version = "v1.1.0"
-
-[[projects]]
-  digest = "1:f8e6f07329067bc182633dcb19a3df53ce5d454b551e1b5a1cac2163748648d9"
-  name = "github.com/emicklei/go-restful"
-  packages = [
-    ".",
-    "log",
-  ]
-  pruneopts = "NUT"
-  revision = "3658237ded108b4134956c1b3050349d93e7b895"
-  version = "v2.7.1"
-
-[[projects]]
-  digest = "1:ad32dc29f37281bacb5dcedff17c9461dc1739dc8a5f63a71ab491c6e92edf8d"
-  name = "github.com/evanphx/json-patch"
-  packages = ["."]
-  pruneopts = "NUT"
-  revision = "afac545df32f2287a079e2dfb7ba2745a643747e"
-  version = "v3.0.0"
-
-[[projects]]
-  digest = "1:81466b4218bf6adddac2572a30ac733a9255919bc2f470b4827a317bd4ee1756"
-  name = "github.com/ghodss/yaml"
-  packages = ["."]
-  pruneopts = "NUT"
-  revision = "0ca9ea5df5451ffdf184b4428c902747c2c11cd7"
-  version = "v1.0.0"
-
-[[projects]]
-  branch = "master"
-  digest = "1:260f7ebefc63024c8dfe2c9f1a2935a89fa4213637a1f522f592f80c001cc441"
-  name = "github.com/go-openapi/jsonpointer"
-  packages = ["."]
-  pruneopts = "NUT"
-  revision = "3a0015ad55fa9873f41605d3e8f28cd279c32ab2"
-
-[[projects]]
-  branch = "master"
-  digest = "1:98abd61947ff5c7c6fcfec5473d02a4821ed3a2dd99a4fbfdb7925b0dd745546"
-  name = "github.com/go-openapi/jsonreference"
-  packages = ["."]
-  pruneopts = "NUT"
-  revision = "3fb327e6747da3043567ee86abd02bb6376b6be2"
-
-[[projects]]
-  branch = "master"
-  digest = "1:e95b560c49fb849a61957a5fb3346ce23b3f67426e00e01179e5396cabc9a12c"
-  name = "github.com/go-openapi/spec"
-  packages = ["."]
-  pruneopts = "NUT"
-  revision = "bcff419492eeeb01f76e77d2ebc714dc97b607f5"
-
-[[projects]]
-  branch = "master"
-  digest = "1:a610c604eb06f0be4b0fc667388b7a221155d77d7f9089f70ac142a4a9daf014"
-  name = "github.com/go-openapi/swag"
-  packages = ["."]
-  pruneopts = "NUT"
-  revision = "811b1089cde9dad18d4d0c2d09fbdbf28dbd27a5"
-
-[[projects]]
-  digest = "1:1b3dd24f14a5280710fc7a3aa2480b6e4d20fdfc905841de9a3aa2aa2f1d4ee9"
-  name = "github.com/gogo/protobuf"
-  packages = [
-    "proto",
-    "sortkeys",
-  ]
-  pruneopts = "NUT"
-  revision = "1adfc126b41513cc696b209667c8656ea7aac67c"
-  version = "v1.0.0"
-
-[[projects]]
-  branch = "master"
-  digest = "1:e2b86e41f3d669fc36b50d31d32d22c8ac656c75aa5ea89717ce7177e134ff2a"
-  name = "github.com/golang/glog"
-  packages = ["."]
-  pruneopts = "NUT"
-  revision = "23def4e6c14b4da8ac2ed8007337bc5eb5007998"
-
-[[projects]]
-  digest = "1:03e14cff610a8a58b774e36bd337fa979482be86aab01be81fb8bbd6d0f07fc8"
-  name = "github.com/golang/protobuf"
-  packages = [
-    "proto",
-    "ptypes",
-    "ptypes/any",
-    "ptypes/duration",
-    "ptypes/timestamp",
-  ]
-  pruneopts = "NUT"
-  revision = "b4deda0973fb4c70b50d226b1af49f3da59f5265"
-  version = "v1.1.0"
-
-[[projects]]
-  branch = "master"
-  digest = "1:52c5834e2bebac9030c97cc0798ac11c3aa8a39f098aeb419f142533da6cd3cc"
-  name = "github.com/google/gofuzz"
-  packages = ["."]
-  pruneopts = "NUT"
-  revision = "24818f796faf91cd76ec7bddd72458fbced7a6c1"
-
-[[projects]]
-  digest = "1:3d7c1446fc5c710351b246c0dc6700fae843ca27f5294d0bd9f68bab2a810c44"
-  name = "github.com/googleapis/gnostic"
-  packages = [
-    "OpenAPIv2",
-    "compiler",
-    "extensions",
-  ]
-  pruneopts = "NUT"
-  revision = "ee43cbb60db7bd22502942cccbc39059117352ab"
-  version = "v0.1.0"
-
-[[projects]]
-  digest = "1:406338ad39ab2e37b7f4452906442a3dbf0eb3379dd1f06aafb5c07e769a5fbb"
-  name = "github.com/inconshreveable/mousetrap"
-  packages = ["."]
-  pruneopts = "NUT"
-  revision = "76626ae9c91c4f2a10f34cad8ce83ea42c93bb75"
-  version = "v1.0"
-
-[[projects]]
-  digest = "1:42c47ace7ccb114261ef7e0d418d274921514ab50a3bf6bdb9e51c3dde8ce13d"
-  name = "github.com/json-iterator/go"
-  packages = ["."]
-  pruneopts = "NUT"
-  revision = "ca39e5af3ece67bbcda3d0f4f56a8e24d9f2dad4"
-  version = "1.1.3"
-
-[[projects]]
-  branch = "master"
-  digest = "1:ada518b8c338e10e0afa443d84671476d3bd1d926e13713938088e8ddbee1a3e"
-  name = "github.com/mailru/easyjson"
-  packages = [
-    "buffer",
-    "jlexer",
-    "jwriter",
-  ]
-  pruneopts = "NUT"
-  revision = "3fdea8d05856a0c8df22ed4bc71b3219245e4485"
-
-[[projects]]
-  digest = "1:2f42fa12d6911c7b7659738758631bec870b7e9b4c6be5444f963cdcfccc191f"
-  name = "github.com/modern-go/concurrent"
-  packages = ["."]
-  pruneopts = "NUT"
-  revision = "bacd9c7ef1dd9b15be4a9909b8ac7a4e313eec94"
-  version = "1.0.3"
-
-[[projects]]
-  digest = "1:314a5881fab303a80d6d2e35a77000f2224bb50f09ef63a9aa4c1f9eaef985d8"
-  name = "github.com/modern-go/reflect2"
-  packages = ["."]
-  pruneopts = "NUT"
-  revision = "1df9eeb2bb81f327b96228865c5687bc2194af3f"
-  version = "1.0.0"
-
-[[projects]]
-  digest = "1:5cf3f025cbee5951a4ee961de067c8a89fc95a5adabead774f82822efabab121"
-  name = "github.com/pkg/errors"
-  packages = ["."]
-  pruneopts = "NUT"
-  revision = "645ef00459ed84a119197bfb8d8205042c6df63d"
-  version = "v0.8.0"
-
-[[projects]]
-  digest = "1:0f156dbd01b40676bdcbc64e51535c09b50f83c9cca5faef3090f82f18bda3c2"
-  name = "github.com/spf13/cobra"
-  packages = ["."]
-  pruneopts = "NUT"
-  revision = "a1f051bc3eba734da4772d60e2d677f47cf93ef4"
-  version = "v0.0.2"
-
-[[projects]]
-  digest = "1:15e5c398fbd9d2c439b635a08ac161b13d04f0c2aa587fe256b65dc0c3efe8b7"
-  name = "github.com/spf13/pflag"
-  packages = ["."]
-  pruneopts = "NUT"
-  revision = "583c0c0531f06d5278b7d917446061adc344b5cd"
-  version = "v1.0.1"
-
-[[projects]]
-  branch = "master"
-  digest = "1:d1a6ebe75268a41b6fbb1d43947cf8688d8580423b7484fa5ae608beef6df24d"
-  name = "golang.org/x/net"
-  packages = [
-    "http2",
-    "http2/hpack",
-    "idna",
-    "lex/httplex",
-  ]
-  pruneopts = "NUT"
-  revision = "1c05540f6879653db88113bc4a2b70aec4bd491f"
-
-[[projects]]
-  digest = "1:e33513a825fcd765e97b5de639a2f7547542d1a8245df0cef18e1fd390b778a9"
-  name = "golang.org/x/text"
-  packages = [
-    "collate",
-    "collate/build",
-    "internal/colltab",
-    "internal/gen",
-    "internal/tag",
-    "internal/triegen",
-    "internal/ucd",
-    "language",
-    "secure/bidirule",
-    "transform",
-    "unicode/bidi",
-    "unicode/cldr",
-    "unicode/norm",
-    "unicode/rangetable",
-    "width",
-  ]
-  pruneopts = "NUT"
-  revision = "f21a4dfb5e38f5895301dc265a8def02365cc3d0"
-  version = "v0.3.0"
-
-[[projects]]
-  digest = "1:2d1fbdc6777e5408cabeb02bf336305e724b925ff4546ded0fa8715a7267922a"
-  name = "gopkg.in/inf.v0"
-  packages = ["."]
-  pruneopts = "NUT"
-  revision = "d2d2541c53f18d2a059457998ce2876cc8e67cbf"
-  version = "v0.9.1"
-
-[[projects]]
-  digest = "1:7c95b35057a0ff2e19f707173cc1a947fa43a6eb5c4d300d196ece0334046082"
-  name = "gopkg.in/yaml.v2"
-  packages = ["."]
-  pruneopts = "NUT"
-  revision = "5420a8b6744d3b0345ab293f6fcba19c978f1183"
-  version = "v2.2.1"
-
-[[projects]]
-  branch = "master"
-  digest = "1:d895c7c24a0dd1ed2ecd061fd88dfea9e1e84d6f280ed859942a2d1aabee10ec"
-  name = "k8s.io/api"
-  packages = [
-    "admissionregistration/v1alpha1",
-    "admissionregistration/v1beta1",
-    "apps/v1",
-    "apps/v1beta1",
-    "apps/v1beta2",
-    "authentication/v1",
-    "authentication/v1beta1",
-    "authorization/v1",
-    "authorization/v1beta1",
-    "autoscaling/v1",
-    "autoscaling/v2beta1",
-    "batch/v1",
-    "batch/v1beta1",
-    "batch/v2alpha1",
-    "certificates/v1beta1",
-    "core/v1",
-    "events/v1beta1",
-    "extensions/v1beta1",
-    "networking/v1",
-    "policy/v1beta1",
-    "rbac/v1",
-    "rbac/v1alpha1",
-    "rbac/v1beta1",
-    "scheduling/v1alpha1",
-    "settings/v1alpha1",
-    "storage/v1",
-    "storage/v1alpha1",
-    "storage/v1beta1",
-  ]
-  pruneopts = "NUT"
-  revision = "53d615ae3f440f957cb9989d989d597f047262d9"
-
-[[projects]]
-  branch = "master"
-  digest = "1:dff69dd9d9fc681ae077ce5a409aca3c24894d09102ab0395ca7972f6ec01811"
-  name = "k8s.io/apimachinery"
-  packages = [
-    "pkg/api/equality",
-    "pkg/api/meta",
-    "pkg/api/resource",
-    "pkg/api/validation",
-    "pkg/apis/meta/v1",
-    "pkg/apis/meta/v1/unstructured",
-    "pkg/apis/meta/v1/validation",
-    "pkg/apis/meta/v1beta1",
-    "pkg/conversion",
-    "pkg/conversion/queryparams",
-    "pkg/fields",
-    "pkg/labels",
-    "pkg/runtime",
-    "pkg/runtime/schema",
-    "pkg/runtime/serializer",
-    "pkg/runtime/serializer/json",
-    "pkg/runtime/serializer/protobuf",
-    "pkg/runtime/serializer/recognizer",
-    "pkg/runtime/serializer/versioning",
-    "pkg/selection",
-    "pkg/types",
-    "pkg/util/errors",
-    "pkg/util/framer",
-    "pkg/util/intstr",
-    "pkg/util/json",
-    "pkg/util/mergepatch",
-    "pkg/util/net",
-    "pkg/util/runtime",
-    "pkg/util/sets",
-    "pkg/util/strategicpatch",
-    "pkg/util/validation",
-    "pkg/util/validation/field",
-    "pkg/util/wait",
-    "pkg/util/yaml",
-    "pkg/watch",
-    "third_party/forked/golang/json",
-    "third_party/forked/golang/reflect",
-  ]
-  pruneopts = "NUT"
-  revision = "13b73596e4b63e03203e86f6d9c7bcc1b937c62f"
-
-[[projects]]
-  digest = "1:ae9ced9ef7b8eb2794a4f80bc3af9d2bc38ec7d60337367bad9a655c1d641458"
-  name = "k8s.io/client-go"
-  packages = ["kubernetes/scheme"]
-  pruneopts = "NUT"
-  revision = "23781f4d6632d88e869066eaebb743857aa1ef9b"
-  version = "v7.0.0"
-
-[[projects]]
-  branch = "master"
-  digest = "1:f4fb3421360af5c51070bfe0c1c7467f8809fa70e278e129f068f5106b5c8a65"
-  name = "k8s.io/kube-openapi"
-  packages = [
-    "pkg/common",
-    "pkg/util/proto",
-  ]
-  pruneopts = "NUT"
-  revision = "b3f03f55328800731ce03a164b80973014ecd455"
-
-[solve-meta]
-  analyzer-name = "dep"
-  analyzer-version = 1
-  input-imports = [
-    "github.com/evanphx/json-patch",
-    "github.com/ghodss/yaml",
-    "github.com/pkg/errors",
-    "github.com/spf13/cobra",
-    "gopkg.in/yaml.v2",
-    "k8s.io/api/core/v1",
-    "k8s.io/apimachinery/pkg/api/validation",
-    "k8s.io/apimachinery/pkg/apis/meta/v1",
-    "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured",
-    "k8s.io/apimachinery/pkg/apis/meta/v1/validation",
-    "k8s.io/apimachinery/pkg/runtime",
-    "k8s.io/apimachinery/pkg/runtime/schema",
-    "k8s.io/apimachinery/pkg/util/mergepatch",
-    "k8s.io/apimachinery/pkg/util/strategicpatch",
-    "k8s.io/apimachinery/pkg/util/validation",
-    "k8s.io/apimachinery/pkg/util/validation/field",
-    "k8s.io/apimachinery/pkg/util/yaml",
-    "k8s.io/client-go/kubernetes/scheme",
-    "k8s.io/kube-openapi/pkg/common",
-  ]
-  solver-name = "gps-cdcl"
-  solver-version = 1

Gopkg.toml

@@ -1,58 +0,0 @@
-# Gopkg.toml example
-#
-# Refer to https://github.com/golang/dep/blob/master/docs/Gopkg.toml.md
-# for detailed Gopkg.toml documentation.
-#
-# required = ["github.com/user/thing/cmd/thing"]
-# ignored = ["github.com/user/project/pkgX", "bitbucket.org/user/project/pkgA/pkgY"]
-#
-# [[constraint]]
-#   name = "github.com/user/project"
-#   version = "1.0.0"
-#
-# [[constraint]]
-#   name = "github.com/user/project2"
-#   branch = "dev"
-#   source = "github.com/myfork/project2"
-#
-# [[override]]
-#  name = "github.com/x/y"
-#  version = "2.4.0"
-
-# prune out unused content from vendor
-[prune]
-  go-tests = true
-  non-go = true
-  unused-packages = true
-
-[[constraint]]
-  name = "github.com/evanphx/json-patch"
-  version = "3.0.0"
-
-[[constraint]]
-  name = "github.com/ghodss/yaml"
-  version = "1.0.0"
-
-[[constraint]]
-  name = "github.com/spf13/cobra"
-  version = "0.0.2"
-
-[[constraint]]
-  branch = "master"
-  name = "k8s.io/api"
-
-[[constraint]]
-  branch = "master"
-  name = "k8s.io/apimachinery"
-
-[[constraint]]
-  name = "k8s.io/client-go"
-  version = "7.0.0"
-
-[[override]]
-  branch = "master"
-  name = "k8s.io/utils"
-
-[[override]]
-  branch = "master"
-  name = "github.com/go-openapi/spec"

Makefile

@@ -0,0 +1,285 @@
+# Copyright 2019 The Kubernetes Authors.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Makefile for kustomize CLI and API.
+
+MYGOBIN := $(shell go env GOPATH)/bin
+SHELL := /bin/bash
+export PATH := $(MYGOBIN):$(PATH)
+
+.PHONY: all
+all: verify-kustomize
+
+.PHONY: verify-kustomize
+verify-kustomize: \
+	lint-kustomize \
+	test-unit-kustomize-all \
+	test-examples-kustomize-against-HEAD \
+	test-examples-kustomize-against-latest
+
+# The following target referenced by a file in
+# https://github.com/kubernetes/test-infra/tree/master/config/jobs/kubernetes-sigs/kustomize
+.PHONY: prow-presubmit-check
+prow-presubmit-check: \
+	lint-kustomize \
+	test-unit-kustomize-all
+
+.PHONY: verify-kustomize-e2e
+verify-kustomize-e2e: test-examples-e2e-kustomize
+
+# Other builds in this repo might want a different linter version.
+# Without one Makefile to rule them all, the different makes
+# cannot assume that golanci-lint is at the version they want
+# since everything uses the same implicit GOPATH.
+# This installs in a temp dir to avoid overwriting someone else's
+# linter, then installs in MYGOBIN with a new name.
+# Version pinned by api/go.mod
+$(MYGOBIN)/golangci-lint-kustomize:
+	( \
+		set -e; \
+		export GOBIN=$$(mktemp -d) \
+		cd api; \
+		GO111MODULE=on go install github.com/golangci/golangci-lint/cmd/golangci-lint; \
+		mv $$GOBIN/golangci-lint $(MYGOBIN)/golangci-lint-kustomize \
+	)
+
+# Version pinned by api/go.mod
+$(MYGOBIN)/mdrip:
+	cd api; \
+	go install github.com/monopole/mdrip
+
+# Version pinned by api/go.mod
+$(MYGOBIN)/stringer:
+	cd api; \
+	go install golang.org/x/tools/cmd/stringer
+
+# Version pinned by api/go.mod
+$(MYGOBIN)/goimports:
+	cd api; \
+	go install golang.org/x/tools/cmd/goimports
+
+# Install resource from whatever is checked out.
+$(MYGOBIN)/resource:
+	cd cmd/resource; \
+	go install .
+
+# To pin pluginator, use this recipe instead:
+#   cd api;
+#   go install sigs.k8s.io/kustomize/pluginator/v2
+$(MYGOBIN)/pluginator:
+	cd pluginator; \
+	go install .
+
+# Install kustomize from whatever is checked out.
+$(MYGOBIN)/kustomize:
+	cd kustomize; \
+	go install .
+
+.PHONY: install-tools
+install-tools: \
+	$(MYGOBIN)/goimports \
+	$(MYGOBIN)/golangci-lint-kustomize \
+	$(MYGOBIN)/mdrip \
+	$(MYGOBIN)/pluginator \
+	$(MYGOBIN)/stringer
+
+### Begin kustomize plugin rules.
+#
+# The rules to deal with builtin plugins are a bit
+# complicated because
+#
+# - Every builtin plugin is a Go plugin -
+#   meaning it gets its own module directory
+#   (outside of the api module) with Go
+#   code in a 'main' package per Go plugin rules.
+# - kustomize locates plugins using the
+#   'apiVersion' and 'kind' fields from the
+#   plugin config file.
+# - k8s wants CamelCase in 'kind' fields.
+# - The module name (the last name in the path)
+#   must be the lowercased 'kind' of the
+#   plugin because Go and related tools
+#   demand lowercase in import paths, but
+#   allow CamelCase in file names.
+# - the generated code must live in the api
+#   module (it's linked into the api).
+
+# Where all generated builtin plugin code should go.
+pGen=api/builtins
+# Where the builtin Go plugin modules live.
+pSrc=plugin/builtin
+
+_builtinplugins = \
+	AnnotationsTransformer.go \
+	ConfigMapGenerator.go \
+	HashTransformer.go \
+	ImageTagTransformer.go \
+	InventoryTransformer.go \
+	LabelTransformer.go \
+	LegacyOrderTransformer.go \
+	NamespaceTransformer.go \
+	PatchJson6902Transformer.go \
+	PatchStrategicMergeTransformer.go \
+	PatchTransformer.go \
+	PrefixSuffixTransformer.go \
+	ReplicaCountTransformer.go \
+	SecretGenerator.go
+
+# Maintaining this explicit list of generated files, and
+# adding it as a dependency to a few targets, to assure
+# they get recreated if deleted.  The rules below on how
+# to make them don't, by themselves, assure they will be
+# recreated if deleted.
+builtinplugins = $(patsubst %,$(pGen)/%,$(_builtinplugins))
+
+# These rules are verbose, but assure that if a source file
+# is modified, the corresponding generated file, and only
+# that file, will be recreated.
+$(pGen)/AnnotationsTransformer.go: $(pSrc)/annotationstransformer/AnnotationsTransformer.go
+$(pGen)/ConfigMapGenerator.go: $(pSrc)/configmapgenerator/ConfigMapGenerator.go
+$(pGen)/HashTransformer.go: $(pSrc)/hashtransformer/HashTransformer.go
+$(pGen)/ImageTagTransformer.go: $(pSrc)/imagetagtransformer/ImageTagTransformer.go
+$(pGen)/InventoryTransformer.go: $(pSrc)/inventorytransformer/InventoryTransformer.go
+$(pGen)/LabelTransformer.go: $(pSrc)/labeltransformer/LabelTransformer.go
+$(pGen)/LegacyOrderTransformer.go: $(pSrc)/legacyordertransformer/LegacyOrderTransformer.go
+$(pGen)/NamespaceTransformer.go: $(pSrc)/namespacetransformer/NamespaceTransformer.go
+$(pGen)/PatchJson6902Transformer.go: $(pSrc)/patchjson6902transformer/PatchJson6902Transformer.go
+$(pGen)/PatchStrategicMergeTransformer.go: $(pSrc)/patchstrategicmergetransformer/PatchStrategicMergeTransformer.go
+$(pGen)/PatchTransformer.go: $(pSrc)/patchtransformer/PatchTransformer.go
+$(pGen)/PrefixSuffixTransformer.go: $(pSrc)/prefixsuffixtransformer/PrefixSuffixTransformer.go
+$(pGen)/ReplicaCountTransformer.go: $(pSrc)/replicacounttransformer/ReplicaCountTransformer.go
+$(pGen)/SecretGenerator.go: $(pSrc)/secretgenerator/SecretGenerator.go
+
+# The (verbose but portable) Makefile way to convert to lowercase.
+toLowerCase = $(subst A,a,$(subst B,b,$(subst C,c,$(subst D,d,$(subst E,e,$(subst F,f,$(subst G,g,$(subst H,h,$(subst I,i,$(subst J,j,$(subst K,k,$(subst L,l,$(subst M,m,$(subst N,n,$(subst O,o,$(subst P,p,$(subst Q,q,$(subst R,r,$(subst S,s,$(subst T,t,$(subst U,u,$(subst V,v,$(subst W,w,$(subst X,x,$(subst Y,y,$(subst Z,z,$1))))))))))))))))))))))))))
+
+$(pGen)/%.go: $(MYGOBIN)/pluginator
+	@echo "generating $*"
+	( \
+		set -e; \
+		cd $(pSrc)/$(call toLowerCase,$*); \
+		go generate .; \
+		cd ../../../$(pGen); \
+		$(MYGOBIN)/goimports -w $*.go \
+	)
+
+# Target is for debugging.
+.PHONY: generate-kustomize-builtin-plugins
+generate-kustomize-builtin-plugins: $(builtinplugins)
+
+.PHONY: kustomize-external-go-plugin-build
+kustomize-external-go-plugin-build:
+	./hack/buildExternalGoPlugins.sh ./plugin
+
+.PHONY: kustomize-external-go-plugin-clean
+kustomize-external-go-plugin-clean:
+	./hack/buildExternalGoPlugins.sh ./plugin clean
+
+### End kustomize plugin rules.
+
+.PHONY: lint-kustomize
+lint-kustomize: install-tools $(builtinplugins)
+	cd api; \
+	$(MYGOBIN)/golangci-lint-kustomize -c ../.golangci-kustomize.yml run ./...
+	cd kustomize; \
+	$(MYGOBIN)/golangci-lint-kustomize -c ../.golangci-kustomize.yml run ./...
+	cd pluginator; \
+	$(MYGOBIN)/golangci-lint-kustomize -c ../.golangci-kustomize.yml run ./...
+
+.PHONY: test-unit-kustomize-api
+test-unit-kustomize-api: $(builtinplugins)
+	cd api; go test ./...
+
+.PHONY: test-unit-kustomize-plugins
+test-unit-kustomize-plugins:
+	./hack/testUnitKustomizePlugins.sh
+
+.PHONY: test-unit-kustomize-cli
+test-unit-kustomize-cli:
+	cd kustomize; go test ./...
+
+.PHONY: test-unit-kustomize-all
+test-unit-kustomize-all: \
+	test-unit-kustomize-api \
+	test-unit-kustomize-cli \
+	test-unit-kustomize-plugins
+
+.PHONY:
+test-examples-kustomize-against-HEAD: $(MYGOBIN)/kustomize $(MYGOBIN)/mdrip
+	./hack/testExamplesAgainstKustomize.sh HEAD
+
+.PHONY:
+test-examples-e2e-kustomize: $(MYGOBIN)/mdrip $(MYGOBIN)/kind
+	( \
+		set -e; \
+		/bin/rm -f $(MYGOBIN)/kustomize; \
+		echo "Installing kustomize from ."; \
+		cd kustomize; go install .; cd ..; \
+		./hack/testExamplesE2EAgainstKustomize.sh .; \
+	)
+
+.PHONY:
+test-examples-kustomize-against-latest: $(MYGOBIN)/mdrip
+	( \
+		set -e; \
+		/bin/rm -f $(MYGOBIN)/kustomize; \
+		echo "Installing kustomize from latest."; \
+		GO111MODULE=on go install sigs.k8s.io/kustomize/kustomize/v3; \
+		./hack/testExamplesAgainstKustomize.sh latest; \
+		echo "Reinstalling kustomize from HEAD."; \
+		cd kustomize; go install .; \
+	)
+
+# linux only.
+# This is for testing an example plugin that
+# uses kubeval for validation.
+# Don't want to add a hard dependence in go.mod file
+# to github.com/instrumenta/kubeval.
+# Instead, download the binary.
+$(MYGOBIN)/kubeval:
+	( \
+		set -e; \
+		d=$(shell mktemp -d); cd $$d; \
+		wget https://github.com/instrumenta/kubeval/releases/latest/download/kubeval-linux-amd64.tar.gz; \
+		tar xf kubeval-linux-amd64.tar.gz; \
+		mv kubeval $(MYGOBIN); \
+		rm -rf $$d; \
+	)
+
+# linux only.
+# This is for testing an example plugin that
+# uses helm to inflate a chart for subsequent kustomization.
+# Don't want to add a hard dependence in go.mod file
+# to helm.
+# Instead, download the binary.
+$(MYGOBIN)/helm:
+	( \
+		set -e; \
+		d=$(shell mktemp -d); cd $$d; \
+		wget https://storage.googleapis.com/kubernetes-helm/helm-v2.13.1-linux-amd64.tar.gz; \
+		tar -xvzf helm-v2.13.1-linux-amd64.tar.gz; \
+		mv linux-amd64/helm $(MYGOBIN); \
+		rm -rf $$d \
+	)
+
+$(MYGOBIN)/kind:
+	( \
+        set -e; \
+        d=$(shell mktemp -d); cd $$d; \
+        wget -O ./kind https://github.com/kubernetes-sigs/kind/releases/download/v0.7.0/kind-$(shell uname)-amd64; \
+        chmod +x ./kind; \
+        mv ./kind $(MYGOBIN); \
+        rm -rf $$d; \
+	)
+
+.PHONY: clean
+clean: kustomize-external-go-plugin-clean
+	go clean --cache
+	rm -f $(builtinplugins)
+	rm -f $(MYGOBIN)/pluginator
+	rm -f $(MYGOBIN)/kustomize
+	rm -f $(MYGOBIN)/golangci-lint-kustomize
+
+.PHONY: nuke
+nuke: clean
+	sudo rm -rf $(shell go env GOPATH)/pkg/mod/sigs.k8s.io

README.md

@@ -7,7 +7,7 @@ untouched and usable as is.
 `kustomize` targets kubernetes; it understands and can
 patch [kubernetes style] API objects.  It's like
 [`make`], in that what it does is declared in a file,
-and it's like [`sed`], in that it emits editted text.
+and it's like [`sed`], in that it emits edited text.
 
 This tool is sponsored by [sig-cli] ([KEP]), and
 inspired by [DAM].
@@ -16,9 +16,23 @@ inspired by [DAM].
 [![Build Status](https://travis-ci.org/kubernetes-sigs/kustomize.svg?branch=master)](https://travis-ci.org/kubernetes-sigs/kustomize)
 [![Go Report Card](https://goreportcard.com/badge/github.com/kubernetes-sigs/kustomize)](https://goreportcard.com/report/github.com/kubernetes-sigs/kustomize)
 
-**Installation**: Download a binary from the [release
-page], or see these [install] notes. Then try one of
-the tested [examples].
+Download a binary from the [release page], or see
+these [instructions](docs/INSTALL.md).
+
+Browse the [docs](docs) or jump right into the
+tested [examples](examples).
+
+## kubectl integration
+
+Since [v1.14][kubectl announcement] the kustomize build system has been included in kubectl.
+
+| kubectl version | kustomize version |
+|---------|--------|
+| v1.16.0 | [v2.0.3](/../../tree/v2.0.3) |
+| v1.15.x | [v2.0.3](/../../tree/v2.0.3) |
+| v1.14.x | [v2.0.3](/../../tree/v2.0.3) |
+
+For examples and guides for using the kubectl integration please see the [kubectl book] or the [kubernetes documentation].
 
 ## Usage
 
@@ -119,51 +133,10 @@ The YAML can be directly [applied] to a cluster:
 
 ## Community
 
-### Filing bug reports
-
-
-##### A good report specifies
-
- * the output of `kustomize version`,
- * the input (the content of `kustomization.yaml`
-   and any files it refers to),
- * the expected YAML output.
-
-##### A _great_ report is a bug reproduction test
- 
-Kustomize has a simple test harness in the
-[target package] for specifying a kustomization's
-input and the expected output.
-See this [example of a target test].
- 
-The pattern is
- * call `NewKustTestHarness`
- * specify kustomization input data (resources,
-   patches, etc.) as inline strings,
- * call `makeKustTarget().MakeCustomizedResMap()`
- * compare the actual output to expected output
-
-In a bug reproduction test, the expected output string
-initially contains the _wrong_ (unexpected) output,
-thus unambiguously reproducing the bug.
-
-Nearby comments should explain what the output
-_should_ be, and have a TODO pointing to the related
-issue.
-
-The person who fixes the bug then has a clear
-bug reproduction and a test to modify when
-the bug is fixed.
-
-The bug reporter can then see the bug was fixed,
-and has permanent regression coverage to prevent
-its reintroduction.
- 
-### Feature requests
-
-Feature requests are welcome.
+To file bugs please read [this](docs/bugs.md).
 
 Before working on an implementation, please
+
  * Read the [eschewed feature list].
  * File an issue describing
    how the new feature would behave
@@ -192,21 +165,22 @@ is governed by the [Kubernetes Code of Conduct].
 [community page]: http://kubernetes.io/community/
 [declarative configuration]: docs/glossary.md#declarative-application-management
 [eschewed feature list]: docs/eschewedFeatures.md
-[example of a target test]: https://github.com/kubernetes-sigs/kustomize/blob/master/pkg/target/baseandoverlaysmall_test.go
-[examples]: examples/README.md
-[imageBase]: docs/base.jpg
-[imageOverlay]: docs/overlay.jpg
-[install]: docs/INSTALL.md
-[kind/feature]: https://github.com/kubernetes-sigs/kustomize/labels/kind%2Ffeature
+[imageBase]: docs/images/base.jpg
+[imageOverlay]: docs/images/overlay.jpg
+[kind/feature]: /../../labels/kind%2Ffeature
+[kubectl announcement]: https://kubernetes.io/blog/2019/03/25/kubernetes-1-14-release-announcement
+[kubectl book]: https://kubectl.docs.kubernetes.io/pages/app_customization/introduction.html
+[kubernetes documentation]: https://kubernetes.io/docs/tasks/manage-kubernetes-objects/kustomization/
 [kubernetes style]: docs/glossary.md#kubernetes-style-object
 [kustomization]: docs/glossary.md#kustomization
 [overlay]: docs/glossary.md#overlay
 [overlays]: docs/glossary.md#overlay
-[release page]: https://github.com/kubernetes-sigs/kustomize/releases
+[release page]: /../../releases
 [resource]: docs/glossary.md#resource
 [resources]: docs/glossary.md#resource
 [sig-cli]: https://github.com/kubernetes/community/blob/master/sig-cli/README.md
-[target package]: https://github.com/kubernetes-sigs/kustomize/tree/master/pkg/target
 [variant]: docs/glossary.md#variant
 [variants]: docs/glossary.md#variant
+[v2.0.3]: /../../releases/tag/v2.0.3
+[v2.1.0]: /../../releases/tag/v2.1.0
 [workflows]: docs/workflows.md

api/builtins/AnnotationsTransformer.go

@@ -0,0 +1,39 @@
+// Code generated by pluginator on AnnotationsTransformer; DO NOT EDIT.
+// pluginator {unknown  1970-01-01T00:00:00Z  }
+
+package builtins
+
+import (
+	"sigs.k8s.io/kustomize/api/resmap"
+	"sigs.k8s.io/kustomize/api/transform"
+	"sigs.k8s.io/kustomize/api/types"
+	"sigs.k8s.io/yaml"
+)
+
+// Add the given annotations to the given field specifications.
+type AnnotationsTransformerPlugin struct {
+	Annotations map[string]string `json:"annotations,omitempty" yaml:"annotations,omitempty"`
+	FieldSpecs  []types.FieldSpec `json:"fieldSpecs,omitempty" yaml:"fieldSpecs,omitempty"`
+}
+
+func (p *AnnotationsTransformerPlugin) Config(
+	_ *resmap.PluginHelpers, c []byte) (err error) {
+	p.Annotations = nil
+	p.FieldSpecs = nil
+	return yaml.Unmarshal(c, p)
+}
+
+func (p *AnnotationsTransformerPlugin) Transform(m resmap.ResMap) error {
+	t, err := transform.NewMapTransformer(
+		p.FieldSpecs,
+		p.Annotations,
+	)
+	if err != nil {
+		return err
+	}
+	return t.Transform(m)
+}
+
+func NewAnnotationsTransformerPlugin() resmap.TransformerPlugin {
+	return &AnnotationsTransformerPlugin{}
+}

api/builtins/ConfigMapGenerator.go

@@ -0,0 +1,43 @@
+// Code generated by pluginator on ConfigMapGenerator; DO NOT EDIT.
+// pluginator {unknown  1970-01-01T00:00:00Z  }
+
+package builtins
+
+import (
+	"sigs.k8s.io/kustomize/api/kv"
+	"sigs.k8s.io/kustomize/api/resmap"
+	"sigs.k8s.io/kustomize/api/types"
+	"sigs.k8s.io/yaml"
+)
+
+type ConfigMapGeneratorPlugin struct {
+	h                *resmap.PluginHelpers
+	types.ObjectMeta `json:"metadata,omitempty" yaml:"metadata,omitempty" protobuf:"bytes,1,opt,name=metadata"`
+	types.GeneratorOptions
+	types.ConfigMapArgs
+}
+
+func (p *ConfigMapGeneratorPlugin) Config(
+	h *resmap.PluginHelpers, config []byte) (err error) {
+	p.GeneratorOptions = types.GeneratorOptions{}
+	p.ConfigMapArgs = types.ConfigMapArgs{}
+	err = yaml.Unmarshal(config, p)
+	if p.ConfigMapArgs.Name == "" {
+		p.ConfigMapArgs.Name = p.Name
+	}
+	if p.ConfigMapArgs.Namespace == "" {
+		p.ConfigMapArgs.Namespace = p.Namespace
+	}
+	p.h = h
+	return
+}
+
+func (p *ConfigMapGeneratorPlugin) Generate() (resmap.ResMap, error) {
+	return p.h.ResmapFactory().FromConfigMapArgs(
+		kv.NewLoader(p.h.Loader(), p.h.Validator()),
+		&p.GeneratorOptions, p.ConfigMapArgs)
+}
+
+func NewConfigMapGeneratorPlugin() resmap.GeneratorPlugin {
+	return &ConfigMapGeneratorPlugin{}
+}

api/builtins/HashTransformer.go

@@ -0,0 +1,39 @@
+// Code generated by pluginator on HashTransformer; DO NOT EDIT.
+// pluginator {unknown  1970-01-01T00:00:00Z  }
+
+package builtins
+
+import (
+	"fmt"
+
+	"sigs.k8s.io/kustomize/api/ifc"
+	"sigs.k8s.io/kustomize/api/resmap"
+)
+
+type HashTransformerPlugin struct {
+	hasher ifc.KunstructuredHasher
+}
+
+func (p *HashTransformerPlugin) Config(
+	h *resmap.PluginHelpers, _ []byte) (err error) {
+	p.hasher = h.ResmapFactory().RF().Hasher()
+	return nil
+}
+
+// Transform appends hash to generated resources.
+func (p *HashTransformerPlugin) Transform(m resmap.ResMap) error {
+	for _, res := range m.Resources() {
+		if res.NeedHashSuffix() {
+			h, err := p.hasher.Hash(res)
+			if err != nil {
+				return err
+			}
+			res.SetName(fmt.Sprintf("%s-%s", res.GetName(), h))
+		}
+	}
+	return nil
+}
+
+func NewHashTransformerPlugin() resmap.TransformerPlugin {
+	return &HashTransformerPlugin{}
+}

api/builtins/ImageTagTransformer.go

@@ -0,0 +1,185 @@
+// Code generated by pluginator on ImageTagTransformer; DO NOT EDIT.
+// pluginator {unknown  1970-01-01T00:00:00Z  }
+
+package builtins
+
+import (
+	"fmt"
+	"regexp"
+	"strings"
+
+	"sigs.k8s.io/kustomize/api/transform"
+	"sigs.k8s.io/kustomize/api/types"
+
+	"sigs.k8s.io/kustomize/api/resmap"
+	"sigs.k8s.io/yaml"
+)
+
+// Find matching image declarations and replace
+// the name, tag and/or digest.
+type ImageTagTransformerPlugin struct {
+	ImageTag   types.Image       `json:"imageTag,omitempty" yaml:"imageTag,omitempty"`
+	FieldSpecs []types.FieldSpec `json:"fieldSpecs,omitempty" yaml:"fieldSpecs,omitempty"`
+}
+
+func (p *ImageTagTransformerPlugin) Config(
+	_ *resmap.PluginHelpers, c []byte) (err error) {
+	p.ImageTag = types.Image{}
+	p.FieldSpecs = nil
+	return yaml.Unmarshal(c, p)
+}
+
+func (p *ImageTagTransformerPlugin) Transform(m resmap.ResMap) error {
+	for _, r := range m.Resources() {
+		for _, path := range p.FieldSpecs {
+			if !r.OrgId().IsSelected(&path.Gvk) {
+				continue
+			}
+			err := transform.MutateField(
+				r.Map(), path.PathSlice(), false, p.mutateImage)
+			if err != nil {
+				return err
+			}
+		}
+		// Kept for backward compatibility
+		if err := p.findAndReplaceImage(r.Map()); err != nil && r.OrgId().Kind != `CustomResourceDefinition` {
+			return err
+		}
+	}
+	return nil
+}
+
+func (p *ImageTagTransformerPlugin) mutateImage(in interface{}) (interface{}, error) {
+	original, ok := in.(string)
+	if !ok {
+		return nil, fmt.Errorf("image path is not of type string but %T", in)
+	}
+
+	if !isImageMatched(original, p.ImageTag.Name) {
+		return original, nil
+	}
+	name, tag := split(original)
+	if p.ImageTag.NewName != "" {
+		name = p.ImageTag.NewName
+	}
+	if p.ImageTag.NewTag != "" {
+		tag = ":" + p.ImageTag.NewTag
+	}
+	if p.ImageTag.Digest != "" {
+		tag = "@" + p.ImageTag.Digest
+	}
+	return name + tag, nil
+}
+
+// findAndReplaceImage replaces the image name and
+// tags inside one object.
+// It searches the object for container session
+// then loops though all images inside containers
+// session, finds matched ones and update the
+// image name and tag name
+func (p *ImageTagTransformerPlugin) findAndReplaceImage(obj map[string]interface{}) error {
+	paths := []string{"containers", "initContainers"}
+	updated := false
+	for _, path := range paths {
+		containers, found := obj[path]
+		if found && containers != nil {
+			if _, err := p.updateContainers(containers); err != nil {
+				return err
+			}
+			updated = true
+		}
+	}
+	if !updated {
+		return p.findContainers(obj)
+	}
+	return nil
+}
+
+func (p *ImageTagTransformerPlugin) updateContainers(in interface{}) (interface{}, error) {
+	containers, ok := in.([]interface{})
+	if !ok {
+		return nil, fmt.Errorf(
+			"containers path is not of type []interface{} but %T", in)
+	}
+	for i := range containers {
+		container := containers[i].(map[string]interface{})
+		containerImage, found := container["image"]
+		if !found {
+			continue
+		}
+		imageName := containerImage.(string)
+		if isImageMatched(imageName, p.ImageTag.Name) {
+			newImage, err := p.mutateImage(imageName)
+			if err != nil {
+				return nil, err
+			}
+			container["image"] = newImage
+		}
+	}
+	return containers, nil
+}
+
+func (p *ImageTagTransformerPlugin) findContainers(obj map[string]interface{}) error {
+	for key := range obj {
+		switch typedV := obj[key].(type) {
+		case map[string]interface{}:
+			err := p.findAndReplaceImage(typedV)
+			if err != nil {
+				return err
+			}
+		case []interface{}:
+			for i := range typedV {
+				item := typedV[i]
+				typedItem, ok := item.(map[string]interface{})
+				if ok {
+					err := p.findAndReplaceImage(typedItem)
+					if err != nil {
+						return err
+					}
+				}
+			}
+		}
+	}
+	return nil
+}
+
+func isImageMatched(s, t string) bool {
+	// Tag values are limited to [a-zA-Z0-9_.-].
+	pattern, _ := regexp.Compile("^" + t + "(@sha256)?(:[a-zA-Z0-9_.-]*)?$")
+	return pattern.MatchString(s)
+}
+
+// split separates and returns the name and tag parts
+// from the image string using either colon `:` or at `@` separators.
+// Note that the returned tag keeps its separator.
+func split(imageName string) (name string, tag string) {
+	// check if image name contains a domain
+	// if domain is present, ignore domain and check for `:`
+	ic := -1
+	if slashIndex := strings.Index(imageName, "/"); slashIndex < 0 {
+		ic = strings.LastIndex(imageName, ":")
+	} else {
+		lastIc := strings.LastIndex(imageName[slashIndex:], ":")
+		// set ic only if `:` is present
+		if lastIc > 0 {
+			ic = slashIndex + lastIc
+		}
+	}
+	ia := strings.LastIndex(imageName, "@")
+	if ic < 0 && ia < 0 {
+		return imageName, ""
+	}
+
+	i := ic
+	if ia > 0 {
+		i = ia
+	}
+
+	name = imageName[:i]
+	tag = imageName[i:]
+	return
+}
+
+func NewImageTagTransformerPlugin() resmap.TransformerPlugin {
+	return &ImageTagTransformerPlugin{}
+}

api/builtins/InventoryTransformer.go

@@ -0,0 +1,128 @@
+// Code generated by pluginator on InventoryTransformer; DO NOT EDIT.
+// pluginator {unknown  1970-01-01T00:00:00Z  }
+
+package builtins
+
+import (
+	"fmt"
+
+	"sigs.k8s.io/kustomize/api/hasher"
+	"sigs.k8s.io/kustomize/api/inventory"
+	"sigs.k8s.io/kustomize/api/kv"
+	"sigs.k8s.io/kustomize/api/resid"
+	"sigs.k8s.io/kustomize/api/resmap"
+	"sigs.k8s.io/kustomize/api/resource"
+	"sigs.k8s.io/kustomize/api/types"
+	"sigs.k8s.io/yaml"
+)
+
+type InventoryTransformerPlugin struct {
+	h                *resmap.PluginHelpers
+	types.ObjectMeta `json:"metadata,omitempty" yaml:"metadata,omitempty" protobuf:"bytes,1,opt,name=metadata"`
+	Policy           string `json:"policy,omitempty" yaml:"policy,omitempty"`
+}
+
+func (p *InventoryTransformerPlugin) Config(
+	h *resmap.PluginHelpers, c []byte) (err error) {
+	p.h = h
+	err = yaml.Unmarshal(c, p)
+	if err != nil {
+		return err
+	}
+	if p.Policy == "" {
+		p.Policy = types.GarbageIgnore.String()
+	}
+	if p.Policy != types.GarbageCollect.String() &&
+		p.Policy != types.GarbageIgnore.String() {
+		return fmt.Errorf(
+			"unrecognized garbagePolicy '%s'", p.Policy)
+	}
+	return nil
+}
+
+// Transform generates an inventory object from the input ResMap.
+// This ConfigMap supports the pruning command in
+// the client side tool proposed here:
+// https://github.com/kubernetes/enhancements/pull/810
+//
+// The inventory data is written to the ConfigMap's
+// annotations, rather than to the key-value pairs in
+// the ConfigMap's data field, since
+//   1. Keys in a ConfigMap's data field are too
+//      constrained for this purpose.
+//   2. Using annotations allow any object to be used,
+//      not just a ConfigMap, should some other object
+//      (e.g. some App object) become more desirable
+//      for this purpose.
+func (p *InventoryTransformerPlugin) Transform(m resmap.ResMap) error {
+
+	inv, h, err := makeInventory(m)
+	if err != nil {
+		return err
+	}
+
+	args := types.ConfigMapArgs{}
+	args.Name = p.Name
+	args.Namespace = p.Namespace
+	opts := &types.GeneratorOptions{
+		Annotations: make(map[string]string),
+	}
+	opts.Annotations[inventory.HashAnnotation] = h
+	err = inv.UpdateAnnotations(opts.Annotations)
+	if err != nil {
+		return err
+	}
+
+	cm, err := p.h.ResmapFactory().RF().MakeConfigMap(
+		kv.NewLoader(p.h.Loader(), p.h.Validator()), opts, &args)
+	if err != nil {
+		return err
+	}
+
+	if p.Policy == types.GarbageCollect.String() {
+		for _, byeBye := range m.AllIds() {
+			m.Remove(byeBye)
+		}
+	}
+	return m.Append(cm)
+}
+
+func makeInventory(m resmap.ResMap) (
+	inv *inventory.Inventory, hash string, err error) {
+	inv = inventory.NewInventory()
+	var keys []string
+	for _, r := range m.Resources() {
+		ns := r.GetNamespace()
+		item := resid.NewResIdWithNamespace(r.GetGvk(), r.GetName(), ns)
+		if _, ok := inv.Current[item]; ok {
+			return nil, "", fmt.Errorf(
+				"item '%v' already in inventory", item)
+		}
+		inv.Current[item], err = computeRefs(r, m)
+		if err != nil {
+			return nil, "", err
+		}
+		keys = append(keys, item.String())
+	}
+	h, err := hasher.SortArrayAndComputeHash(keys)
+	return inv, h, err
+}
+
+func computeRefs(
+	r *resource.Resource, m resmap.ResMap) (refs []resid.ResId, err error) {
+	for _, refid := range r.GetRefBy() {
+		ref, err := m.GetByCurrentId(refid)
+		if err != nil {
+			return nil, err
+		}
+		refs = append(
+			refs,
+			resid.NewResIdWithNamespace(
+				ref.GetGvk(), ref.GetName(), ref.GetNamespace()))
+	}
+	return
+}
+
+func NewInventoryTransformerPlugin() resmap.TransformerPlugin {
+	return &InventoryTransformerPlugin{}
+}

api/builtins/LabelTransformer.go

@@ -0,0 +1,39 @@
+// Code generated by pluginator on LabelTransformer; DO NOT EDIT.
+// pluginator {unknown  1970-01-01T00:00:00Z  }
+
+package builtins
+
+import (
+	"sigs.k8s.io/kustomize/api/resmap"
+	"sigs.k8s.io/kustomize/api/transform"
+	"sigs.k8s.io/kustomize/api/types"
+	"sigs.k8s.io/yaml"
+)
+
+// Add the given labels to the given field specifications.
+type LabelTransformerPlugin struct {
+	Labels     map[string]string `json:"labels,omitempty" yaml:"labels,omitempty"`
+	FieldSpecs []types.FieldSpec `json:"fieldSpecs,omitempty" yaml:"fieldSpecs,omitempty"`
+}
+
+func (p *LabelTransformerPlugin) Config(
+	_ *resmap.PluginHelpers, c []byte) (err error) {
+	p.Labels = nil
+	p.FieldSpecs = nil
+	return yaml.Unmarshal(c, p)
+}
+
+func (p *LabelTransformerPlugin) Transform(m resmap.ResMap) error {
+	t, err := transform.NewMapTransformer(
+		p.FieldSpecs,
+		p.Labels,
+	)
+	if err != nil {
+		return err
+	}
+	return t.Transform(m)
+}
+
+func NewLabelTransformerPlugin() resmap.TransformerPlugin {
+	return &LabelTransformerPlugin{}
+}

api/builtins/LegacyOrderTransformer.go

@@ -0,0 +1,46 @@
+// Code generated by pluginator on LegacyOrderTransformer; DO NOT EDIT.
+// pluginator {unknown  1970-01-01T00:00:00Z  }
+
+package builtins
+
+import (
+	"sort"
+
+	"github.com/pkg/errors"
+	"sigs.k8s.io/kustomize/api/resmap"
+	"sigs.k8s.io/kustomize/api/resource"
+)
+
+// Sort the resources using an ordering defined in the Gvk class.
+// This puts cluster-wide basic resources with no
+// dependencies (like Namespace, StorageClass, etc.)
+// first, and resources with a high number of dependencies
+// (like ValidatingWebhookConfiguration) last.
+type LegacyOrderTransformerPlugin struct{}
+
+// Nothing needed for configuration.
+func (p *LegacyOrderTransformerPlugin) Config(
+	_ *resmap.PluginHelpers, _ []byte) (err error) {
+	return nil
+}
+
+func (p *LegacyOrderTransformerPlugin) Transform(m resmap.ResMap) (err error) {
+	resources := make([]*resource.Resource, m.Size())
+	ids := m.AllIds()
+	sort.Sort(resmap.IdSlice(ids))
+	for i, id := range ids {
+		resources[i], err = m.GetByCurrentId(id)
+		if err != nil {
+			return errors.Wrap(err, "expected match for sorting")
+		}
+	}
+	m.Clear()
+	for _, r := range resources {
+		m.Append(r)
+	}
+	return nil
+}
+
+func NewLegacyOrderTransformerPlugin() resmap.TransformerPlugin {
+	return &LegacyOrderTransformerPlugin{}
+}

api/builtins/NamespaceTransformer.go

@@ -0,0 +1,131 @@
+// Code generated by pluginator on NamespaceTransformer; DO NOT EDIT.
+// pluginator {unknown  1970-01-01T00:00:00Z  }
+
+package builtins
+
+import (
+	"fmt"
+
+	"sigs.k8s.io/kustomize/api/transform"
+
+	"sigs.k8s.io/kustomize/api/resid"
+	"sigs.k8s.io/kustomize/api/resmap"
+	"sigs.k8s.io/kustomize/api/resource"
+	"sigs.k8s.io/kustomize/api/types"
+	"sigs.k8s.io/yaml"
+)
+
+// Change or set the namespace of non-cluster level resources.
+type NamespaceTransformerPlugin struct {
+	types.ObjectMeta `json:"metadata,omitempty" yaml:"metadata,omitempty" protobuf:"bytes,1,opt,name=metadata"`
+	FieldSpecs       []types.FieldSpec `json:"fieldSpecs,omitempty" yaml:"fieldSpecs,omitempty"`
+}
+
+func (p *NamespaceTransformerPlugin) Config(
+	_ *resmap.PluginHelpers, c []byte) (err error) {
+	p.Namespace = ""
+	p.FieldSpecs = nil
+	return yaml.Unmarshal(c, p)
+}
+
+func (p *NamespaceTransformerPlugin) Transform(m resmap.ResMap) error {
+	if len(p.Namespace) == 0 {
+		return nil
+	}
+	for _, r := range m.Resources() {
+		if len(r.Map()) == 0 {
+			// Don't mutate empty objects?
+			continue
+		}
+
+		id := r.OrgId()
+		applicableFs := p.applicableFieldSpecs(id)
+
+		for _, fs := range applicableFs {
+			err := transform.MutateField(
+				r.Map(), fs.PathSlice(), fs.CreateIfNotPresent,
+				p.changeNamespace(r))
+			if err != nil {
+				return err
+			}
+		}
+
+		matches := m.GetMatchingResourcesByCurrentId(r.CurId().Equals)
+		if len(matches) != 1 {
+			return fmt.Errorf("namespace tranformation produces ID conflict: %+v", matches)
+		}
+	}
+	return nil
+}
+
+const metaNamespace = "metadata/namespace"
+
+// Special casing metadata.namespace since
+// all objects have it, even "ClusterKind" objects
+// that don't exist in a namespace (the Namespace
+// object itself doesn't live in a namespace).
+func (p *NamespaceTransformerPlugin) applicableFieldSpecs(id resid.ResId) []types.FieldSpec {
+	var res []types.FieldSpec
+	for _, fs := range p.FieldSpecs {
+		if id.IsSelected(&fs.Gvk) && (fs.Path != metaNamespace || (fs.Path == metaNamespace && id.IsNamespaceableKind())) {
+			res = append(res, fs)
+		}
+	}
+	return res
+}
+
+func (p *NamespaceTransformerPlugin) changeNamespace(
+	_ *resource.Resource) func(in interface{}) (interface{}, error) {
+	return func(in interface{}) (interface{}, error) {
+		switch in.(type) {
+		case string:
+			// will happen when the metadata/namespace
+			// value is replaced
+			return p.Namespace, nil
+		case []interface{}:
+			l, _ := in.([]interface{})
+			for idx, item := range l {
+				switch item.(type) {
+				case map[string]interface{}:
+					// Will happen when mutating the subjects
+					// field of ClusterRoleBinding and RoleBinding
+					inMap, _ := item.(map[string]interface{})
+					if _, ok := inMap["name"]; !ok {
+						continue
+					}
+					name, ok := inMap["name"].(string)
+					if !ok {
+						continue
+					}
+					// The only case we need to force the namespace
+					// if for the "service account". "default" is
+					// kind of hardcoded here for right now.
+					if name != "default" {
+						continue
+					}
+					inMap["namespace"] = p.Namespace
+					l[idx] = inMap
+				default:
+					// nothing to do for right now
+				}
+			}
+			return in, nil
+		case map[string]interface{}:
+			// Will happen if the createField=true
+			// when the namespace is added to the
+			// object
+			inMap := in.(map[string]interface{})
+			if len(inMap) == 0 {
+				return p.Namespace, nil
+			} else {
+				return in, nil
+			}
+		default:
+			return in, nil
+		}
+	}
+}
+
+func NewNamespaceTransformerPlugin() resmap.TransformerPlugin {
+	return &NamespaceTransformerPlugin{}
+}

api/builtins/PatchJson6902Transformer.go

@@ -0,0 +1,100 @@
+// Code generated by pluginator on PatchJson6902Transformer; DO NOT EDIT.
+// pluginator {unknown  1970-01-01T00:00:00Z  }
+
+package builtins
+
+import (
+	"fmt"
+
+	jsonpatch "github.com/evanphx/json-patch"
+	"github.com/pkg/errors"
+	"sigs.k8s.io/kustomize/api/ifc"
+	"sigs.k8s.io/kustomize/api/resid"
+	"sigs.k8s.io/kustomize/api/resmap"
+	"sigs.k8s.io/kustomize/api/types"
+	"sigs.k8s.io/yaml"
+)
+
+type PatchJson6902TransformerPlugin struct {
+	ldr          ifc.Loader
+	decodedPatch jsonpatch.Patch
+	Target       types.PatchTarget `json:"target,omitempty" yaml:"target,omitempty"`
+	Path         string            `json:"path,omitempty" yaml:"path,omitempty"`
+	JsonOp       string            `json:"jsonOp,omitempty" yaml:"jsonOp,omitempty"`
+}
+
+func (p *PatchJson6902TransformerPlugin) Config(
+	h *resmap.PluginHelpers, c []byte) (err error) {
+	p.ldr = h.Loader()
+	err = yaml.Unmarshal(c, p)
+	if err != nil {
+		return err
+	}
+	if p.Target.Name == "" {
+		return fmt.Errorf("must specify the target name")
+	}
+	if p.Path == "" && p.JsonOp == "" {
+		return fmt.Errorf("empty file path and empty jsonOp")
+	}
+	if p.Path != "" {
+		if p.JsonOp != "" {
+			return fmt.Errorf("must specify a file path or jsonOp, not both")
+		}
+		rawOp, err := p.ldr.Load(p.Path)
+		if err != nil {
+			return err
+		}
+		p.JsonOp = string(rawOp)
+		if p.JsonOp == "" {
+			return fmt.Errorf("patch file '%s' empty seems to be empty", p.Path)
+		}
+	}
+	if p.JsonOp[0] != '[' {
+		// if it doesn't seem to be JSON, imagine
+		// it is YAML, and convert to JSON.
+		op, err := yaml.YAMLToJSON([]byte(p.JsonOp))
+		if err != nil {
+			return err
+		}
+		p.JsonOp = string(op)
+	}
+	p.decodedPatch, err = jsonpatch.DecodePatch([]byte(p.JsonOp))
+	if err != nil {
+		return errors.Wrapf(err, "decoding %s", p.JsonOp)
+	}
+	if len(p.decodedPatch) == 0 {
+		return fmt.Errorf(
+			"patch appears to be empty; file=%s, JsonOp=%s", p.Path, p.JsonOp)
+	}
+	return err
+}
+
+func (p *PatchJson6902TransformerPlugin) Transform(m resmap.ResMap) error {
+	id := resid.NewResIdWithNamespace(
+		resid.Gvk{
+			Group:   p.Target.Group,
+			Version: p.Target.Version,
+			Kind:    p.Target.Kind,
+		},
+		p.Target.Name,
+		p.Target.Namespace,
+	)
+	obj, err := m.GetById(id)
+	if err != nil {
+		return err
+	}
+	rawObj, err := obj.MarshalJSON()
+	if err != nil {
+		return err
+	}
+	modifiedObj, err := p.decodedPatch.Apply(rawObj)
+	if err != nil {
+		return errors.Wrapf(
+			err, "failed to apply json patch '%s'", p.JsonOp)
+	}
+	return obj.UnmarshalJSON(modifiedObj)
+}
+
+func NewPatchJson6902TransformerPlugin() resmap.TransformerPlugin {
+	return &PatchJson6902TransformerPlugin{}
+}

api/builtins/PatchStrategicMergeTransformer.go

@@ -0,0 +1,90 @@
+// Code generated by pluginator on PatchStrategicMergeTransformer; DO NOT EDIT.
+// pluginator {unknown  1970-01-01T00:00:00Z  }
+
+package builtins
+
+import (
+	"fmt"
+
+	"sigs.k8s.io/kustomize/api/resmap"
+	"sigs.k8s.io/kustomize/api/resource"
+	"sigs.k8s.io/kustomize/api/types"
+	"sigs.k8s.io/yaml"
+)
+
+type PatchStrategicMergeTransformerPlugin struct {
+	h             *resmap.PluginHelpers
+	loadedPatches []*resource.Resource
+	Paths         []types.PatchStrategicMerge `json:"paths,omitempty" yaml:"paths,omitempty"`
+	Patches       string                      `json:"patches,omitempty" yaml:"patches,omitempty"`
+}
+
+func (p *PatchStrategicMergeTransformerPlugin) Config(
+	h *resmap.PluginHelpers, c []byte) (err error) {
+	p.h = h
+	err = yaml.Unmarshal(c, p)
+	if err != nil {
+		return err
+	}
+	if len(p.Paths) == 0 && p.Patches == "" {
+		return fmt.Errorf("empty file path and empty patch content")
+	}
+	if len(p.Paths) != 0 {
+		for _, onePath := range p.Paths {
+			res, err := p.h.ResmapFactory().RF().SliceFromBytes([]byte(onePath))
+			if err == nil {
+				p.loadedPatches = append(p.loadedPatches, res...)
+				continue
+			}
+			res, err = p.h.ResmapFactory().RF().SliceFromPatches(
+				p.h.Loader(), []types.PatchStrategicMerge{onePath})
+			if err != nil {
+				return err
+			}
+			p.loadedPatches = append(p.loadedPatches, res...)
+		}
+	}
+	if p.Patches != "" {
+		res, err := p.h.ResmapFactory().RF().SliceFromBytes([]byte(p.Patches))
+		if err != nil {
+			return err
+		}
+		p.loadedPatches = append(p.loadedPatches, res...)
+	}
+
+	if len(p.loadedPatches) == 0 {
+		return fmt.Errorf(
+			"patch appears to be empty; files=%v, Patch=%s", p.Paths, p.Patches)
+	}
+	return err
+}
+
+func (p *PatchStrategicMergeTransformerPlugin) Transform(m resmap.ResMap) error {
+	patches, err := p.h.ResmapFactory().MergePatches(p.loadedPatches)
+	if err != nil {
+		return err
+	}
+	for _, patch := range patches.Resources() {
+		target, err := m.GetById(patch.OrgId())
+		if err != nil {
+			return err
+		}
+		err = target.Patch(patch.Kunstructured)
+		if err != nil {
+			return err
+		}
+		// remove the resource from resmap
+		// when the patch is to $patch: delete that target
+		if len(target.Map()) == 0 {
+			err = m.Remove(target.CurId())
+			if err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+func NewPatchStrategicMergeTransformerPlugin() resmap.TransformerPlugin {
+	return &PatchStrategicMergeTransformerPlugin{}
+}

api/builtins/PatchTransformer.go

@@ -0,0 +1,145 @@
+// Code generated by pluginator on PatchTransformer; DO NOT EDIT.
+// pluginator {unknown  1970-01-01T00:00:00Z  }
+
+package builtins
+
+import (
+	"fmt"
+
+	jsonpatch "github.com/evanphx/json-patch"
+	"github.com/pkg/errors"
+	"sigs.k8s.io/kustomize/api/resmap"
+	"sigs.k8s.io/kustomize/api/resource"
+	"sigs.k8s.io/kustomize/api/types"
+	"sigs.k8s.io/yaml"
+)
+
+type PatchTransformerPlugin struct {
+	loadedPatch  *resource.Resource
+	decodedPatch jsonpatch.Patch
+	Path         string          `json:"path,omitempty" yaml:"path,omitempty"`
+	Patch        string          `json:"patch,omitempty" yaml:"patch,omitempty"`
+	Target       *types.Selector `json:"target,omitempty" yaml:"target,omitempty"`
+}
+
+func (p *PatchTransformerPlugin) Config(
+	h *resmap.PluginHelpers, c []byte) (err error) {
+	err = yaml.Unmarshal(c, p)
+	if err != nil {
+		return err
+	}
+	if p.Patch == "" && p.Path == "" {
+		err = fmt.Errorf(
+			"must specify one of patch and path in\n%s", string(c))
+		return
+	}
+	if p.Patch != "" && p.Path != "" {
+		err = fmt.Errorf(
+			"patch and path can't be set at the same time\n%s", string(c))
+		return
+	}
+	var in []byte
+	if p.Path != "" {
+		in, err = h.Loader().Load(p.Path)
+		if err != nil {
+			return
+		}
+	}
+	if p.Patch != "" {
+		in = []byte(p.Patch)
+	}
+
+	patchSM, errSM := h.ResmapFactory().RF().FromBytes(in)
+	patchJson, errJson := jsonPatchFromBytes(in)
+	if errSM != nil && errJson != nil {
+		err = fmt.Errorf(
+			"unable to get either a Strategic Merge Patch or JSON patch 6902 from %s", p.Patch)
+		return
+	}
+	if errSM == nil && errJson != nil {
+		p.loadedPatch = patchSM
+	}
+	if errJson == nil && errSM != nil {
+		p.decodedPatch = patchJson
+	}
+	if patchSM != nil && patchJson != nil {
+		err = fmt.Errorf(
+			"a patch can't be both a Strategic Merge Patch and JSON patch 6902 %s", p.Patch)
+	}
+
+	return nil
+}
+
+func (p *PatchTransformerPlugin) Transform(m resmap.ResMap) error {
+	if p.loadedPatch != nil && p.Target == nil {
+		target, err := m.GetById(p.loadedPatch.OrgId())
+		if err != nil {
+			return err
+		}
+		err = target.Patch(p.loadedPatch.Kunstructured)
+		if err != nil {
+			return err
+		}
+		return nil
+	}
+
+	if p.Target == nil {
+		return fmt.Errorf("must specify a target for patch %s", p.Patch)
+	}
+
+	resources, err := m.Select(*p.Target)
+	if err != nil {
+		return err
+	}
+	for _, res := range resources {
+		if p.decodedPatch != nil {
+			rawObj, err := res.MarshalJSON()
+			if err != nil {
+				return err
+			}
+			modifiedObj, err := p.decodedPatch.Apply(rawObj)
+			if err != nil {
+				return errors.Wrapf(
+					err, "failed to apply json patch '%s'", p.Patch)
+			}
+			err = res.UnmarshalJSON(modifiedObj)
+			if err != nil {
+				return err
+			}
+		}
+		if p.loadedPatch != nil {
+			patchCopy := p.loadedPatch.DeepCopy()
+			patchCopy.SetName(res.GetName())
+			patchCopy.SetNamespace(res.GetNamespace())
+			patchCopy.SetGvk(res.GetGvk())
+			err = res.Patch(patchCopy.Kunstructured)
+			if err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+// jsonPatchFromBytes loads a Json 6902 patch from
+// a bytes input
+func jsonPatchFromBytes(
+	in []byte) (jsonpatch.Patch, error) {
+	ops := string(in)
+	if ops == "" {
+		return nil, fmt.Errorf("empty json patch operations")
+	}
+
+	if ops[0] != '[' {
+		jsonOps, err := yaml.YAMLToJSON(in)
+		if err != nil {
+			return nil, err
+		}
+		ops = string(jsonOps)
+	}
+	return jsonpatch.DecodePatch([]byte(ops))
+}
+
+func NewPatchTransformerPlugin() resmap.TransformerPlugin {
+	return &PatchTransformerPlugin{}
+}

api/builtins/PrefixSuffixTransformer.go

@@ -0,0 +1,123 @@
+// Code generated by pluginator on PrefixSuffixTransformer; DO NOT EDIT.
+// pluginator {unknown  1970-01-01T00:00:00Z  }
+
+package builtins
+
+import (
+	"errors"
+	"fmt"
+
+	"sigs.k8s.io/kustomize/api/transform"
+	"sigs.k8s.io/kustomize/api/types"
+
+	"sigs.k8s.io/kustomize/api/resid"
+	"sigs.k8s.io/kustomize/api/resmap"
+	"sigs.k8s.io/yaml"
+)
+
+// Add the given prefix and suffix to the field.
+type PrefixSuffixTransformerPlugin struct {
+	Prefix     string            `json:"prefix,omitempty" yaml:"prefix,omitempty"`
+	Suffix     string            `json:"suffix,omitempty" yaml:"suffix,omitempty"`
+	FieldSpecs []types.FieldSpec `json:"fieldSpecs,omitempty" yaml:"fieldSpecs,omitempty"`
+}
+
+// Not placed in a file yet due to lack of demand.
+var prefixSuffixFieldSpecsToSkip = []types.FieldSpec{
+	{
+		Gvk: resid.Gvk{Kind: "CustomResourceDefinition"},
+	},
+	{
+		Gvk: resid.Gvk{Group: "apiregistration.k8s.io", Kind: "APIService"},
+	},
+}
+
+func (p *PrefixSuffixTransformerPlugin) Config(
+	_ *resmap.PluginHelpers, c []byte) (err error) {
+	p.Prefix = ""
+	p.Suffix = ""
+	p.FieldSpecs = nil
+	err = yaml.Unmarshal(c, p)
+	if err != nil {
+		return
+	}
+	if p.FieldSpecs == nil {
+		return errors.New("fieldSpecs is not expected to be nil")
+	}
+	return
+}
+
+func (p *PrefixSuffixTransformerPlugin) Transform(m resmap.ResMap) error {
+
+	// Even if both the Prefix and Suffix are empty we want
+	// to proceed with the transformation. This allows to add contextual
+	// information to the resources (AddNamePrefix and AddNameSuffix).
+
+	for _, r := range m.Resources() {
+		if p.shouldSkip(r.OrgId()) {
+			// Don't change the actual definition
+			// of a CRD.
+			continue
+		}
+		id := r.OrgId()
+		// current default configuration contains
+		// only one entry: "metadata/name" with no GVK
+		for _, path := range p.FieldSpecs {
+			if !id.IsSelected(&path.Gvk) {
+				// With the currrent default configuration,
+				// because no Gvk is specified, so a wild
+				// card
+				continue
+			}
+
+			if smellsLikeANameChange(&path) {
+				// "metadata/name" is the only field.
+				// this will add a prefix and a suffix
+				// to the resource even if those are
+				// empty
+				r.AddNamePrefix(p.Prefix)
+				r.AddNameSuffix(p.Suffix)
+			}
+
+			// the addPrefixSuffix method will not
+			// change the name if both the prefix and suffix
+			// are empty.
+			err := transform.MutateField(
+				r.Map(),
+				path.PathSlice(),
+				path.CreateIfNotPresent,
+				p.addPrefixSuffix)
+			if err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+func smellsLikeANameChange(fs *types.FieldSpec) bool {
+	return fs.Path == "metadata/name"
+}
+
+func (p *PrefixSuffixTransformerPlugin) shouldSkip(
+	id resid.ResId) bool {
+	for _, path := range prefixSuffixFieldSpecsToSkip {
+		if id.IsSelected(&path.Gvk) {
+			return true
+		}
+	}
+	return false
+}
+
+func (p *PrefixSuffixTransformerPlugin) addPrefixSuffix(
+	in interface{}) (interface{}, error) {
+	s, ok := in.(string)
+	if !ok {
+		return nil, fmt.Errorf("%#v is expected to be %T", in, s)
+	}
+	return fmt.Sprintf("%s%s%s", p.Prefix, s, p.Suffix), nil
+}
+
+func NewPrefixSuffixTransformerPlugin() resmap.TransformerPlugin {
+	return &PrefixSuffixTransformerPlugin{}
+}

api/builtins/ReplicaCountTransformer.go

@@ -0,0 +1,88 @@
+// Code generated by pluginator on ReplicaCountTransformer; DO NOT EDIT.
+// pluginator {unknown  1970-01-01T00:00:00Z  }
+
+package builtins
+
+import (
+	"fmt"
+
+	"sigs.k8s.io/kustomize/api/transform"
+
+	"sigs.k8s.io/kustomize/api/resid"
+	"sigs.k8s.io/kustomize/api/resmap"
+	"sigs.k8s.io/kustomize/api/types"
+	"sigs.k8s.io/yaml"
+)
+
+// Find matching replicas declarations and replace the count.
+// Eases the kustomization configuration of replica changes.
+type ReplicaCountTransformerPlugin struct {
+	Replica    types.Replica     `json:"replica,omitempty" yaml:"replica,omitempty"`
+	FieldSpecs []types.FieldSpec `json:"fieldSpecs,omitempty" yaml:"fieldSpecs,omitempty"`
+}
+
+func (p *ReplicaCountTransformerPlugin) Config(
+	_ *resmap.PluginHelpers, c []byte) (err error) {
+	p.Replica = types.Replica{}
+	p.FieldSpecs = nil
+	return yaml.Unmarshal(c, p)
+}
+
+func (p *ReplicaCountTransformerPlugin) Transform(m resmap.ResMap) error {
+	found := false
+	for i, replicaSpec := range p.FieldSpecs {
+		matcher := p.createMatcher(i)
+		matchOriginal := m.GetMatchingResourcesByOriginalId(matcher)
+		matchCurrent := m.GetMatchingResourcesByCurrentId(matcher)
+
+		for _, res := range append(matchOriginal, matchCurrent...) {
+			found = true
+			err := transform.MutateField(
+				res.Map(), replicaSpec.PathSlice(),
+				replicaSpec.CreateIfNotPresent, p.addReplicas)
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	if !found {
+		gvks := make([]string, len(p.FieldSpecs))
+		for i, replicaSpec := range p.FieldSpecs {
+			gvks[i] = replicaSpec.Gvk.String()
+		}
+		return fmt.Errorf("resource with name %s does not match a config with the following GVK %v",
+			p.Replica.Name, gvks)
+	}
+
+	return nil
+}
+
+// Match Replica.Name and FieldSpec
+func (p *ReplicaCountTransformerPlugin) createMatcher(i int) resmap.IdMatcher {
+	return func(r resid.ResId) bool {
+		return r.Name == p.Replica.Name &&
+			r.Gvk.IsSelected(&p.FieldSpecs[i].Gvk)
+	}
+}
+
+func (p *ReplicaCountTransformerPlugin) addReplicas(in interface{}) (interface{}, error) {
+	switch m := in.(type) {
+	case int64:
+		// Was already in the field.
+	case map[string]interface{}:
+		if len(m) != 0 {
+			// A map was already in the replicas field, don't want to
+			// discard this data silently.
+			return nil, fmt.Errorf("%#v is expected to be %T", in, m)
+		}
+		// Just got added, default type is map, but we can return anything.
+	default:
+		return nil, fmt.Errorf("%#v is expected to be %T", in, m)
+	}
+	return p.Replica.Count, nil
+}
+
+func NewReplicaCountTransformerPlugin() resmap.TransformerPlugin {
+	return &ReplicaCountTransformerPlugin{}
+}

api/builtins/SecretGenerator.go

@@ -0,0 +1,42 @@
+// Code generated by pluginator on SecretGenerator; DO NOT EDIT.
+// pluginator {unknown  1970-01-01T00:00:00Z  }
+
+package builtins
+
+import (
+	"sigs.k8s.io/kustomize/api/kv"
+	"sigs.k8s.io/kustomize/api/resmap"
+	"sigs.k8s.io/kustomize/api/types"
+	"sigs.k8s.io/yaml"
+)
+
+type SecretGeneratorPlugin struct {
+	h                *resmap.PluginHelpers
+	types.ObjectMeta `json:"metadata,omitempty" yaml:"metadata,omitempty" protobuf:"bytes,1,opt,name=metadata"`
+	types.GeneratorOptions
+	types.SecretArgs
+}
+
+func (p *SecretGeneratorPlugin) Config(h *resmap.PluginHelpers, config []byte) (err error) {
+	p.GeneratorOptions = types.GeneratorOptions{}
+	p.SecretArgs = types.SecretArgs{}
+	err = yaml.Unmarshal(config, p)
+	if p.SecretArgs.Name == "" {
+		p.SecretArgs.Name = p.Name
+	}
+	if p.SecretArgs.Namespace == "" {
+		p.SecretArgs.Namespace = p.Namespace
+	}
+	p.h = h
+	return
+}
+
+func (p *SecretGeneratorPlugin) Generate() (resmap.ResMap, error) {
+	return p.h.ResmapFactory().FromSecretArgs(
+		kv.NewLoader(p.h.Loader(), p.h.Validator()),
+		&p.GeneratorOptions, p.SecretArgs)
+}
+
+func NewSecretGeneratorPlugin() resmap.GeneratorPlugin {
+	return &SecretGeneratorPlugin{}
+}

api/builtins/doc.go

@@ -0,0 +1,8 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+// Package builtins holds code generated from the builtin plugins.
+// The "builtin" plugins are written as normal plugins and can
+// be used as such, but they are also used to generate the code
+// in this package so they can be statically linked to client code.
+package builtins

api/filesys/confirmeddir.go

@@ -1,20 +1,7 @@
-/*
-Copyright 2018 The Kubernetes Authors.
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
 
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package fs
+package filesys
 
 import (
 	"io/ioutil"
@@ -27,7 +14,7 @@ import (
 type ConfirmedDir string
 
 // NewTmpConfirmedDir returns a temporary dir, else error.
-// The directory is cleaned, no symlinks, etc. so its
+// The directory is cleaned, no symlinks, etc. so it's
 // returned as a ConfirmedDir.
 func NewTmpConfirmedDir() (ConfirmedDir, error) {
 	n, err := ioutil.TempDir("", "kustomize-")
@@ -36,13 +23,12 @@ func NewTmpConfirmedDir() (ConfirmedDir, error) {
 	}
 
 	// In MacOs `ioutil.TempDir` creates a directory
-	// with root in the `/var` folder, which is in turn a symlinked path
-	// to `/private/var`.
+	// with root in the `/var` folder, which is in turn
+	// a symlinked path to `/private/var`.
 	// Function `filepath.EvalSymlinks`is used to
 	// resolve the real absolute path.
 	deLinked, err := filepath.EvalSymlinks(n)
 	return ConfirmedDir(deLinked), err
-
 }
 
 // HasPrefix returns true if the directory argument

api/filesys/confirmeddir_test.go

@@ -1,28 +1,17 @@
-/*
-Copyright 2018 The Kubernetes Authors.
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
 
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package fs
+package filesys_test
 
 import (
 	"path/filepath"
 	"testing"
+
+	. "sigs.k8s.io/kustomize/api/filesys"
 )
 
 func TestJoin(t *testing.T) {
-	fSys := MakeFakeFS()
+	fSys := MakeFsInMemory()
 	err := fSys.Mkdir("/foo")
 	if err != nil {
 		t.Fatalf("unexpected err: %v", err)
@@ -40,7 +29,8 @@ func TestJoin(t *testing.T) {
 }
 
 func TestHasPrefix_Slash(t *testing.T) {
-	d, f, err := MakeFakeFS().CleanedAbs("/")
+	fSys := MakeFsInMemory()
+	d, f, err := fSys.CleanedAbs("/")
 	if err != nil {
 		t.Fatalf("unexpected err: %v", err)
 	}
@@ -56,7 +46,7 @@ func TestHasPrefix_Slash(t *testing.T) {
 }
 
 func TestHasPrefix_SlashFoo(t *testing.T) {
-	fSys := MakeFakeFS()
+	fSys := MakeFsInMemory()
 	err := fSys.Mkdir("/foo")
 	if err != nil {
 		t.Fatalf("unexpected err: %v", err)
@@ -77,7 +67,7 @@ func TestHasPrefix_SlashFoo(t *testing.T) {
 }
 
 func TestHasPrefix_SlashFooBar(t *testing.T) {
-	fSys := MakeFakeFS()
+	fSys := MakeFsInMemory()
 	err := fSys.MkdirAll("/foo/bar")
 	if err != nil {
 		t.Fatalf("unexpected err: %v", err)

api/filesys/file.go

@@ -0,0 +1,15 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package filesys
+
+import (
+	"io"
+	"os"
+)
+
+// File groups the basic os.File methods.
+type File interface {
+	io.ReadWriteCloser
+	Stat() (os.FileInfo, error)
+}

api/filesys/fileinfo.go

@@ -0,0 +1,34 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package filesys
+
+import (
+	"os"
+	"time"
+)
+
+var _ os.FileInfo = fileInfo{}
+
+// fileInfo implements os.FileInfo for a fileInMemory instance.
+type fileInfo struct {
+	node *fsNode
+}
+
+// Name returns the name of the file
+func (fi fileInfo) Name() string { return fi.node.Name() }
+
+// Size returns the size of the file
+func (fi fileInfo) Size() int64 { return fi.node.Size() }
+
+// Mode returns the file mode
+func (fi fileInfo) Mode() os.FileMode { return 0777 }
+
+// ModTime returns a bogus time
+func (fi fileInfo) ModTime() time.Time { return time.Time{} }
+
+// IsDir returns true if it is a directory
+func (fi fileInfo) IsDir() bool { return fi.node.isNodeADir() }
+
+// Sys should return underlying data source, but it now returns nil
+func (fi fileInfo) Sys() interface{} { return nil }

api/filesys/fileondisk.go

@@ -0,0 +1,27 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package filesys
+
+import (
+	"os"
+)
+
+var _ File = &fileOnDisk{}
+
+// fileOnDisk implements File using the local filesystem.
+type fileOnDisk struct {
+	file *os.File
+}
+
+// Close closes a file.
+func (f *fileOnDisk) Close() error { return f.file.Close() }
+
+// Read reads a file's content.
+func (f *fileOnDisk) Read(p []byte) (n int, err error) { return f.file.Read(p) }
+
+// Write writes bytes to a file
+func (f *fileOnDisk) Write(p []byte) (n int, err error) { return f.file.Write(p) }
+
+// Stat returns an interface which has all the information regarding the file.
+func (f *fileOnDisk) Stat() (os.FileInfo, error) { return f.file.Stat() }

api/filesys/filesystem.go

@@ -0,0 +1,50 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+// Package filesys provides a file system abstraction layer.
+package filesys
+
+import (
+	"path/filepath"
+)
+
+const (
+	Separator = string(filepath.Separator)
+	SelfDir   = "."
+	ParentDir = ".."
+)
+
+// FileSystem groups basic os filesystem methods.
+// It's supposed be functional subset of https://golang.org/pkg/os
+type FileSystem interface {
+	// Create a file.
+	Create(path string) (File, error)
+	// MkDir makes a directory.
+	Mkdir(path string) error
+	// MkDirAll makes a directory path, creating intervening directories.
+	MkdirAll(path string) error
+	// RemoveAll removes path and any children it contains.
+	RemoveAll(path string) error
+	// Open opens the named file for reading.
+	Open(path string) (File, error)
+	// IsDir returns true if the path is a directory.
+	IsDir(path string) bool
+	// CleanedAbs converts the given path into a
+	// directory and a file name, where the directory
+	// is represented as a ConfirmedDir and all that implies.
+	// If the entire path is a directory, the file component
+	// is an empty string.
+	CleanedAbs(path string) (ConfirmedDir, string, error)
+	// Exists is true if the path exists in the file system.
+	Exists(path string) bool
+	// Glob returns the list of matching files,
+	// emulating https://golang.org/pkg/path/filepath/#Glob
+	Glob(pattern string) ([]string, error)
+	// ReadFile returns the contents of the file at the given path.
+	ReadFile(path string) ([]byte, error)
+	// WriteFile writes the data to a file at the given path,
+	// overwriting anything that's already there.
+	WriteFile(path string, data []byte) error
+	// Walk walks the file system with the given WalkFunc.
+	Walk(path string, walkFn filepath.WalkFunc) error
+}

api/filesys/fsnode.go

@@ -0,0 +1,557 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package filesys
+
+import (
+	"bytes"
+	"fmt"
+	"log"
+	"os"
+	"path/filepath"
+	"regexp"
+	"sort"
+	"strings"
+
+	"github.com/pkg/errors"
+)
+
+var _ File = &fsNode{}
+var _ FileSystem = &fsNode{}
+
+// fsNode is either a file or a directory.
+type fsNode struct {
+	// What node owns me?
+	parent *fsNode
+
+	// Value to return as the Name() when the
+	// parent is nil.
+	nilParentName string
+
+	// A directory mapping names to nodes.
+	// If dir is nil, then self node is a file.
+	// If dir is non-nil, then self node is a directory,
+	// albeit possibly an empty directory.
+	dir map[string]*fsNode
+
+	// if this node is a file, this is the content.
+	content []byte
+
+	// if this node is a file, this tracks whether or
+	// not it is "open".
+	open bool
+}
+
+// MakeEmptyDirInMemory returns an empty directory.
+// The paths of nodes in this object will never
+// report a leading Separator, meaning they
+// aren't "absolute" in the sense defined by
+// https://golang.org/pkg/path/filepath/#IsAbs.
+func MakeEmptyDirInMemory() *fsNode {
+	return &fsNode{
+		dir: make(map[string]*fsNode),
+	}
+}
+
+// MakeFsInMemory returns an empty 'file system'.
+// The paths of nodes in this object will always
+// report a leading Separator, meaning they
+// are "absolute" in the sense defined by
+// https://golang.org/pkg/path/filepath/#IsAbs.
+// This is a relevant difference when using Walk,
+// Glob, Match, etc.
+func MakeFsInMemory() FileSystem {
+	return &fsNode{
+		nilParentName: Separator,
+		dir:           make(map[string]*fsNode),
+	}
+}
+
+// Name returns the name of the node.
+func (n *fsNode) Name() string {
+	if n.parent == nil {
+		// Unable to lookup name in parent.
+		return n.nilParentName
+	}
+	if !n.parent.isNodeADir() {
+		log.Fatal("parent not a dir")
+	}
+	for key, value := range n.parent.dir {
+		if value == n {
+			return key
+		}
+	}
+	log.Fatal("unable to find fsNode name")
+	return ""
+}
+
+// Path returns the full path to the node.
+func (n *fsNode) Path() string {
+	if n.parent == nil {
+		return n.nilParentName
+	}
+	if !n.parent.isNodeADir() {
+		log.Fatal("parent not a dir, structural error")
+	}
+	return filepath.Join(n.parent.Path(), n.Name())
+}
+
+// mySplit trims trailing separators from the directory
+// result of filepath.Split.
+func mySplit(s string) (string, string) {
+	dName, fName := filepath.Split(s)
+	return StripTrailingSeps(dName), fName
+}
+
+func (n *fsNode) addFile(name string, c []byte) (result *fsNode, err error) {
+	parent := n
+	dName, fileName := mySplit(name)
+	if dName != "" {
+		parent, err = parent.addDir(dName)
+		if err != nil {
+			return nil, err
+		}
+	}
+	if !isLegalFileNameForCreation(fileName) {
+		return nil, fmt.Errorf(
+			"illegal name '%s' in file creation", fileName)
+	}
+	result, ok := parent.dir[fileName]
+	if ok {
+		// File already exists; overwrite it.
+		result.content = c
+		return result, nil
+	}
+	result = &fsNode{
+		content: c,
+		parent:  parent,
+	}
+	parent.dir[fileName] = result
+	return result, nil
+}
+
+// Create implements FileSystem.
+// Create makes an empty file.
+func (n *fsNode) Create(path string) (result File, err error) {
+	return n.AddFile(path, []byte{})
+}
+
+// WriteFile implements FileSystem.
+func (n *fsNode) WriteFile(path string, d []byte) error {
+	_, err := n.AddFile(path, d)
+	return err
+}
+
+// AddFile adds a file and any necessary containing
+// directories to the node.
+func (n *fsNode) AddFile(
+	name string, c []byte) (result *fsNode, err error) {
+	if n.dir == nil {
+		return nil, fmt.Errorf(
+			"cannot add a file to a non-directory '%s'", n.Name())
+	}
+	return n.addFile(cleanQueryPath(name), c)
+}
+
+func (n *fsNode) addDir(path string) (result *fsNode, err error) {
+	parent := n
+	dName, subDirName := mySplit(path)
+	if dName != "" {
+		parent, err = n.addDir(dName)
+		if err != nil {
+			return nil, err
+		}
+	}
+	switch subDirName {
+	case "", SelfDir:
+		return n, nil
+	case ParentDir:
+		if n.parent == nil {
+			return nil, fmt.Errorf(
+				"cannot add a directory above '%s'", n.Path())
+		}
+		return n.parent, nil
+	default:
+		if !isLegalFileNameForCreation(subDirName) {
+			return nil, fmt.Errorf(
+				"illegal name '%s' in directory creation", subDirName)
+		}
+		result, ok := parent.dir[subDirName]
+		if ok {
+			if result.isNodeADir() {
+				// it's already there.
+				return result, nil
+			}
+			return nil, fmt.Errorf(
+				"cannot make dir '%s'; a file of that name already exists in '%s'",
+				subDirName, parent.Name())
+		}
+		result = &fsNode{
+			dir:    make(map[string]*fsNode),
+			parent: parent,
+		}
+		parent.dir[subDirName] = result
+		return result, nil
+	}
+}
+
+// Mkdir implements FileSystem.
+// Mkdir creates a directory.
+func (n *fsNode) Mkdir(path string) error {
+	_, err := n.AddDir(path)
+	return err
+}
+
+// MkdirAll implements FileSystem.
+// MkdirAll creates a directory.
+func (n *fsNode) MkdirAll(path string) error {
+	_, err := n.AddDir(path)
+	return err
+}
+
+// AddDir adds a directory to the node, not complaining
+// if it is already there.
+func (n *fsNode) AddDir(path string) (result *fsNode, err error) {
+	if n.dir == nil {
+		return nil, fmt.Errorf(
+			"cannot add a directory to file node '%s'", n.Name())
+	}
+	return n.addDir(cleanQueryPath(path))
+}
+
+// CleanedAbs implements FileSystem.
+func (n *fsNode) CleanedAbs(path string) (ConfirmedDir, string, error) {
+	node, err := n.Find(path)
+	if err != nil {
+		return "", "", errors.Wrap(err, "unable to clean")
+	}
+	if node == nil {
+		return "", "", fmt.Errorf("'%s' doesn't exist", path)
+	}
+	if node.isNodeADir() {
+		return ConfirmedDir(node.Path()), "", nil
+	}
+	return ConfirmedDir(node.parent.Path()), node.Name(), nil
+}
+
+// Exists implements FileSystem.
+// Exists returns true if the path exists.
+func (n *fsNode) Exists(path string) bool {
+	if !n.isNodeADir() {
+		return n.Name() == path
+	}
+	result, err := n.Find(path)
+	if err != nil {
+		return false
+	}
+	return result != nil
+}
+
+func cleanQueryPath(path string) string {
+	// Always ignore leading separator?
+	// Remember that filepath.Clean returns "." if
+	// given an empty string argument.
+	return filepath.Clean(StripLeadingSeps(path))
+}
+
+// Find finds the given node, else nil if not found.
+// Return error on structural/argument errors.
+func (n *fsNode) Find(path string) (*fsNode, error) {
+	if !n.isNodeADir() {
+		return nil, fmt.Errorf("can only find inside a dir")
+	}
+	if path == "" {
+		// Special case; check *before* cleaning and *before*
+		// comparison to nilParentName.
+		return nil, nil
+	}
+	if (n.parent == nil && path == n.nilParentName) || path == SelfDir {
+		// Special case
+		return n, nil
+	}
+	return n.findIt(cleanQueryPath(path))
+}
+
+func (n *fsNode) findIt(path string) (result *fsNode, err error) {
+	parent := n
+	dName, item := mySplit(path)
+	if dName != "" {
+		parent, err = n.findIt(dName)
+		if err != nil {
+			return nil, err
+		}
+		if parent == nil {
+			// all done, target doesn't exist.
+			return nil, nil
+		}
+	}
+	if !parent.isNodeADir() {
+		return nil, fmt.Errorf("'%s' is not a directory", parent.Path())
+	}
+	return parent.dir[item], nil
+}
+
+// RemoveAll implements FileSystem.
+// RemoveAll removes an item and everything it contains.
+func (n *fsNode) RemoveAll(path string) error {
+	result, err := n.Find(path)
+	if err != nil {
+		return err
+	}
+	if result == nil {
+		return fmt.Errorf("cannot find '%s' to remove it", path)
+	}
+	return result.Remove()
+}
+
+// Remove drop the node, and everything it contains, from its parent.
+func (n *fsNode) Remove() error {
+	if n.parent == nil {
+		return fmt.Errorf("cannot remove a root node")
+	}
+	if !n.parent.isNodeADir() {
+		log.Fatal("parent not a dir")
+	}
+	for key, value := range n.parent.dir {
+		if value == n {
+			delete(n.parent.dir, key)
+			return nil
+		}
+	}
+	log.Fatal("unable to find self in parent")
+	return nil
+}
+
+// isNodeADir returns true if the node is a directory.
+// Cannot collide with the poorly named "IsDir".
+func (n *fsNode) isNodeADir() bool {
+	return n.dir != nil
+}
+
+// IsDir implements FileSystem.
+// IsDir returns true if the argument resolves
+// to a directory rooted at the node.
+func (n *fsNode) IsDir(path string) bool {
+	result, err := n.Find(path)
+	if err != nil || result == nil {
+		return false
+	}
+	return result.isNodeADir()
+}
+
+// Size returns the size of the node.
+func (n *fsNode) Size() int64 {
+	if n.isNodeADir() {
+		return int64(len(n.dir))
+	}
+	return int64(len(n.content))
+}
+
+// Open implements FileSystem.
+// Open opens the node for reading (just marks it).
+func (n *fsNode) Open(path string) (File, error) {
+	result, err := n.Find(path)
+	if err != nil {
+		return nil, err
+	}
+	if result == nil {
+		return nil, fmt.Errorf("cannot find '%s' to open it", path)
+	}
+	result.open = true
+	return result, nil
+}
+
+// Close marks the node closed.
+func (n *fsNode) Close() error {
+	n.open = false
+	return nil
+}
+
+// ReadFile implements FileSystem.
+func (n *fsNode) ReadFile(path string) (c []byte, err error) {
+	result, err := n.Find(path)
+	if err != nil {
+		return nil, err
+	}
+	if result == nil {
+		return nil, fmt.Errorf("cannot find '%s' to read it", path)
+	}
+	c = make([]byte, len(result.content))
+	_, err = result.Read(c)
+	return c, err
+}
+
+// Read returns the content of the file node.
+func (n *fsNode) Read(d []byte) (c int, err error) {
+	if n.isNodeADir() {
+		return 0, fmt.Errorf(
+			"cannot read content from non-file '%s'", n.Path())
+	}
+	return copy(d, n.content), nil
+}
+
+// Write saves the contents of the argument to the file node.
+func (n *fsNode) Write(p []byte) (c int, err error) {
+	if n.isNodeADir() {
+		return 0, fmt.Errorf(
+			"cannot write content to non-file '%s'", n.Path())
+	}
+	n.content = make([]byte, len(p))
+	return copy(n.content, p), nil
+}
+
+// ContentMatches returns true if v matches fake file's content.
+func (n *fsNode) ContentMatches(v []byte) bool {
+	return bytes.Equal(v, n.content)
+}
+
+// GetContent the content of a fake file.
+func (n *fsNode) GetContent() []byte {
+	return n.content
+}
+
+// Stat returns an instance of FileInfo.
+func (n *fsNode) Stat() (os.FileInfo, error) {
+	return fileInfo{node: n}, nil
+}
+
+// Walk implements FileSystem.
+func (n *fsNode) Walk(path string, walkFn filepath.WalkFunc) error {
+	result, err := n.Find(path)
+	if err != nil {
+		return err
+	}
+	if result == nil {
+		return fmt.Errorf("cannot find '%s' to walk it", path)
+	}
+	return result.WalkMe(walkFn)
+}
+
+// Walk runs the given walkFn on each node.
+func (n *fsNode) WalkMe(walkFn filepath.WalkFunc) error {
+	fi, err := n.Stat()
+	// always visit self first
+	err = walkFn(n.Path(), fi, err)
+	if !n.isNodeADir() {
+		// it's a file, so nothing more to do
+		return err
+	}
+	// process self as a directory
+	if err == filepath.SkipDir {
+		return nil
+	}
+	// Walk is supposed to visit in lexical order.
+	for _, k := range n.sortedDirEntries() {
+		if err := n.dir[k].WalkMe(walkFn); err != nil {
+			if err == filepath.SkipDir {
+				// stop processing this directory
+				break
+			}
+			// bail out completely
+			return err
+		}
+	}
+	return nil
+}
+
+func (n *fsNode) sortedDirEntries() []string {
+	keys := make([]string, len(n.dir))
+	i := 0
+	for k := range n.dir {
+		keys[i] = k
+		i++
+	}
+	sort.Strings(keys)
+	return keys
+}
+
+// FileCount returns a count of files.
+// Directories, empty or otherwise, not counted.
+func (n *fsNode) FileCount() int {
+	count := 0
+	n.WalkMe(func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if !info.IsDir() {
+			count++
+		}
+		return nil
+	})
+	return count
+}
+
+func (n *fsNode) DebugPrint() {
+	n.WalkMe(func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			fmt.Printf("err '%v' at path %q\n", err, path)
+			return nil
+		}
+		if info.IsDir() {
+			if info.Size() == 0 {
+				fmt.Println("empty dir: " + path)
+			}
+		} else {
+			fmt.Println("     file: " + path)
+		}
+		return nil
+	})
+}
+
+var legalFileNamePattern = regexp.MustCompile("^[a-zA-Z0-9-_.]+$")
+
+// This rules enforced here should be simpler and tighter
+// than what's allowed on a real OS.
+// Should be fine for testing or in-memory purposes.
+func isLegalFileNameForCreation(n string) bool {
+	if n == "" || n == SelfDir || !legalFileNamePattern.MatchString(n) {
+		return false
+	}
+	return !strings.Contains(n, ParentDir)
+}
+
+// RegExpGlob returns a list of file paths matching the regexp.
+// Excludes directories.
+func (n *fsNode) RegExpGlob(pattern string) ([]string, error) {
+	var result []string
+	var expression = regexp.MustCompile(pattern)
+	n.WalkMe(func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if !info.IsDir() {
+			if expression.MatchString(path) {
+				result = append(result, path)
+			}
+		}
+		return nil
+	})
+	sort.Strings(result)
+	return result, nil
+}
+
+// Glob implements FileSystem.
+// Glob returns the list of file paths matching
+// per filepath.Match semantics, i.e. unlike RegExpGlob,
+// Match("foo/a*") will not match sub-sub directories of foo.
+// This is how /bin/ls behaves.
+func (n *fsNode) Glob(pattern string) ([]string, error) {
+	var result []string
+	n.WalkMe(func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if !info.IsDir() {
+			match, err := filepath.Match(pattern, path)
+			if err != nil {
+				return err
+			}
+			if match {
+				result = append(result, path)
+			}
+		}
+		return nil
+	})
+	sort.Strings(result)
+	return result, nil
+}

api/filesys/fsnode_test.go

@@ -0,0 +1,788 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package filesys
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+	"testing"
+)
+
+const content = `
+Lorem ipsum dolor sit amet,
+consectetur adipiscing elit,
+sed do eiusmod tempor incididunt
+ut labore et dolore magna aliqua.
+`
+const shortContent = "hi"
+
+var topCases = []pathCase{
+	{
+		what:   "dotdot",
+		arg:    ParentDir,
+		errStr: "illegal name '..' in file creation",
+	},
+	{
+		what:   "empty",
+		arg:    "",
+		name:   "",
+		errStr: "illegal name '.' in file creation",
+	},
+	{
+		what: "simple",
+		arg:  "bob",
+		name: "bob",
+		path: "bob",
+	},
+	{
+		what: "longer",
+		arg:  filepath.Join("longer", "bob"),
+		name: "bob",
+		path: filepath.Join("longer", "bob"),
+	},
+	{
+		what: "longer yet",
+		arg:  filepath.Join("longer", "foo", "bar", "beans", "bob"),
+		name: "bob",
+		path: filepath.Join("longer", "foo", "bar", "beans", "bob"),
+	},
+	{
+		what: "tricky",
+		arg:  filepath.Join("bob", ParentDir, "sally"),
+		name: "sally",
+		path: "sally",
+	},
+	{
+		what: "trickier",
+		arg:  filepath.Join("bob", "sally", ParentDir, ParentDir, "jean"),
+		name: "jean",
+		path: "jean",
+	},
+}
+
+func TestMakeEmptyDirInMemory(t *testing.T) {
+	n := MakeEmptyDirInMemory()
+	if !n.isNodeADir() {
+		t.Fatalf("not a directory")
+	}
+	if n.Size() != 0 {
+		t.Fatalf("unexpected size %d", n.Size())
+	}
+	if n.Name() != "" {
+		t.Fatalf("unexpected name '%s'", n.Name())
+	}
+	if n.Path() != "" {
+		t.Fatalf("unexpected path '%s'", n.Path())
+	}
+	runBasicOperations(
+		t, "MakeEmptyDirInMemory", false, topCases, n)
+}
+
+func TestMakeFsInMemory(t *testing.T) {
+	runBasicOperations(
+		t, "MakeFsInMemory", true, topCases, MakeFsInMemory())
+}
+
+//nolint:gocyclo
+func runBasicOperations(
+	t *testing.T, tName string, isFSysRooted bool,
+	cases []pathCase, fSys FileSystem) {
+	buff := make([]byte, 500)
+	for _, c := range cases {
+		err := fSys.WriteFile(c.arg, []byte(content))
+		if c.errStr != "" {
+			if err == nil {
+				t.Fatalf("%s; expected error writing to  '%s'!", c.what, c.arg)
+			}
+			if !strings.Contains(err.Error(), c.errStr) {
+				t.Fatalf("%s; expected err containing '%s', got '%v'",
+					c.what, c.errStr, err)
+			}
+			continue
+		}
+		if err != nil {
+			t.Fatalf("%s; unexpected error: %v", c.what, err)
+		}
+		if !fSys.Exists(c.path) {
+			t.Fatalf("%s; expect existence of '%s'", c.what, c.path)
+		}
+		stuff, err := fSys.ReadFile(c.path)
+		if err != nil {
+			t.Fatalf("%s; unexpected error: %v", c.what, err)
+		}
+		if string(stuff) != content {
+			t.Fatalf("%s; unexpected content '%s'", c.what, stuff)
+		}
+		f, err := fSys.Open(c.arg)
+		if err != nil {
+			t.Fatalf("%s; unexpected error: %v", c.what, err)
+		}
+		fi, err := f.Stat()
+		if err != nil {
+			t.Fatalf("%s; unexpected error: %v", c.what, err)
+		}
+		if fi.Name() != c.name {
+			t.Fatalf("%s; expected name '%s', got '%s'", c.what, c.name, fi.Name())
+		}
+		count, err := f.Read(buff)
+		if err != nil {
+			t.Fatalf("%s; unexpected error: %v", c.what, err)
+		}
+		if string(buff[:count]) != content {
+			t.Fatalf("%s; unexpected buff '%s'", c.what, buff)
+		}
+		count, err = f.Write([]byte(shortContent))
+		if err != nil {
+			t.Fatalf("%s; unexpected error: %v", c.what, err)
+		}
+		if count != len(shortContent) {
+			t.Fatalf("%s; unexpected count: %d", c.what, len(shortContent))
+		}
+		stuff, err = fSys.ReadFile(c.path)
+		if err != nil {
+			t.Fatalf("%s; unexpected error: %v", c.what, err)
+		}
+		if string(stuff) != shortContent {
+			t.Fatalf("%s; unexpected content '%s'", c.what, stuff)
+		}
+	}
+
+	var actualPaths []string
+	var err error
+	prefix := ""
+	{
+		root := SelfDir
+		if isFSysRooted {
+			root = Separator
+			prefix = Separator
+		}
+		err = fSys.Walk(root, func(path string, info os.FileInfo, err error) error {
+			if err != nil {
+				fmt.Printf("err '%v' at path %q\n", err, path)
+				return nil
+			}
+			if !info.IsDir() {
+				actualPaths = append(actualPaths, path)
+			}
+			return nil
+		})
+	}
+	if err != nil {
+		t.Fatalf("unexpected error %v", err)
+	}
+	var expectedPaths []string
+	for _, c := range cases {
+		if c.errStr == "" {
+			expectedPaths = append(expectedPaths, prefix+c.path)
+		}
+	}
+	sort.Strings(expectedPaths)
+	assertEqualStringSlices(t, expectedPaths, actualPaths, tName)
+}
+
+type pathCase struct {
+	what   string
+	arg    string
+	name   string
+	path   string
+	errStr string
+}
+
+func TestAddDir(t *testing.T) {
+	cases := []pathCase{
+		{
+			what:   "dotdot",
+			arg:    ParentDir,
+			errStr: "cannot add a directory above ''",
+		},
+		{
+			what: "empty",
+			arg:  "",
+			name: "",
+			path: "",
+		},
+		{
+			what: "simple",
+			arg:  "bob",
+			name: "bob",
+			path: "bob",
+		},
+		{
+			what: "longer",
+			arg:  filepath.Join("longer", "bob"),
+			name: "bob",
+			path: filepath.Join("longer", "bob"),
+		},
+		{
+			what: "longer yet",
+			arg:  filepath.Join("longer", "foo", "bar", "beans", "bob"),
+			name: "bob",
+			path: filepath.Join("longer", "foo", "bar", "beans", "bob"),
+		},
+		{
+			what: "tricky",
+			arg:  filepath.Join("bob", ParentDir, "sally"),
+			name: "sally",
+			path: "sally",
+		},
+		{
+			what: "trickier",
+			arg:  filepath.Join("bob", "sally", ParentDir, ParentDir, "jean"),
+			name: "jean",
+			path: "jean",
+		},
+	}
+	for _, c := range cases {
+		n := MakeEmptyDirInMemory()
+		f, err := n.AddDir(c.arg)
+		if c.errStr != "" {
+			if err == nil {
+				t.Fatalf("%s; expected error!", c.what)
+			}
+			if !strings.Contains(err.Error(), c.errStr) {
+				t.Fatalf(
+					"%s; expected error with '%s', got '%v'",
+					c.what, c.errStr, err)
+			}
+			continue
+		}
+		if err != nil {
+			t.Fatalf("%s; unexpected error: %v", c.what, err)
+		}
+		checkNode(t, c.what, f, c.name, 0, true, c.path)
+		checkOsStat(t, c.what, f, f.Name(), 0, true)
+	}
+}
+
+var bagOfCases = []pathCase{
+	{
+		what:   "empty",
+		arg:    "",
+		errStr: "illegal name '.' in file creation",
+	},
+	{
+		what: "simple",
+		arg:  "bob",
+		name: "bob",
+		path: "bob",
+	},
+	{
+		what: "longer",
+		arg:  filepath.Join("longer", "bob"),
+		name: "bob",
+		path: filepath.Join("longer", "bob"),
+	},
+	{
+		what: "longer",
+		arg:  filepath.Join("longer", "sally"),
+		name: "sally",
+		path: filepath.Join("longer", "sally"),
+	},
+	{
+		what: "even longer",
+		arg:  filepath.Join("longer", "than", "the", "other", "bob"),
+		name: "bob",
+		path: filepath.Join("longer", "than", "the", "other", "bob"),
+	},
+	{
+		what: "even longer",
+		arg:  filepath.Join("even", "much", "longer", "than", "the", "other", "bob"),
+		name: "bob",
+		path: filepath.Join("even", "much", "longer", "than", "the", "other", "bob"),
+	},
+}
+
+func TestAddFile(t *testing.T) {
+	n := MakeEmptyDirInMemory()
+	if n.FileCount() != 0 {
+		t.Fatalf("expected no files, got %d", n.FileCount())
+	}
+	expectedFileCount := 0
+	for _, c := range bagOfCases {
+		f, err := n.AddFile(c.arg, []byte(content))
+		if c.errStr != "" {
+			if err == nil {
+				t.Fatalf("%s; expected error!", c.what)
+			}
+			if !strings.Contains(err.Error(), c.errStr) {
+				t.Fatalf("%s; expected err containing '%s', got '%v'",
+					c.what, c.errStr, err)
+			}
+			continue
+		}
+		if err != nil {
+			t.Fatalf("%s; unexpected error %v", c.what, err)
+		}
+		checkNode(t, c.what, f, c.name, len(content), false, c.path)
+		checkOsStat(t, c.what, f, f.Name(), len(content), false)
+
+		result, err := n.Find(c.arg)
+		if err != nil {
+			t.Fatalf("%s; unexpected find error %v", c.what, err)
+		}
+		if result != f {
+			t.Fatalf("%s; unexpected find result %v", c.what, result)
+		}
+
+		result, err = n.Find(filepath.Join("longer", "bogus"))
+		if err != nil {
+			t.Fatalf("%s; unexpected find error %v", c.what, err)
+		}
+		if result != nil {
+			t.Fatalf("%s; unexpected find result %v", c.what, result)
+		}
+
+		expectedFileCount++
+		fc := n.FileCount()
+		if fc != expectedFileCount {
+			t.Fatalf("expected file count %d, got %d",
+				expectedFileCount, fc)
+		}
+	}
+}
+
+func checkNode(
+	t *testing.T, what string, f *fsNode, name string,
+	size int, isDir bool, path string) {
+	if f.isNodeADir() != isDir {
+		t.Fatalf("%s; unexpected isNodeADir = %v", what, f.isNodeADir())
+	}
+	if f.Size() != int64(size) {
+		t.Fatalf("%s; unexpected size %d", what, f.Size())
+	}
+	if name != f.Name() {
+		t.Fatalf("%s; expected name '%s', got '%s'", what, name, f.Name())
+	}
+	if path != f.Path() {
+		t.Fatalf("%s; expected path '%s', got '%s'", what, path, f.Path())
+	}
+}
+
+func checkOsStat(
+	t *testing.T, what string, f File, name string,
+	size int, isDir bool) {
+	info, err := f.Stat()
+	if err != nil {
+		t.Fatalf("%s; unexpected stat error %v", what, err)
+	}
+	if info.IsDir() != isDir {
+		t.Fatalf("%s; unexpected info.isNodeADir = %v", what, info.IsDir())
+	}
+	if info.Size() != int64(size) {
+		t.Fatalf("%s; unexpected info.size %d", what, info.Size())
+	}
+	if info.Name() != name {
+		t.Fatalf("%s; expected name '%s', got info.Name '%s'", what, name, info.Name())
+	}
+}
+
+var bunchOfFiles = []struct {
+	path     string
+	addAsDir bool
+}{
+	{
+		path: filepath.Join("b", "e", "a", "c", "g"),
+	},
+	{
+		path: filepath.Join("z", "r", "a", "b", "g"),
+	},
+	{
+		path: filepath.Join("b", "q", "a", "c", "g"),
+	},
+	{
+		path:     filepath.Join("b", "a", "a", "m", "g"),
+		addAsDir: true,
+	},
+	{
+		path: filepath.Join("b", "w"),
+	},
+	{
+		path: filepath.Join("b", "d", "a", "c", "m"),
+	},
+	{
+		path: filepath.Join("b", "d", "z"),
+	},
+	{
+		path: filepath.Join("b", "d", "y"),
+	},
+	{
+		path: filepath.Join("b", "d", "ignore", "c", "n"),
+	},
+	{
+		path: filepath.Join("b", "d", "x"),
+	},
+	{
+		path: filepath.Join("b", "d", "ignore", "c", "o"),
+	},
+	{
+		path: filepath.Join("b", "d", "ignore", "c", "m"),
+	},
+	{
+		path:     filepath.Join("b", "d", "a", "c", "i"),
+		addAsDir: true,
+	},
+	{
+		path: filepath.Join("x"),
+	},
+	{
+		path: filepath.Join("y"),
+	},
+	{
+		path: filepath.Join("b", "d", "a", "c", "i", "beans"),
+	},
+	{
+		path:     filepath.Join("b", "d", "a", "c", "r", "w"),
+		addAsDir: true,
+	},
+	{
+		path: filepath.Join("b", "d", "a", "c", "u"),
+	},
+}
+
+func makeLoadedFileTree(t *testing.T) *fsNode {
+	n := MakeEmptyDirInMemory()
+	var err error
+	expectedFileCount := 0
+	for _, item := range bunchOfFiles {
+		if item.addAsDir {
+			_, err = n.AddDir(item.path)
+		} else {
+			_, err = n.AddFile(item.path, []byte(content))
+			expectedFileCount++
+		}
+		if err != nil {
+			t.Fatalf("unexpected error %v", err)
+		}
+	}
+	fc := n.FileCount()
+	if fc != expectedFileCount {
+		t.Fatalf("expected file count %d, got %d",
+			expectedFileCount, fc)
+	}
+	return n
+}
+
+func TestWalkMe(t *testing.T) {
+	n := makeLoadedFileTree(t)
+	var actualPaths []string
+	err := n.WalkMe(func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			fmt.Printf("err '%v' at path %q\n", err, path)
+			return nil
+		}
+		if info.IsDir() {
+			if info.Name() == "ignore" {
+				return filepath.SkipDir
+			}
+		} else {
+			actualPaths = append(actualPaths, path)
+		}
+		return nil
+	})
+	if err != nil {
+		t.Fatalf("unexpected error %v", err)
+	}
+	var expectedPaths []string
+	for _, c := range bunchOfFiles {
+		if !c.addAsDir && !strings.Contains(c.path, "ignore") {
+			expectedPaths = append(expectedPaths, c.path)
+		}
+	}
+	sort.Strings(expectedPaths)
+	assertEqualStringSlices(t, expectedPaths, actualPaths, "testWalkMe")
+}
+
+func TestRemove(t *testing.T) {
+	n := makeLoadedFileTree(t)
+	orgCount := n.FileCount()
+
+	// Remove the "ignore" directory and everything below it.
+	path := filepath.Join("b", "d", "ignore")
+	result, err := n.Find(path)
+	if err != nil {
+		t.Fatalf("%s; unexpected error %v", path, err)
+	}
+	if result == nil {
+		t.Fatalf("%s; expected to find '%s'", path, path)
+	}
+	if !result.isNodeADir() {
+		t.Fatalf("%s; expected to find a directory", path)
+	}
+	err = result.Remove()
+	if err != nil {
+		t.Fatalf("%s; unable to remove: %v", path, err)
+	}
+	result, err = n.Find(path)
+	if err != nil {
+		// Just because it's gone doesn't mean error.
+		t.Fatalf("%s; unexpected error %v", path, err)
+	}
+	if result != nil {
+		t.Fatalf("%s; should not have been able to find '%s'", path, path)
+	}
+
+	// There were three files below "ignore".
+	orgCount -= 3
+
+	// Now drop one more for a total of four dropped.
+	result, _ = n.Find(filepath.Join("y"))
+	err = result.Remove()
+	if err != nil {
+		t.Fatalf("%s; unable to remove: %v", path, err)
+	}
+	orgCount -= 1
+
+	fc := n.FileCount()
+	if fc != orgCount {
+		t.Fatalf("expected file count %d, got %d",
+			orgCount, fc)
+	}
+}
+
+func TestExists(t *testing.T) {
+	n := makeLoadedFileTree(t)
+	path := filepath.Join("b", "d", "a")
+	if !n.Exists(path) {
+		t.Fatalf("expected existence at %s", path)
+	}
+	if !n.IsDir(path) {
+		t.Fatalf("expected directory at %s", path)
+	}
+}
+
+func TestRegExpGlob(t *testing.T) {
+	n := makeLoadedFileTree(t)
+	expected := []string{
+		filepath.Join("b", "d", "a", "c", "i", "beans"),
+		filepath.Join("b", "d", "a", "c", "m"),
+		filepath.Join("b", "d", "a", "c", "u"),
+		filepath.Join("b", "d", "ignore", "c", "m"),
+		filepath.Join("b", "d", "ignore", "c", "n"),
+		filepath.Join("b", "d", "ignore", "c", "o"),
+		filepath.Join("b", "d", "x"),
+		filepath.Join("b", "d", "y"),
+		filepath.Join("b", "d", "z"),
+	}
+	paths, err := n.RegExpGlob("b/d/*")
+	if err != nil {
+		t.Fatalf("glob error: %v", err)
+	}
+	assertEqualStringSlices(t, expected, paths, "glob test")
+}
+
+func TestGlob(t *testing.T) {
+	n := makeLoadedFileTree(t)
+	expected := []string{
+		filepath.Join("b", "d", "x"),
+		filepath.Join("b", "d", "y"),
+		filepath.Join("b", "d", "z"),
+	}
+	paths, err := n.Glob("b/d/*")
+	if err != nil {
+		t.Fatalf("glob error: %v", err)
+	}
+	assertEqualStringSlices(t, expected, paths, "glob test")
+}
+
+func assertEqualStringSlices(t *testing.T, expected, actual []string, message string) {
+	if len(expected) != len(actual) {
+		t.Fatalf(
+			"%s; unequal sizes; len(expected)=%d, len(actual)=%d\n%+v\n%+v\n",
+			message, len(expected), len(actual), expected, actual)
+	}
+	for i := range expected {
+		if expected[i] != actual[i] {
+			t.Fatalf(
+				"%s; unequal entries; expected=%s, actual=%s",
+				message, expected[i], actual[i])
+		}
+	}
+}
+
+func TestFind(t *testing.T) {
+	cases := []struct {
+		what       string
+		arg        string
+		expectDir  bool
+		expectFile bool
+		errStr     string
+	}{
+		{
+			what: "garbage",
+			arg:  "///1(*&SA",
+		},
+		{
+			what: "simple",
+			arg:  "bob",
+		},
+		{
+			what: "no directory",
+			arg:  filepath.Join("b", "rrrrrr"),
+		},
+		{
+			what:      "is a directory",
+			arg:       filepath.Join("b", "d", "ignore"),
+			expectDir: true,
+		},
+		{
+			what:       "longer, ending in file",
+			arg:        filepath.Join("b", "d", "x"),
+			expectFile: true,
+		},
+		{
+			what:       "moar longer, ending in file",
+			arg:        filepath.Join("b", "d", "a", "c", "u"),
+			expectFile: true,
+		},
+		{
+			what:      "directory",
+			arg:       filepath.Join("b"),
+			expectDir: true,
+		},
+		{
+			// Querying for the empty string could
+			// 1) be an error,
+			// 2) return no result (and no error) as with
+			//    any illegal and therefore non-existent
+			//    file name,
+			// 3) return the node itself, like running
+			//    'ls' with no argument.
+			// Going with option 2 (no result, no error),
+			// since at this low level it makes more sense
+			// if the results for the empty string query
+			// differ from the results for the "." query.
+			what: "empty name",
+			arg:  "",
+		},
+		{
+			what:      "self dir",
+			arg:       SelfDir,
+			expectDir: true,
+		},
+		{
+			what: "parent dir - doesn't exist",
+			arg:  ParentDir,
+		},
+		{
+			what: "many parents - doesn't exist",
+			arg:  filepath.Join(ParentDir, ParentDir, ParentDir),
+		},
+	}
+
+	n := makeLoadedFileTree(t)
+	for _, item := range cases {
+		result, err := n.Find(item.arg)
+		if item.errStr != "" {
+			if err == nil {
+				t.Fatalf("%s; expected error", item.what)
+			}
+			if !strings.Contains(err.Error(), item.errStr) {
+				t.Fatalf("%s; expected err containing '%s', got '%v'",
+					item.what, item.errStr, err)
+			}
+			continue
+		}
+		if err != nil {
+			t.Fatalf("%s; unexpected error: %v", item.what, err)
+		}
+		if result == nil {
+			if item.expectDir {
+				t.Fatalf(
+					"%s; expected to find directory '%s'", item.what, item.arg)
+			}
+			if item.expectFile {
+				t.Fatalf(
+					"%s; expected to find file '%s'", item.what, item.arg)
+			}
+			continue
+		}
+		if item.expectDir {
+			if !result.isNodeADir() {
+				t.Fatalf(
+					"%s; expected '%s' to be a directory", item.what, item.arg)
+			}
+			continue
+		}
+		if item.expectFile {
+			if result.isNodeADir() {
+				t.Fatalf("%s; expected '%s' to be a file", item.what, item.arg)
+			}
+			continue
+		}
+		t.Fatalf(
+			"%s; expected nothing for '%s', but got '%s'",
+			item.what, item.arg, result.Path())
+	}
+}
+
+func TestCleanedAbs(t *testing.T) {
+	cases := []struct {
+		what   string
+		full   string
+		cDir   string
+		name   string
+		errStr string
+	}{
+		{
+			what:   "empty",
+			full:   "",
+			errStr: "doesn't exist",
+		},
+		{
+			what:   "simple",
+			full:   "bob",
+			errStr: "'bob' doesn't exist",
+		},
+		{
+			what:   "no directory",
+			full:   filepath.Join("b", "rrrrrr"),
+			errStr: "'b/rrrrrr' doesn't exist",
+		},
+		{
+			what: "longer, ending in file",
+			full: filepath.Join("b", "d", "x"),
+			cDir: filepath.Join("b", "d"),
+			name: "x",
+		},
+		{
+			what: "moar longer, ending in file",
+			full: filepath.Join("b", "d", "a", "c", "u"),
+			cDir: filepath.Join("b", "d", "a", "c"),
+			name: "u",
+		},
+		{
+			what: "directory",
+			full: filepath.Join("b", "d"),
+			cDir: filepath.Join("b", "d"),
+			name: "",
+		},
+	}
+
+	n := makeLoadedFileTree(t)
+	for _, item := range cases {
+		cDir, name, err := n.CleanedAbs(item.full)
+		if item.errStr != "" {
+			if err == nil {
+				t.Fatalf("%s; expected error", item.what)
+			}
+			if !strings.Contains(err.Error(), item.errStr) {
+				t.Fatalf("%s; expected err containing '%s', got '%v'",
+					item.what, item.errStr, err)
+			}
+			continue
+		}
+		if err != nil {
+			t.Fatalf("%s; unexpected error: %v", item.what, err)
+		}
+		if cDir != ConfirmedDir(item.cDir) {
+			t.Fatalf("%s; expected cDir=%s, got '%s'", item.what, item.cDir, cDir)
+		}
+		if name != item.name {
+			t.Fatalf("%s; expected name=%s, got '%s'", item.what, item.name, name)
+		}
+	}
+}

api/filesys/fsondisk.go

@@ -1,20 +1,7 @@
-/*
-Copyright 2018 The Kubernetes Authors.
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
 
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package fs
+package filesys
 
 import (
 	"fmt"
@@ -24,43 +11,43 @@ import (
 	"path/filepath"
 )
 
-var _ FileSystem = realFS{}
+var _ FileSystem = fsOnDisk{}
 
-// realFS implements FileSystem using the local filesystem.
-type realFS struct{}
+// fsOnDisk implements FileSystem using the local filesystem.
+type fsOnDisk struct{}
 
-// MakeRealFS makes an instance of realFS.
-func MakeRealFS() FileSystem {
-	return realFS{}
+// MakeFsOnDisk makes an instance of fsOnDisk.
+func MakeFsOnDisk() FileSystem {
+	return fsOnDisk{}
 }
 
 // Create delegates to os.Create.
-func (realFS) Create(name string) (File, error) { return os.Create(name) }
+func (fsOnDisk) Create(name string) (File, error) { return os.Create(name) }
 
 // Mkdir delegates to os.Mkdir.
-func (realFS) Mkdir(name string) error {
+func (fsOnDisk) Mkdir(name string) error {
 	return os.Mkdir(name, 0777|os.ModeDir)
 }
 
 // MkdirAll delegates to os.MkdirAll.
-func (realFS) MkdirAll(name string) error {
+func (fsOnDisk) MkdirAll(name string) error {
 	return os.MkdirAll(name, 0777|os.ModeDir)
 }
 
 // RemoveAll delegates to os.RemoveAll.
-func (realFS) RemoveAll(name string) error {
+func (fsOnDisk) RemoveAll(name string) error {
 	return os.RemoveAll(name)
 }
 
 // Open delegates to os.Open.
-func (realFS) Open(name string) (File, error) { return os.Open(name) }
+func (fsOnDisk) Open(name string) (File, error) { return os.Open(name) }
 
-// CleanedAbs returns a cleaned, absolute path
-// with no symbolic links split into directory
-// and file components.  If the entire path is
-// a directory, the file component is an empty
-// string.
-func (x realFS) CleanedAbs(
+// CleanedAbs converts the given path into a
+// directory and a file name, where the directory
+// is represented as a ConfirmedDir and all that implies.
+// If the entire path is a directory, the file component
+// is an empty string.
+func (x fsOnDisk) CleanedAbs(
 	path string) (ConfirmedDir, string, error) {
 	absRoot, err := filepath.Abs(path)
 	if err != nil {
@@ -94,18 +81,18 @@ func (x realFS) CleanedAbs(
 }
 
 // Exists returns true if os.Stat succeeds.
-func (realFS) Exists(name string) bool {
+func (fsOnDisk) Exists(name string) bool {
 	_, err := os.Stat(name)
 	return err == nil
 }
 
 // Glob returns the list of matching files
-func (realFS) Glob(pattern string) ([]string, error) {
+func (fsOnDisk) Glob(pattern string) ([]string, error) {
 	return filepath.Glob(pattern)
 }
 
 // IsDir delegates to os.Stat and FileInfo.IsDir
-func (realFS) IsDir(name string) bool {
+func (fsOnDisk) IsDir(name string) bool {
 	info, err := os.Stat(name)
 	if err != nil {
 		return false
@@ -114,9 +101,14 @@ func (realFS) IsDir(name string) bool {
 }
 
 // ReadFile delegates to ioutil.ReadFile.
-func (realFS) ReadFile(name string) ([]byte, error) { return ioutil.ReadFile(name) }
+func (fsOnDisk) ReadFile(name string) ([]byte, error) { return ioutil.ReadFile(name) }
 
 // WriteFile delegates to ioutil.WriteFile with read/write permissions.
-func (realFS) WriteFile(name string, c []byte) error {
+func (fsOnDisk) WriteFile(name string, c []byte) error {
 	return ioutil.WriteFile(name, c, 0666)
 }
+
+// Walk delegates to filepath.Walk.
+func (fsOnDisk) Walk(path string, walkFn filepath.WalkFunc) error {
+	return filepath.Walk(path, walkFn)
+}

api/filesys/fsondisk_test.go

@@ -1,20 +1,7 @@
-/*
-Copyright 2018 The Kubernetes Authors.
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
 
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package fs
+package filesys_test
 
 import (
 	"io/ioutil"
@@ -23,10 +10,12 @@ import (
 	"path/filepath"
 	"reflect"
 	"testing"
+
+	. "sigs.k8s.io/kustomize/api/filesys"
 )
 
 func makeTestDir(t *testing.T) (FileSystem, string) {
-	x := MakeRealFS()
+	fSys := MakeFsOnDisk()
 	td, err := ioutil.TempDir("", "kustomize_testing_dir")
 	if err != nil {
 		t.Fatalf("unexpected error %s", err)
@@ -35,20 +24,20 @@ func makeTestDir(t *testing.T) (FileSystem, string) {
 	if err != nil {
 		t.Fatalf("unexpected error %s", err)
 	}
-	if !x.Exists(testDir) {
+	if !fSys.Exists(testDir) {
 		t.Fatalf("expected existence")
 	}
-	if !x.IsDir(testDir) {
+	if !fSys.IsDir(testDir) {
 		t.Fatalf("expected directory")
 	}
-	return x, testDir
+	return fSys, testDir
 }
 
 func TestCleanedAbs_1(t *testing.T) {
-	x, testDir := makeTestDir(t)
+	fSys, testDir := makeTestDir(t)
 	defer os.RemoveAll(testDir)
 
-	d, f, err := x.CleanedAbs("")
+	d, f, err := fSys.CleanedAbs("")
 	if err != nil {
 		t.Fatalf("unexpected err=%v", err)
 	}
@@ -65,10 +54,10 @@ func TestCleanedAbs_1(t *testing.T) {
 }
 
 func TestCleanedAbs_2(t *testing.T) {
-	x, testDir := makeTestDir(t)
+	fSys, testDir := makeTestDir(t)
 	defer os.RemoveAll(testDir)
 
-	d, f, err := x.CleanedAbs("/")
+	d, f, err := fSys.CleanedAbs("/")
 	if err != nil {
 		t.Fatalf("unexpected err=%v", err)
 	}
@@ -81,16 +70,16 @@ func TestCleanedAbs_2(t *testing.T) {
 }
 
 func TestCleanedAbs_3(t *testing.T) {
-	x, testDir := makeTestDir(t)
+	fSys, testDir := makeTestDir(t)
 	defer os.RemoveAll(testDir)
 
-	err := x.WriteFile(
+	err := fSys.WriteFile(
 		filepath.Join(testDir, "foo"), []byte(`foo`))
 	if err != nil {
 		t.Fatalf("unexpected err=%v", err)
 	}
 
-	d, f, err := x.CleanedAbs(filepath.Join(testDir, "foo"))
+	d, f, err := fSys.CleanedAbs(filepath.Join(testDir, "foo"))
 	if err != nil {
 		t.Fatalf("unexpected err=%v", err)
 	}
@@ -100,25 +89,24 @@ func TestCleanedAbs_3(t *testing.T) {
 	if f != "foo" {
 		t.Fatalf("unexpected f=%s", f)
 	}
-
 }
 
 func TestCleanedAbs_4(t *testing.T) {
-	x, testDir := makeTestDir(t)
+	fSys, testDir := makeTestDir(t)
 	defer os.RemoveAll(testDir)
 
-	err := x.MkdirAll(filepath.Join(testDir, "d1", "d2"))
+	err := fSys.MkdirAll(filepath.Join(testDir, "d1", "d2"))
 	if err != nil {
 		t.Fatalf("unexpected err=%v", err)
 	}
-	err = x.WriteFile(
+	err = fSys.WriteFile(
 		filepath.Join(testDir, "d1", "d2", "bar"),
 		[]byte(`bar`))
 	if err != nil {
 		t.Fatalf("unexpected err=%v", err)
 	}
 
-	d, f, err := x.CleanedAbs(
+	d, f, err := fSys.CleanedAbs(
 		filepath.Join(testDir, "d1", "d2"))
 	if err != nil {
 		t.Fatalf("unexpected err=%v", err)
@@ -130,7 +118,7 @@ func TestCleanedAbs_4(t *testing.T) {
 		t.Fatalf("unexpected f=%s", f)
 	}
 
-	d, f, err = x.CleanedAbs(
+	d, f, err = fSys.CleanedAbs(
 		filepath.Join(testDir, "d1", "d2", "bar"))
 	if err != nil {
 		t.Fatalf("unexpected err=%v", err)
@@ -144,26 +132,26 @@ func TestCleanedAbs_4(t *testing.T) {
 }
 
 func TestReadFilesRealFS(t *testing.T) {
-	x, testDir := makeTestDir(t)
+	fSys, testDir := makeTestDir(t)
 	defer os.RemoveAll(testDir)
 
-	err := x.WriteFile(path.Join(testDir, "foo"), []byte(`foo`))
+	err := fSys.WriteFile(path.Join(testDir, "foo"), []byte(`foo`))
 	if err != nil {
 		t.Fatalf("unexpected error %s", err)
 	}
-	if !x.Exists(path.Join(testDir, "foo")) {
+	if !fSys.Exists(path.Join(testDir, "foo")) {
 		t.Fatalf("expected foo")
 	}
-	if x.IsDir(path.Join(testDir, "foo")) {
+	if fSys.IsDir(path.Join(testDir, "foo")) {
 		t.Fatalf("expected foo not to be a directory")
 	}
 
-	err = x.WriteFile(path.Join(testDir, "bar"), []byte(`bar`))
+	err = fSys.WriteFile(path.Join(testDir, "bar"), []byte(`bar`))
 	if err != nil {
 		t.Fatalf("unexpected error %s", err)
 	}
 
-	files, err := x.Glob(path.Join("testDir", "*"))
+	files, err := fSys.Glob(path.Join("testDir", "*"))
 	expected := []string{
 		path.Join(testDir, "bar"),
 		path.Join(testDir, "foo"),

api/filesys/util.go

@@ -0,0 +1,30 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package filesys
+
+import "path/filepath"
+
+// RootedPath returns a rooted path, e.g. "/foo/bar" as
+// opposed to "foo/bar".
+func RootedPath(elem ...string) string {
+	return Separator + filepath.Join(elem...)
+}
+
+// StripTrailingSeps trims trailing filepath separators from input.
+func StripTrailingSeps(s string) string {
+	k := len(s)
+	for k > 0 && s[k-1] == filepath.Separator {
+		k--
+	}
+	return s[:k]
+}
+
+// StripLeadingSeps trims leading filepath separators from input.
+func StripLeadingSeps(s string) string {
+	k := 0
+	for k < len(s) && s[k] == filepath.Separator {
+		k++
+	}
+	return s[k:]
+}

api/filesys/util_test.go

@@ -0,0 +1,205 @@
+package filesys_test
+
+import (
+	"path/filepath"
+	"testing"
+
+	. "sigs.k8s.io/kustomize/api/filesys"
+)
+
+// Confirm behavior of filepath.Match
+func TestFilePathMatch(t *testing.T) {
+	cases := []struct {
+		pattern  string
+		path     string
+		expected bool
+	}{
+		{
+			pattern:  "*e*",
+			path:     "hey",
+			expected: true,
+		},
+		{
+			pattern:  "*e*",
+			path:     "hay",
+			expected: false,
+		},
+		{
+			pattern:  "*e*",
+			path:     filepath.Join("h", "e", "y"),
+			expected: false,
+		},
+		{
+			pattern:  "*/e/*",
+			path:     filepath.Join("h", "e", "y"),
+			expected: true,
+		},
+		{
+			pattern:  "h/e/*",
+			path:     filepath.Join("h", "e", "y"),
+			expected: true,
+		},
+		{
+			pattern:  "*/e/y",
+			path:     filepath.Join("h", "e", "y"),
+			expected: true,
+		},
+		{
+			pattern:  "*/*/*",
+			path:     filepath.Join("h", "e", "y"),
+			expected: true,
+		},
+		{
+			pattern:  "*/*/*",
+			path:     filepath.Join("h", "e", "y", "there"),
+			expected: false,
+		},
+		{
+			pattern:  "*/*/*/t*e",
+			path:     filepath.Join("h", "e", "y", "there"),
+			expected: true,
+		},
+	}
+	for _, item := range cases {
+		match, err := filepath.Match(item.pattern, item.path)
+		if err != nil {
+			t.Fatalf("unexpected err: %v", err)
+		}
+		if match != item.expected {
+			t.Fatalf("'%s' '%s' %v\n", item.pattern, item.path, match)
+		}
+	}
+}
+
+// Confirm behavior of filepath.Split
+func TestFilePathSplit(t *testing.T) {
+	cases := []struct {
+		full string
+		dir  string
+		file string
+	}{
+		{
+			full: "",
+			dir:  "",
+			file: "",
+		},
+		{
+			full: SelfDir,
+			dir:  "",
+			file: SelfDir,
+		},
+		{
+			full: "rabbit.jpg",
+			dir:  "",
+			file: "rabbit.jpg",
+		},
+		{
+			full: "/beans",
+			dir:  "/",
+			file: "beans",
+		},
+		{
+			full: "/home/foo/bar",
+			dir:  "/home/foo/",
+			file: "bar",
+		},
+		{
+			full: "/usr/local/",
+			dir:  "/usr/local/",
+			file: "",
+		},
+		{
+			full: "/usr//local//go",
+			dir:  "/usr//local//",
+			file: "go",
+		},
+	}
+	for _, p := range cases {
+		dir, file := filepath.Split(p.full)
+		if dir != p.dir || file != p.file {
+			t.Fatalf(
+				"in '%s',\ngot dir='%s' (expected '%s'),\n got file='%s' (expected %s).",
+				p.full, dir, p.dir, file, p.file)
+		}
+	}
+}
+
+func TestStripTrailingSeps(t *testing.T) {
+	cases := []struct {
+		full string
+		rem  string
+	}{
+		{
+			full: "foo",
+			rem:  "foo",
+		},
+		{
+			full: "",
+			rem:  "",
+		},
+		{
+			full: "foo/",
+			rem:  "foo",
+		},
+		{
+			full: "foo///bar///",
+			rem:  "foo///bar",
+		},
+		{
+			full: "/////",
+			rem:  "",
+		},
+		{
+			full: "/",
+			rem:  "",
+		},
+	}
+	for _, p := range cases {
+		dir := StripTrailingSeps(p.full)
+		if dir != p.rem {
+			t.Fatalf(
+				"in '%s', got dir='%s' (expected '%s')",
+				p.full, dir, p.rem)
+		}
+	}
+}
+
+func TestStripLeadingSeps(t *testing.T) {
+	cases := []struct {
+		full string
+		rem  string
+	}{
+		{
+			full: "foo",
+			rem:  "foo",
+		},
+		{
+			full: "",
+			rem:  "",
+		},
+		{
+			full: "/foo",
+			rem:  "foo",
+		},
+		{
+			full: "///foo///bar///",
+			rem:  "foo///bar///",
+		},
+		{
+			full: "/////",
+			rem:  "",
+		},
+		{
+			full: "/",
+			rem:  "",
+		},
+	}
+	for _, p := range cases {
+		dir := StripLeadingSeps(p.full)
+		if dir != p.rem {
+			t.Fatalf(
+				"in '%s', got dir='%s' (expected '%s')",
+				p.full, dir, p.rem)
+		}
+	}
+}

api/go.mod

@@ -0,0 +1,18 @@
+module sigs.k8s.io/kustomize/api
+
+go 1.13
+
+require (
+	github.com/evanphx/json-patch v4.5.0+incompatible
+	github.com/go-openapi/spec v0.19.4
+	github.com/golangci/golangci-lint v1.21.0
+	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510
+	github.com/pkg/errors v0.8.1
+	golang.org/x/tools v0.0.0-20191010075000-0337d82405ff
+	gopkg.in/yaml.v2 v2.2.4
+	k8s.io/api v0.17.0
+	k8s.io/apimachinery v0.17.0
+	k8s.io/client-go v0.17.0
+	k8s.io/kube-openapi v0.0.0-20191107075043-30be4d16710a
+	sigs.k8s.io/yaml v1.1.0
+)

api/go.sum

@@ -0,0 +1,479 @@
+cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
+github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI=
+github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0=
+github.com/Azure/go-autorest/autorest/date v0.1.0/go.mod h1:plvfp3oPSKwf2DNjlBjWF/7vwR+cUD/ELuzDCXwHUVA=
+github.com/Azure/go-autorest/autorest/mocks v0.1.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0=
+github.com/Azure/go-autorest/autorest/mocks v0.2.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0=
+github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc=
+github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk=
+github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ=
+github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
+github.com/OpenPeeDeeP/depguard v1.0.1 h1:VlW4R6jmBIv3/u1JNlawEvJMM4J+dPORPaZasQee8Us=
+github.com/OpenPeeDeeP/depguard v1.0.1/go.mod h1:xsIw86fROiiwelg+jB2uM9PiKihMMmUx/1V+TNhjQvM=
+github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
+github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI=
+github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
+github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
+github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M=
+github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
+github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
+github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
+github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
+github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
+github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
+github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
+github.com/bombsimon/wsl v1.2.5 h1:9gTOkIwVtoDZywvX802SDHokeX4kW1cKnV8ZTVAPkRs=
+github.com/bombsimon/wsl v1.2.5/go.mod h1:43lEF/i0kpXbLCeDXL9LMT8c92HyBywXb0AsgMHYngM=
+github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
+github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
+github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
+github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=
+github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
+github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
+github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
+github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE=
+github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
+github.com/davecgh/go-spew v0.0.0-20151105211317-5215b55f46b2/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
+github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
+github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM=
+github.com/elazarl/goproxy v0.0.0-20170405201442-c4fc26588b6e/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
+github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633 h1:H2pdYOb3KQ1/YsqVWoWNLQO+fusocsw354rqGTZtAgw=
+github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
+github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
+github.com/evanphx/json-patch v4.5.0+incompatible h1:ouOWdg56aJriqS0huScTkVXPC5IcNrDCXZ6OoTAWu7M=
+github.com/evanphx/json-patch v4.5.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
+github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys=
+github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
+github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
+github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
+github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
+github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
+github.com/go-critic/go-critic v0.3.5-0.20190904082202-d79a9f0c64db h1:GYXWx7Vr3+zv833u+8IoXbNnQY0AdXsxAgI0kX7xcwA=
+github.com/go-critic/go-critic v0.3.5-0.20190904082202-d79a9f0c64db/go.mod h1:+sE8vrLDS2M0pZkBk0wy6+nLdKexVDrl/jBqQOTDThA=
+github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
+github.com/go-lintpack/lintpack v0.5.2 h1:DI5mA3+eKdWeJ40nU4d6Wc26qmdG8RCi/btYq0TuRN0=
+github.com/go-lintpack/lintpack v0.5.2/go.mod h1:NwZuYi2nUHho8XEIZ6SIxihrnPoqBTDqfpXvXAN0sXM=
+github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
+github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
+github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas=
+github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8=
+github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+35s3my2LFTysnkMfxsJBAMHj/DoqoB9knIWoYG/Vk0=
+github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg=
+github.com/go-openapi/jsonpointer v0.19.3 h1:gihV7YNZK1iK6Tgwwsxo2rJbD1GTbdm72325Bq8FI3w=
+github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
+github.com/go-openapi/jsonreference v0.0.0-20160704190145-13c6e3589ad9/go.mod h1:W3Z9FmVs9qj+KR4zFKmDPGiLdk1D9Rlm7cyMvf57TTg=
+github.com/go-openapi/jsonreference v0.19.2 h1:o20suLFB4Ri0tuzpWtyHlh7E7HnkqTNLq6aR6WVNS1w=
+github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc=
+github.com/go-openapi/spec v0.0.0-20160808142527-6aced65f8501/go.mod h1:J8+jY1nAiCcj+friV/PDoE1/3eeccG9LYBs0tYvLOWc=
+github.com/go-openapi/spec v0.19.4 h1:ixzUSnHTd6hCemgtAJgluaTSGYpLNpJY4mA2DIkdOAo=
+github.com/go-openapi/spec v0.19.4/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo=
+github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dpr1UfpPtxFw+EFuQ41HhCWZfha5jSVRG7C7I=
+github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
+github.com/go-openapi/swag v0.19.5 h1:lTz6Ys4CmqqCQmZPBlbQENR1/GucA2bzYTE12Pw4tFY=
+github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
+github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
+github.com/go-toolsmith/astcast v1.0.0 h1:JojxlmI6STnFVG9yOImLeGREv8W2ocNUM+iOhR6jE7g=
+github.com/go-toolsmith/astcast v1.0.0/go.mod h1:mt2OdQTeAQcY4DQgPSArJjHCcOwlX+Wl/kwN+LbLGQ4=
+github.com/go-toolsmith/astcopy v1.0.0 h1:OMgl1b1MEpjFQ1m5ztEO06rz5CUd3oBv9RF7+DyvdG8=
+github.com/go-toolsmith/astcopy v1.0.0/go.mod h1:vrgyG+5Bxrnz4MZWPF+pI4R8h3qKRjjyvV/DSez4WVQ=
+github.com/go-toolsmith/astequal v0.0.0-20180903214952-dcb477bfacd6/go.mod h1:H+xSiq0+LtiDC11+h1G32h7Of5O3CYFJ99GVbS5lDKY=
+github.com/go-toolsmith/astequal v1.0.0 h1:4zxD8j3JRFNyLN46lodQuqz3xdKSrur7U/sr0SDS/gQ=
+github.com/go-toolsmith/astequal v1.0.0/go.mod h1:H+xSiq0+LtiDC11+h1G32h7Of5O3CYFJ99GVbS5lDKY=
+github.com/go-toolsmith/astfmt v0.0.0-20180903215011-8f8ee99c3086/go.mod h1:mP93XdblcopXwlyN4X4uodxXQhldPGZbcEJIimQHrkg=
+github.com/go-toolsmith/astfmt v1.0.0 h1:A0vDDXt+vsvLEdbMFJAUBI/uTbRw1ffOPnxsILnFL6k=
+github.com/go-toolsmith/astfmt v1.0.0/go.mod h1:cnWmsOAuq4jJY6Ct5YWlVLmcmLMn1JUPuQIHCY7CJDw=
+github.com/go-toolsmith/astinfo v0.0.0-20180906194353-9809ff7efb21/go.mod h1:dDStQCHtmZpYOmjRP/8gHHnCCch3Zz3oEgCdZVdtweU=
+github.com/go-toolsmith/astp v0.0.0-20180903215135-0af7e3c24f30/go.mod h1:SV2ur98SGypH1UjcPpCatrV5hPazG6+IfNHbkDXBRrk=
+github.com/go-toolsmith/astp v1.0.0 h1:alXE75TXgcmupDsMK1fRAy0YUzLzqPVvBKoyWV+KPXg=
+github.com/go-toolsmith/astp v1.0.0/go.mod h1:RSyrtpVlfTFGDYRbrjyWP1pYu//tSFcvdYrA8meBmLI=
+github.com/go-toolsmith/pkgload v0.0.0-20181119091011-e9e65178eee8/go.mod h1:WoMrjiy4zvdS+Bg6z9jZH82QXwkcgCBX6nOfnmdaHks=
+github.com/go-toolsmith/pkgload v1.0.0 h1:4DFWWMXVfbcN5So1sBNW9+yeiMqLFGl1wFLTL5R0Tgg=
+github.com/go-toolsmith/pkgload v1.0.0/go.mod h1:5eFArkbO80v7Z0kdngIxsRXRMTaX4Ilcwuh3clNrQJc=
+github.com/go-toolsmith/strparse v1.0.0 h1:Vcw78DnpCAKlM20kSbAyO4mPfJn/lyYA4BJUDxe2Jb4=
+github.com/go-toolsmith/strparse v1.0.0/go.mod h1:YI2nUKP9YGZnL/L1/DLFBfixrcjslWct4wyljWhSRy8=
+github.com/go-toolsmith/typep v1.0.0 h1:zKymWyA1TRYvqYrYDrfEMZULyrhcnGY3x7LDKU2XQaA=
+github.com/go-toolsmith/typep v1.0.0/go.mod h1:JSQCQMUPdRlMZFswiq3TGpNp1GMktqkR2Ns5AIQkATU=
+github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y=
+github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=
+github.com/gofrs/flock v0.0.0-20190320160742-5135e617513b h1:ekuhfTjngPhisSjOJ0QWKpPQE8/rbknHaes6WVJj5Hw=
+github.com/gofrs/flock v0.0.0-20190320160742-5135e617513b/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU=
+github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
+github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
+github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d h1:3PaI8p3seN09VjbTYC/QWlUZdZ1qS1zGjy7LH2Wt07I=
+github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/protobuf v0.0.0-20161109072736-4bd1920723d7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs=
+github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golangci/check v0.0.0-20180506172741-cfe4005ccda2 h1:23T5iq8rbUYlhpt5DB4XJkc6BU31uODLD1o1gKvZmD0=
+github.com/golangci/check v0.0.0-20180506172741-cfe4005ccda2/go.mod h1:k9Qvh+8juN+UKMCS/3jFtGICgW8O96FVaZsaxdzDkR4=
+github.com/golangci/dupl v0.0.0-20180902072040-3e9179ac440a h1:w8hkcTqaFpzKqonE9uMCefW1WDie15eSP/4MssdenaM=
+github.com/golangci/dupl v0.0.0-20180902072040-3e9179ac440a/go.mod h1:ryS0uhF+x9jgbj/N71xsEqODy9BN81/GonCZiOzirOk=
+github.com/golangci/errcheck v0.0.0-20181223084120-ef45e06d44b6 h1:YYWNAGTKWhKpcLLt7aSj/odlKrSrelQwlovBpDuf19w=
+github.com/golangci/errcheck v0.0.0-20181223084120-ef45e06d44b6/go.mod h1:DbHgvLiFKX1Sh2T1w8Q/h4NAI8MHIpzCdnBUDTXU3I0=
+github.com/golangci/go-misc v0.0.0-20180628070357-927a3d87b613 h1:9kfjN3AdxcbsZBf8NjltjWihK2QfBBBZuv91cMFfDHw=
+github.com/golangci/go-misc v0.0.0-20180628070357-927a3d87b613/go.mod h1:SyvUF2NxV+sN8upjjeVYr5W7tyxaT1JVtvhKhOn2ii8=
+github.com/golangci/goconst v0.0.0-20180610141641-041c5f2b40f3 h1:pe9JHs3cHHDQgOFXJJdYkK6fLz2PWyYtP4hthoCMvs8=
+github.com/golangci/goconst v0.0.0-20180610141641-041c5f2b40f3/go.mod h1:JXrF4TWy4tXYn62/9x8Wm/K/dm06p8tCKwFRDPZG/1o=
+github.com/golangci/gocyclo v0.0.0-20180528134321-2becd97e67ee h1:J2XAy40+7yz70uaOiMbNnluTg7gyQhtGqLQncQh+4J8=
+github.com/golangci/gocyclo v0.0.0-20180528134321-2becd97e67ee/go.mod h1:ozx7R9SIwqmqf5pRP90DhR2Oay2UIjGuKheCBCNwAYU=
+github.com/golangci/gofmt v0.0.0-20190930125516-244bba706f1a h1:iR3fYXUjHCR97qWS8ch1y9zPNsgXThGwjKPrYfqMPks=
+github.com/golangci/gofmt v0.0.0-20190930125516-244bba706f1a/go.mod h1:9qCChq59u/eW8im404Q2WWTrnBUQKjpNYKMbU4M7EFU=
+github.com/golangci/golangci-lint v1.21.0 h1:HxAxpR8Z0M8omihvQdsD3PF0qPjlqYqp2vMJzstoKeI=
+github.com/golangci/golangci-lint v1.21.0/go.mod h1:phxpHK52q7SE+5KpPnti4oZTdFCEsn/tKN+nFvCKXfk=
+github.com/golangci/ineffassign v0.0.0-20190609212857-42439a7714cc h1:gLLhTLMk2/SutryVJ6D4VZCU3CUqr8YloG7FPIBWFpI=
+github.com/golangci/ineffassign v0.0.0-20190609212857-42439a7714cc/go.mod h1:e5tpTHCfVze+7EpLEozzMB3eafxo2KT5veNg1k6byQU=
+github.com/golangci/lint-1 v0.0.0-20191013205115-297bf364a8e0 h1:MfyDlzVjl1hoaPzPD4Gpb/QgoRfSBR0jdhwGyAWwMSA=
+github.com/golangci/lint-1 v0.0.0-20191013205115-297bf364a8e0/go.mod h1:66R6K6P6VWk9I95jvqGxkqJxVWGFy9XlDwLwVz1RCFg=
+github.com/golangci/maligned v0.0.0-20180506175553-b1d89398deca h1:kNY3/svz5T29MYHubXix4aDDuE3RWHkPvopM/EDv/MA=
+github.com/golangci/maligned v0.0.0-20180506175553-b1d89398deca/go.mod h1:tvlJhZqDe4LMs4ZHD0oMUlt9G2LWuDGoisJTBzLMV9o=
+github.com/golangci/misspell v0.0.0-20180809174111-950f5d19e770 h1:EL/O5HGrF7Jaq0yNhBLucz9hTuRzj2LdwGBOaENgxIk=
+github.com/golangci/misspell v0.0.0-20180809174111-950f5d19e770/go.mod h1:dEbvlSfYbMQDtrpRMQU675gSDLDNa8sCPPChZ7PhiVA=
+github.com/golangci/prealloc v0.0.0-20180630174525-215b22d4de21 h1:leSNB7iYzLYSSx3J/s5sVf4Drkc68W2wm4Ixh/mr0us=
+github.com/golangci/prealloc v0.0.0-20180630174525-215b22d4de21/go.mod h1:tf5+bzsHdTM0bsB7+8mt0GUMvjCgwLpTapNZHU8AajI=
+github.com/golangci/revgrep v0.0.0-20180526074752-d9c87f5ffaf0 h1:HVfrLniijszjS1aiNg8JbBMO2+E1WIQ+j/gL4SQqGPg=
+github.com/golangci/revgrep v0.0.0-20180526074752-d9c87f5ffaf0/go.mod h1:qOQCunEYvmd/TLamH+7LlVccLvUH5kZNhbCgTHoBbp4=
+github.com/golangci/unconvert v0.0.0-20180507085042-28b1c447d1f4 h1:zwtduBRr5SSWhqsYNgcuWO2kFlpdOZbP0+yRjmvPGys=
+github.com/golangci/unconvert v0.0.0-20180507085042-28b1c447d1f4/go.mod h1:Izgrg8RkN3rCIMLGE9CyYmU9pY2Jer6DgANEnZ/L/cQ=
+github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
+github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
+github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/gofuzz v0.0.0-20161122191042-44d81051d367/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI=
+github.com/google/gofuzz v1.0.0 h1:A8PeW59pxE9IoFRqBp37U+mSNaQoZ46F1f0f863XSXw=
+github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
+github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
+github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
+github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4=
+github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ=
+github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
+github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d h1:7XGaL1e6bYS1yIonGp9761ExpPPV1ui0SAC59Yube9k=
+github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY=
+github.com/gophercloud/gophercloud v0.1.0/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEoIEcSTewFxm1c5g8=
+github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
+github.com/gostaticanalysis/analysisutil v0.0.0-20190318220348-4088753ea4d3 h1:JVnpOZS+qxli+rgVl98ILOXVNbW+kb5wcxeGx8ShUIw=
+github.com/gostaticanalysis/analysisutil v0.0.0-20190318220348-4088753ea4d3/go.mod h1:eEOZF4jCKGi+aprrirO9e7WKB3beBRtWgqGunKl6pKE=
+github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
+github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
+github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
+github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
+github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
+github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
+github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
+github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
+github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
+github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
+github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
+github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
+github.com/json-iterator/go v0.0.0-20180612202835-f2b4162afba3/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
+github.com/json-iterator/go v1.1.8 h1:QiWkFLKq0T7mpzwOTu6BzNDbfTE8OLrYhVKYMLF46Ok=
+github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
+github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
+github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
+github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
+github.com/kisielk/gotool v1.0.0 h1:AV2c/EiW3KqPNT9ZKl07ehoAGi4C5/01Cfbblndcapg=
+github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/klauspost/compress v1.4.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
+github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
+github.com/klauspost/cpuid v0.0.0-20180405133222-e7e905edc00e/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
+github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
+github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk=
+github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
+github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA=
+github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw=
+github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
+github.com/logrusorgru/aurora v0.0.0-20181002194514-a7b3b318ed4e/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4=
+github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDePerRcY=
+github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
+github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
+github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
+github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e h1:hB2xlXdHp/pmPZq0y3QnmWAArdw9PqbmotexnWx/FU8=
+github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
+github.com/matoous/godox v0.0.0-20190911065817-5d6d842e92eb h1:RHba4YImhrUVQDHUCe2BNSOz4tVy2yGyXhvYDvxGgeE=
+github.com/matoous/godox v0.0.0-20190911065817-5d6d842e92eb/go.mod h1:1BELzlh859Sh1c6+90blK8lbYy0kwQf1bYlBhBysy1s=
+github.com/mattn/go-colorable v0.1.4 h1:snbPLB8fVfU9iwbbo30TPtbLRzwWu6aJS6Xh4eaaviA=
+github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
+github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE=
+github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
+github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw=
+github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
+github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
+github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
+github.com/mitchellh/go-ps v0.0.0-20190716172923-621e5597135b/go.mod h1:r1VsdOzOPt1ZSrGZWFoNhsAedKnEd6r9Np1+5blZCWk=
+github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE=
+github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
+github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/reflect2 v0.0.0-20180320133207-05fbef0ca5da/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI=
+github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/mozilla/tls-observatory v0.0.0-20190404164649-a3c1b6cfecfd/go.mod h1:SrKMQvPiws7F7iqYp8/TX+IhxCYhzr6N/1yb8cwHsGk=
+github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
+github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
+github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw=
+github.com/nbutton23/zxcvbn-go v0.0.0-20180912185939-ae427f1e4c1d h1:AREM5mwr4u1ORQBMvzfzBgpsctsbQikCVpvC+tX285E=
+github.com/nbutton23/zxcvbn-go v0.0.0-20180912185939-ae427f1e4c1d/go.mod h1:o96djdrsSGy3AWPyBgZMAGfxZNfgntdJG+11KU4QvbU=
+github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
+github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.10.1 h1:q/mM8GF/n0shIN8SaAZ0V+jnLPzen6WIVZdiwrRlMlo=
+github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
+github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME=
+github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc=
+github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
+github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
+github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
+github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso=
+github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
+github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
+github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
+github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
+github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
+github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
+github.com/quasilyte/go-consistent v0.0.0-20190521200055-c6f3937de18c/go.mod h1:5STLWrekHfjyYwxBRVRXNOSewLJ3PWfDJd1VyTS21fI=
+github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
+github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
+github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
+github.com/securego/gosec v0.0.0-20191002120514-e680875ea14d h1:BzRvVq1EHuIjxpijCEKpAxzKUUMurOQ4sknehIATRh8=
+github.com/securego/gosec v0.0.0-20191002120514-e680875ea14d/go.mod h1:w5+eXa0mYznDkHaMCXA4XYffjlH+cy1oyKbfzJXa2Do=
+github.com/shirou/gopsutil v0.0.0-20190901111213-e4ec7b275ada/go.mod h1:WWnYX4lzhCH5h/3YBfyVA3VbLYjlMZZAQcW9ojMexNc=
+github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc=
+github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e h1:MZM7FHLqUHYI0Y/mQAt3d2aYa0SiNms/hFqC9qJYolM=
+github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk=
+github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041 h1:llrF3Fs4018ePo4+G/HV/uQUqEI1HMDjCeOf2V6puPc=
+github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041/go.mod h1:N5mDOmsrJOB+vfqUK+7DmDyjhSLIIBnXo9lvZJj3MWQ=
+github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
+github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4=
+github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
+github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
+github.com/sourcegraph/go-diff v0.5.1 h1:gO6i5zugwzo1RVTvgvfwCOSVegNuvnNi6bAD1QCmkHs=
+github.com/sourcegraph/go-diff v0.5.1/go.mod h1:j2dHj3m8aZgQO8lMTcTnBcXkRRRqi34cd2MNlA9u1mE=
+github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
+github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
+github.com/spf13/afero v1.2.2 h1:5jhuqJyZCZf2JRofRvN/nIFgIWNzPa3/Vz8mYylgbWc=
+github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
+github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8=
+github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
+github.com/spf13/cobra v0.0.5 h1:f0B+LkLX6DtmRH1isoNA9VTtNUK9K8xYd28JNNfOv/s=
+github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU=
+github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk=
+github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
+github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
+github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
+github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
+github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
+github.com/spf13/viper v1.4.0 h1:yXHLWeravcrgGyFSyCgdYpXQ9dR9c/WED3pg1RhxqEU=
+github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48=
+github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
+github.com/stretchr/testify v0.0.0-20151208002404-e3a8ff8ce365/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/timakin/bodyclose v0.0.0-20190930140734-f7f2e9bca95e h1:RumXZ56IrCj4CL+g1b9OL/oH0QnsF976bC8xQFYUD5Q=
+github.com/timakin/bodyclose v0.0.0-20190930140734-f7f2e9bca95e/go.mod h1:Qimiffbc6q9tBWlVV6x0P9sat/ao1xEkREYPPj9hphk=
+github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
+github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=
+github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
+github.com/ultraware/funlen v0.0.2 h1:Av96YVBwwNSe4MLR7iI/BIa3VyI7/djnto/pK3Uxbdo=
+github.com/ultraware/funlen v0.0.2/go.mod h1:Dp4UiAus7Wdb9KUZsYWZEWiRzGuM2kXM1lPbfaF6xhA=
+github.com/ultraware/whitespace v0.0.4 h1:If7Va4cM03mpgrNH9k49/VOicWpGoG70XPBFFODYDsg=
+github.com/ultraware/whitespace v0.0.4/go.mod h1:aVMh/gQve5Maj9hQ/hg+F75lr/X5A89uZnzAmWSineA=
+github.com/uudashr/gocognit v0.0.0-20190926065955-1655d0de0517 h1:ChMKTho2hWKpks/nD/FL2KqM1wuVt62oJeiE8+eFpGs=
+github.com/uudashr/gocognit v0.0.0-20190926065955-1655d0de0517/go.mod h1:j44Ayx2KW4+oB6SWMv8KsmHzZrOInQav7D3cQMJ5JUM=
+github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
+github.com/valyala/fasthttp v1.2.0/go.mod h1:4vX61m6KN+xDduDNwXrhIAVZaZaZiQ1luJk8LWSxF3s=
+github.com/valyala/quicktemplate v1.2.0/go.mod h1:EH+4AkTd43SvgIbQHYu59/cJyxDoOVRUAfrukLPuGJ4=
+github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio=
+github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
+github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
+go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
+go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
+go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
+go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
+go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
+golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY=
+golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
+golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
+golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180911220305-26e67e76b6c3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
+golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20191004110552-13f9640d40b9 h1:rjwSpXsdiK0dV8/Naq3kAw9ymfAeJIyd0upUIElB+lI=
+golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190209173611-3b5209105503/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190922100055-0a153f010e69 h1:rOhMmluY6kLMhdnrivzec6lLgaVbMHMn2ISQXJeJ5EM=
+golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
+golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20181011042414-1f849cf54d09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20181117154741-2ddaf7f79a09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190110163146-51295c7ec13a/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
+golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190311215038-5c2858a9cfe5/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190322203728-c1a832b0ad89/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190521203540-521d6ed310dd/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59 h1:QjA/9ArTfVTLfEhClDCG7SGrZkZixxWpwNCDiwJfh88=
+golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190719005602-e377ae9d6386/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI=
+golang.org/x/tools v0.0.0-20190910044552-dd2b5c81c578/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20190930201159-7c411dea38b0/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191010075000-0337d82405ff h1:XdBG6es/oFDr1HwaxkxgVve7NB281QhxgK/i4voubFs=
+golang.org/x/tools v0.0.0-20191010075000-0337d82405ff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
+google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
+google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
+google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
+gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
+gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
+gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
+gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
+gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
+gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
+gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
+gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I=
+gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.1-2019.2.3 h1:3JgtbtFHMiCmsznwGVTUWbgGov+pVqnlf1dEJTNAXeM=
+honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
+k8s.io/api v0.17.0 h1:H9d/lw+VkZKEVIUc8F3wgiQ+FUXTTr21M87jXLU7yqM=
+k8s.io/api v0.17.0/go.mod h1:npsyOePkeP0CPwyGfXDHxvypiYMJxBWAMpQxCaJ4ZxI=
+k8s.io/apimachinery v0.17.0 h1:xRBnuie9rXcPxUkDizUsGvPf1cnlZCFu210op7J7LJo=
+k8s.io/apimachinery v0.17.0/go.mod h1:b9qmWdKlLuU9EBh+06BtLcSf/Mu89rWL33naRxs1uZg=
+k8s.io/client-go v0.17.0 h1:8QOGvUGdqDMFrm9sD6IUFl256BcffynGoe80sxgTEDg=
+k8s.io/client-go v0.17.0/go.mod h1:TYgR6EUHs6k45hb6KWjVD6jFZvJV4gHDikv/It0xz+k=
+k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
+k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk=
+k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk=
+k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8=
+k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I=
+k8s.io/kube-openapi v0.0.0-20191107075043-30be4d16710a h1:UcxjrRMyNx/i/y8G7kPvLyy7rfbeuf1PYyBf973pgyU=
+k8s.io/kube-openapi v0.0.0-20191107075043-30be4d16710a/go.mod h1:1TqjTSzOxsLGIKfj0lK8EeCP7K1iUG65v09OM0/WG5E=
+k8s.io/utils v0.0.0-20191114184206-e782cd3c129f/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew=
+mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed h1:WX1yoOaKQfddO/mLzdV4wptyWgoH/6hwLs7QHTixo0I=
+mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed/go.mod h1:Xkxe497xwlCKkIaQYRfC7CSLworTXY9RMqwhhCm+8Nc=
+mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b h1:DxJ5nJdkhDlLok9K6qO+5290kphDJbHOQO1DFFFTeBo=
+mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b/go.mod h1:2odslEg/xrtNQqCYg2/jCoyKnw3vv5biOc3JnIcYfL4=
+mvdan.cc/unparam v0.0.0-20190720180237-d51796306d8f h1:Cq7MalBHYACRd6EesksG1Q8EoIAKOsiZviGKbOLIej4=
+mvdan.cc/unparam v0.0.0-20190720180237-d51796306d8f/go.mod h1:4G1h5nDURzA3bwVMZIVpwbkw+04kSxk3rAtzlimaUJw=
+sigs.k8s.io/structured-merge-diff v0.0.0-20190525122527-15d366b2352e/go.mod h1:wWxsB5ozmmv/SG7nM11ayaAW51xMvak/t1r0CSlcokI=
+sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs=
+sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=
+sourcegraph.com/sqs/pbtypes v0.0.0-20180604144634-d3ebe8f20ae4 h1:JPJh2pk3+X4lXAkZIk2RuE/7/FoK9maXw+TNPJhVS/c=
+sourcegraph.com/sqs/pbtypes v0.0.0-20180604144634-d3ebe8f20ae4/go.mod h1:ketZ/q3QxT9HOBeFhu6RdvsftgpsbFHBF5Cas6cDKZ0=

api/hasher/hasher.go

@@ -0,0 +1,52 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package hasher
+
+import (
+	"crypto/sha256"
+	"encoding/json"
+	"fmt"
+	"sort"
+)
+
+// SortArrayAndComputeHash sorts a string array and
+// returns a hash for it
+func SortArrayAndComputeHash(s []string) (string, error) {
+	sort.Strings(s)
+	data, err := json.Marshal(s)
+	if err != nil {
+		return "", err
+	}
+	return Encode(Hash(string(data)))
+}
+
+// Copied from https://github.com/kubernetes/kubernetes
+// /blob/master/pkg/kubectl/util/hash/hash.go
+func Encode(hex string) (string, error) {
+	if len(hex) < 10 {
+		return "", fmt.Errorf(
+			"input length must be at least 10")
+	}
+	enc := []rune(hex[:10])
+	for i := range enc {
+		switch enc[i] {
+		case '0':
+			enc[i] = 'g'
+		case '1':
+			enc[i] = 'h'
+		case '3':
+			enc[i] = 'k'
+		case 'a':
+			enc[i] = 'm'
+		case 'e':
+			enc[i] = 't'
+		}
+	}
+	return string(enc), nil
+}
+
+// Hash returns the hex form of the sha256 of the argument.
+func Hash(data string) string {
+	return fmt.Sprintf("%x", sha256.Sum256([]byte(data)))
+}

api/hasher/hasher_test.go

@@ -0,0 +1,41 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package hasher_test
+
+import (
+	"testing"
+
+	. "sigs.k8s.io/kustomize/api/hasher"
+)
+
+func TestSortArrayAndComputeHash(t *testing.T) {
+	array1 := []string{"a", "b", "c", "d"}
+	array2 := []string{"c", "b", "d", "a"}
+	h1, err := SortArrayAndComputeHash(array1)
+	if err != nil {
+		t.Errorf("unexpected error %v", err)
+	}
+	if h1 == "" {
+		t.Errorf("failed to hash %v", array1)
+	}
+	h2, err := SortArrayAndComputeHash(array2)
+	if err != nil {
+		t.Errorf("unexpected error %v", err)
+	}
+	if h2 == "" {
+		t.Errorf("failed to hash %v", array2)
+	}
+	if h1 != h2 {
+		t.Errorf("hash is not consistent with reordered list: %s %s", h1, h2)
+	}
+}
+
+func TestHash(t *testing.T) {
+	// hash the empty string to be sure that sha256 is being used
+	expect := "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
+	sum := Hash("")
+	if expect != sum {
+		t.Errorf("expected hash %q but got %q", expect, sum)
+	}
+}

api/ifc/ifc.go

@@ -0,0 +1,95 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+// Package ifc holds miscellaneous interfaces used by kustomize.
+package ifc
+
+import (
+	"sigs.k8s.io/kustomize/api/resid"
+	"sigs.k8s.io/kustomize/api/types"
+)
+
+// Validator provides functions to validate annotations and labels
+type Validator interface {
+	MakeAnnotationValidator() func(map[string]string) error
+	MakeAnnotationNameValidator() func([]string) error
+	MakeLabelValidator() func(map[string]string) error
+	MakeLabelNameValidator() func([]string) error
+	ValidateNamespace(string) []string
+	ErrIfInvalidKey(string) error
+	IsEnvVarName(k string) error
+}
+
+// KvLoader reads and validates KV pairs.
+type KvLoader interface {
+	Validator() Validator
+	Load(args types.KvPairSources) (all []types.Pair, err error)
+}
+
+// Loader interface exposes methods to read bytes.
+type Loader interface {
+	// Root returns the root location for this Loader.
+	Root() string
+	// New returns Loader located at newRoot.
+	New(newRoot string) (Loader, error)
+	// Load returns the bytes read from the location or an error.
+	Load(location string) ([]byte, error)
+	// Cleanup cleans the loader
+	Cleanup() error
+}
+
+// Kunstructured allows manipulation of k8s objects
+// that do not have Golang structs.
+type Kunstructured interface {
+	Map() map[string]interface{}
+	SetMap(map[string]interface{})
+	Copy() Kunstructured
+	GetFieldValue(string) (interface{}, error)
+	GetString(string) (string, error)
+	GetStringSlice(string) ([]string, error)
+	GetBool(path string) (bool, error)
+	GetFloat64(path string) (float64, error)
+	GetInt64(path string) (int64, error)
+	GetSlice(path string) ([]interface{}, error)
+	GetStringMap(path string) (map[string]string, error)
+	GetMap(path string) (map[string]interface{}, error)
+	MarshalJSON() ([]byte, error)
+	UnmarshalJSON([]byte) error
+	GetGvk() resid.Gvk
+	SetGvk(resid.Gvk)
+	GetKind() string
+	GetName() string
+	SetName(string)
+	SetNamespace(string)
+	GetLabels() map[string]string
+	SetLabels(map[string]string)
+	GetAnnotations() map[string]string
+	SetAnnotations(map[string]string)
+	MatchesLabelSelector(selector string) (bool, error)
+	MatchesAnnotationSelector(selector string) (bool, error)
+	Patch(Kunstructured) error
+}
+
+// KunstructuredFactory makes instances of Kunstructured.
+type KunstructuredFactory interface {
+	SliceFromBytes([]byte) ([]Kunstructured, error)
+	FromMap(m map[string]interface{}) Kunstructured
+	Hasher() KunstructuredHasher
+	MakeConfigMap(
+		kvLdr KvLoader,
+		options *types.GeneratorOptions,
+		args *types.ConfigMapArgs) (Kunstructured, error)
+	MakeSecret(
+		kvLdr KvLoader,
+		options *types.GeneratorOptions,
+		args *types.SecretArgs) (Kunstructured, error)
+}
+
+// KunstructuredHasher returns a hash of the argument
+// or an error.
+type KunstructuredHasher interface {
+	Hash(Kunstructured) (string, error)
+}
+
+// See core.v1.SecretTypeOpaque
+const SecretTypeOpaque = "Opaque"

api/internal/accumulator/expansion/expand.go

@@ -1,24 +1,12 @@
-/*
-Copyright 2018 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
 
 // Package expansion provides functions find and replace $(FOO) style variables in strings.
 package expansion
 
 import (
 	"bytes"
+	"fmt"
 )
 
 const (
@@ -36,15 +24,22 @@ func syntaxWrap(input string) string {
 // implements the expansion semantics defined in the expansion spec; it
 // returns the input string wrapped in the expansion syntax if no mapping
 // for the input is found.
-func MappingFuncFor(context ...map[string]string) func(string) string {
-	return func(input string) string {
+func MappingFuncFor(
+	counts map[string]int,
+	context ...map[string]interface{}) func(string) interface{} {
+	return func(input string) interface{} {
 		for _, vars := range context {
 			val, ok := vars[input]
 			if ok {
-				return val
+				counts[input]++
+				switch typedV := val.(type) {
+				case string, int64, float64, bool:
+					return typedV
+				default:
+					return syntaxWrap(input)
+				}
 			}
 		}
-
 		return syntaxWrap(input)
 	}
 }
@@ -52,7 +47,7 @@ func MappingFuncFor(context ...map[string]string) func(string) string {
 // Expand replaces variable references in the input string according to
 // the expansion spec using the given mapping function to resolve the
 // values of variables.
-func Expand(input string, mapping func(string) string) string {
+func Expand(input string, mapping func(string) interface{}) interface{} {
 	var buf bytes.Buffer
 	checkpoint := 0
 	for cursor := 0; cursor < len(input); cursor++ {
@@ -69,7 +64,14 @@ func Expand(input string, mapping func(string) string) string {
 				// We were able to read a variable name correctly;
 				// apply the mapping to the variable name and copy the
 				// bytes into the buffer
-				buf.WriteString(mapping(read))
+				mapped := mapping(read)
+				if input == syntaxWrap(read) {
+					// Preserve the type of variable
+					return mapped
+				}
+
+				// Variable is used in a middle of a string
+				buf.WriteString(fmt.Sprintf("%v", mapped))
 			} else {
 				// Not a variable name; copy the read bytes into the buffer
 				buf.WriteString(read)

api/internal/accumulator/expansion/expand_test.go

@@ -1,29 +1,24 @@
-/*
-Copyright 2018 The Kubernetes Authors.
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
 
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package expansion
+package expansion_test
 
 import (
+	"fmt"
 	"testing"
+
+	. "sigs.k8s.io/kustomize/api/internal/accumulator/expansion"
 )
 
+type expected struct {
+	count  int
+	edited string
+}
+
 func TestMapReference(t *testing.T) {
 	type env struct {
 		Name  string
-		Value string
+		Value interface{}
 	}
 	envs := []env{
 		{
@@ -38,31 +33,55 @@ func TestMapReference(t *testing.T) {
 			Name:  "BLU",
 			Value: "$(ZOO)-2",
 		},
+		{
+			Name:  "INT",
+			Value: 2,
+		},
+		{
+			Name:  "ZINT",
+			Value: "$(INT)",
+		},
+		{
+			Name:  "BOOL",
+			Value: true,
+		},
+		{
+			Name:  "ZBOOL",
+			Value: "$(BOOL)",
+		},
 	}
 
-	declaredEnv := map[string]string{
-		"FOO": "bar",
-		"ZOO": "$(FOO)-1",
-		"BLU": "$(ZOO)-2",
+	declaredEnv := map[string]interface{}{
+		"FOO":   "bar",
+		"ZOO":   "$(FOO)-1",
+		"BLU":   "$(ZOO)-2",
+		"INT":   "2",
+		"ZINT":  "$(INT)",
+		"BOOL":  "true",
+		"ZBOOL": "$(BOOL)",
 	}
 
-	serviceEnv := map[string]string{}
-
-	mapping := MappingFuncFor(declaredEnv, serviceEnv)
+	counts := make(map[string]int)
+	mapping := MappingFuncFor(counts, declaredEnv)
 
 	for _, env := range envs {
-		declaredEnv[env.Name] = Expand(env.Value, mapping)
+		declaredEnv[env.Name] = Expand(fmt.Sprintf("%v", env.Value), mapping)
 	}
 
-	expectedEnv := map[string]string{
-		"FOO": "bar",
-		"ZOO": "bar-1",
-		"BLU": "bar-1-2",
+	expectedEnv := map[string]expected{
+		"FOO":   {count: 1, edited: "bar"},
+		"ZOO":   {count: 1, edited: "bar-1"},
+		"BLU":   {count: 0, edited: "bar-1-2"},
+		"INT":   {count: 1, edited: "2"},
+		"ZINT":  {count: 0, edited: "2"},
+		"BOOL":  {count: 1, edited: "true"},
+		"ZBOOL": {count: 0, edited: "true"},
 	}
 
 	for k, v := range expectedEnv {
-		if e, a := v, declaredEnv[k]; e != a {
-			t.Errorf("Expected %v, got %v", e, a)
+		if e, a := v, declaredEnv[k]; e.edited != a || e.count != counts[k] {
+			t.Errorf("Expected %v count=%d, got %v count=%d",
+				e.edited, e.count, a, counts[k])
 		} else {
 			delete(declaredEnv, k)
 		}
@@ -74,73 +93,84 @@ func TestMapReference(t *testing.T) {
 }
 
 func TestMapping(t *testing.T) {
-	context := map[string]string{
+	context := map[string]interface{}{
 		"VAR_A":     "A",
 		"VAR_B":     "B",
 		"VAR_C":     "C",
 		"VAR_REF":   "$(VAR_A)",
 		"VAR_EMPTY": "",
 	}
-	mapping := MappingFuncFor(context)
-
-	doExpansionTest(t, mapping)
+	doExpansionTest(t, context)
 }
 
 func TestMappingDual(t *testing.T) {
-	context := map[string]string{
+	context := map[string]interface{}{
 		"VAR_A":     "A",
 		"VAR_EMPTY": "",
 	}
-	context2 := map[string]string{
+	context2 := map[string]interface{}{
 		"VAR_B":   "B",
 		"VAR_C":   "C",
 		"VAR_REF": "$(VAR_A)",
 	}
-	mapping := MappingFuncFor(context, context2)
 
-	doExpansionTest(t, mapping)
+	doExpansionTest(t, context, context2)
 }
 
-func doExpansionTest(t *testing.T, mapping func(string) string) {
+func doExpansionTest(t *testing.T, context ...map[string]interface{}) {
 	cases := []struct {
 		name     string
 		input    string
 		expected string
+		counts   map[string]int
 	}{
 		{
 			name:     "whole string",
 			input:    "$(VAR_A)",
 			expected: "A",
+			counts:   map[string]int{"VAR_A": 1},
 		},
 		{
 			name:     "repeat",
 			input:    "$(VAR_A)-$(VAR_A)",
 			expected: "A-A",
+			counts:   map[string]int{"VAR_A": 2},
+		},
+		{
+			name:     "multiple repeats",
+			input:    "$(VAR_A)-$(VAR_B)-$(VAR_B)-$(VAR_B)-$(VAR_A)",
+			expected: "A-B-B-B-A",
+			counts:   map[string]int{"VAR_A": 2, "VAR_B": 3},
 		},
 		{
 			name:     "beginning",
 			input:    "$(VAR_A)-1",
 			expected: "A-1",
+			counts:   map[string]int{"VAR_A": 1},
 		},
 		{
 			name:     "middle",
 			input:    "___$(VAR_B)___",
 			expected: "___B___",
+			counts:   map[string]int{"VAR_B": 1},
 		},
 		{
 			name:     "end",
 			input:    "___$(VAR_C)",
 			expected: "___C",
+			counts:   map[string]int{"VAR_C": 1},
 		},
 		{
 			name:     "compound",
 			input:    "$(VAR_A)_$(VAR_B)_$(VAR_C)",
 			expected: "A_B_C",
+			counts:   map[string]int{"VAR_A": 1, "VAR_B": 1, "VAR_C": 1},
 		},
 		{
 			name:     "escape & expand",
 			input:    "$$(VAR_B)_$(VAR_A)",
 			expected: "$(VAR_B)_A",
+			counts:   map[string]int{"VAR_A": 1},
 		},
 		{
 			name:     "compound escape",
@@ -156,16 +186,19 @@ func doExpansionTest(t *testing.T, mapping func(string) string) {
 			name:     "backslash escape ignored",
 			input:    "foo\\$(VAR_C)bar",
 			expected: "foo\\Cbar",
+			counts:   map[string]int{"VAR_C": 1},
 		},
 		{
 			name:     "backslash escape ignored",
 			input:    "foo\\\\$(VAR_C)bar",
 			expected: "foo\\\\Cbar",
+			counts:   map[string]int{"VAR_C": 1},
 		},
 		{
 			name:     "lots of backslashes",
 			input:    "foo\\\\\\\\$(VAR_A)bar",
 			expected: "foo\\\\\\\\Abar",
+			counts:   map[string]int{"VAR_A": 1},
 		},
 		{
 			name:     "nested var references",
@@ -181,16 +214,19 @@ func doExpansionTest(t *testing.T, mapping func(string) string) {
 			name:     "value is a reference",
 			input:    "$(VAR_REF)",
 			expected: "$(VAR_A)",
+			counts:   map[string]int{"VAR_REF": 1},
 		},
 		{
 			name:     "value is a reference x 2",
 			input:    "%%$(VAR_REF)--$(VAR_REF)%%",
 			expected: "%%$(VAR_A)--$(VAR_A)%%",
+			counts:   map[string]int{"VAR_REF": 2},
 		},
 		{
 			name:     "empty var",
 			input:    "foo$(VAR_EMPTY)bar",
 			expected: "foobar",
+			counts:   map[string]int{"VAR_EMPTY": 1},
 		},
 		{
 			name:     "unterminated expression",
@@ -236,6 +272,7 @@ func doExpansionTest(t *testing.T, mapping func(string) string) {
 			name:     "multiple (odd) operators, var defined",
 			input:    "$$$$$$$(VAR_A)",
 			expected: "$$$A",
+			counts:   map[string]int{"VAR_A": 1},
 		},
 		{
 			name:     "missing open expression",
@@ -251,16 +288,19 @@ func doExpansionTest(t *testing.T, mapping func(string) string) {
 			name:     "trailing incomplete expression not consumed",
 			input:    "$(VAR_B)_______$(A",
 			expected: "B_______$(A",
+			counts:   map[string]int{"VAR_B": 1},
 		},
 		{
 			name:     "trailing incomplete expression, no content, is not consumed",
 			input:    "$(VAR_C)_______$(",
 			expected: "C_______$(",
+			counts:   map[string]int{"VAR_C": 1},
 		},
 		{
 			name:     "operator at end of input string is preserved",
 			input:    "$(VAR_A)foobarzab$",
 			expected: "Afoobarzab$",
+			counts:   map[string]int{"VAR_A": 1},
 		},
 		{
 			name:     "shell escaped incomplete expr",
@@ -295,9 +335,31 @@ func doExpansionTest(t *testing.T, mapping func(string) string) {
 	}
 
 	for _, tc := range cases {
-		expanded := Expand(tc.input, mapping)
+		counts := make(map[string]int)
+		mapping := MappingFuncFor(counts, context...)
+		expanded := Expand(fmt.Sprintf("%v", tc.input), mapping)
 		if e, a := tc.expected, expanded; e != a {
 			t.Errorf("%v: expected %q, got %q", tc.name, e, a)
 		}
+		if len(counts) != len(tc.counts) {
+			t.Errorf("%v: len(counts)=%d != len(tc.counts)=%d",
+				tc.name, len(counts), len(tc.counts))
+		}
+		if len(tc.counts) > 0 {
+			for k, expectedCount := range tc.counts {
+				c, ok := counts[k]
+				if ok {
+					if c != expectedCount {
+						t.Errorf(
+							"%v: k=%s, expected count %d, got %d",
+							tc.name, k, expectedCount, c)
+					}
+				} else {
+					t.Errorf(
+						"%v: k=%s, expected count %d, got zero",
+						tc.name, k, expectedCount)
+				}
+			}
+		}
 	}
 }

api/internal/accumulator/loadconfigfromcrds.go

@@ -1,30 +1,21 @@
-/*
-Copyright 2018 The Kubernetes Authors.
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
 
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package config
+package accumulator
 
 import (
 	"encoding/json"
 	"strings"
 
-	"github.com/ghodss/yaml"
 	"github.com/go-openapi/spec"
+	"github.com/pkg/errors"
 	"k8s.io/kube-openapi/pkg/common"
-	"sigs.k8s.io/kustomize/pkg/gvk"
-	"sigs.k8s.io/kustomize/pkg/ifc"
+	"sigs.k8s.io/kustomize/api/filesys"
+	"sigs.k8s.io/kustomize/api/ifc"
+	"sigs.k8s.io/kustomize/api/internal/plugins/builtinconfig"
+	"sigs.k8s.io/kustomize/api/resid"
+	"sigs.k8s.io/kustomize/api/types"
+	"sigs.k8s.io/yaml"
 )
 
 type myProperties map[string]spec.Schema
@@ -32,8 +23,8 @@ type nameToApiMap map[string]common.OpenAPIDefinition
 
 // LoadConfigFromCRDs parse CRD schemas from paths into a TransformerConfig
 func LoadConfigFromCRDs(
-	ldr ifc.Loader, paths []string) (*TransformerConfig, error) {
-	tc := MakeEmptyConfig()
+	ldr ifc.Loader, paths []string) (*builtinconfig.TransformerConfig, error) {
+	tc := builtinconfig.MakeEmptyConfig()
 	for _, path := range paths {
 		content, err := ldr.Load(path)
 		if err != nil {
@@ -41,7 +32,7 @@ func LoadConfigFromCRDs(
 		}
 		m, err := makeNameToApiMap(content)
 		if err != nil {
-			return nil, err
+			return nil, errors.Wrapf(err, "unable to parse open API definition from '%s'", path)
 		}
 		otherTc, err := makeConfigFromApiMap(m)
 		if err != nil {
@@ -64,13 +55,13 @@ func makeNameToApiMap(content []byte) (result nameToApiMap, err error) {
 	return
 }
 
-func makeConfigFromApiMap(m nameToApiMap) (*TransformerConfig, error) {
-	result := MakeEmptyConfig()
+func makeConfigFromApiMap(m nameToApiMap) (*builtinconfig.TransformerConfig, error) {
+	result := builtinconfig.MakeEmptyConfig()
 	for name, api := range m {
 		if !looksLikeAk8sType(api.Schema.SchemaProps.Properties) {
 			continue
 		}
-		tc := MakeEmptyConfig()
+		tc := builtinconfig.MakeEmptyConfig()
 		err := loadCrdIntoConfig(
 			tc, makeGvkFromTypeName(name), m, name, []string{})
 		if err != nil {
@@ -87,10 +78,10 @@ func makeConfigFromApiMap(m nameToApiMap) (*TransformerConfig, error) {
 // TODO: Get Group and Version for CRD from the
 // openAPI definition once
 // "x-kubernetes-group-version-kind" is available in CRD
-func makeGvkFromTypeName(n string) gvk.Gvk {
-	names := strings.Split(n, ".")
+func makeGvkFromTypeName(n string) resid.Gvk {
+	names := strings.Split(n, filesys.SelfDir)
 	kind := names[len(names)-1]
-	return gvk.Gvk{Kind: kind}
+	return resid.Gvk{Kind: kind}
 }
 
 func looksLikeAk8sType(properties myProperties) bool {
@@ -103,10 +94,7 @@ func looksLikeAk8sType(properties myProperties) bool {
 		return false
 	}
 	_, ok = properties["metadata"]
-	if !ok {
-		return false
-	}
-	return true
+	return ok
 }
 
 const (
@@ -132,7 +120,7 @@ const (
 
 // loadCrdIntoConfig loads a CRD spec into a TransformerConfig
 func loadCrdIntoConfig(
-	theConfig *TransformerConfig, theGvk gvk.Gvk, theMap nameToApiMap,
+	theConfig *builtinconfig.TransformerConfig, theGvk resid.Gvk, theMap nameToApiMap,
 	typeName string, path []string) (err error) {
 	api, ok := theMap[typeName]
 	if !ok {
@@ -172,9 +160,9 @@ func loadCrdIntoConfig(
 					nameKey = "name"
 				}
 				err = theConfig.AddNamereferenceFieldSpec(
-					NameBackReferences{
-						Gvk: gvk.Gvk{Kind: kind, Version: version},
-						FieldSpecs: []FieldSpec{
+					builtinconfig.NameBackReferences{
+						Gvk: resid.Gvk{Kind: kind, Version: version},
+						FieldSpecs: []types.FieldSpec{
 							makeFs(theGvk, append(path, propName, nameKey))},
 					})
 				if err != nil {
@@ -191,8 +179,8 @@ func loadCrdIntoConfig(
 	return nil
 }
 
-func makeFs(in gvk.Gvk, path []string) FieldSpec {
-	return FieldSpec{
+func makeFs(in resid.Gvk, path []string) types.FieldSpec {
+	return types.FieldSpec{
 		CreateIfNotPresent: false,
 		Gvk:                in,
 		Path:               strings.Join(path, "/"),

api/internal/accumulator/loadconfigfromcrds_test.go

@@ -1,28 +1,19 @@
-/*
-Copyright 2018 The Kubernetes Authors.
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
 
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package config
+package accumulator_test
 
 import (
 	"reflect"
 	"testing"
 
-	"sigs.k8s.io/kustomize/pkg/gvk"
-	"sigs.k8s.io/kustomize/pkg/ifc"
-	"sigs.k8s.io/kustomize/pkg/internal/loadertest"
+	"sigs.k8s.io/kustomize/api/filesys"
+	"sigs.k8s.io/kustomize/api/loader"
+
+	. "sigs.k8s.io/kustomize/api/internal/accumulator"
+	"sigs.k8s.io/kustomize/api/internal/plugins/builtinconfig"
+	"sigs.k8s.io/kustomize/api/resid"
+	"sigs.k8s.io/kustomize/api/types"
 )
 
 // This defines two CRD's:  Bee and MyKind.
@@ -145,44 +136,42 @@ If it is not set we generate a secret dynamically",
 `
 )
 
-func makeLoader(t *testing.T) ifc.Loader {
-	ldr := loadertest.NewFakeLoader("/testpath")
-	err := ldr.AddFile("/testpath/crd.json", []byte(crdContent))
-	if err != nil {
-		t.Fatalf("Failed to setup fake ldr.")
-	}
-	return ldr
-}
-
 func TestLoadCRDs(t *testing.T) {
-	nbrs := []NameBackReferences{
+	nbrs := []builtinconfig.NameBackReferences{
 		{
-			Gvk: gvk.Gvk{Kind: "Secret", Version: "v1"},
-			FieldSpecs: []FieldSpec{
+			Gvk: resid.Gvk{Kind: "Secret", Version: "v1"},
+			FieldSpecs: []types.FieldSpec{
 				{
 					CreateIfNotPresent: false,
-					Gvk:                gvk.Gvk{Kind: "MyKind"},
+					Gvk:                resid.Gvk{Kind: "MyKind"},
 					Path:               "spec/secretRef/name",
 				},
 			},
 		},
 		{
-			Gvk: gvk.Gvk{Kind: "Bee", Version: "v1beta1"},
-			FieldSpecs: []FieldSpec{
+			Gvk: resid.Gvk{Kind: "Bee", Version: "v1beta1"},
+			FieldSpecs: []types.FieldSpec{
 				{
 					CreateIfNotPresent: false,
-					Gvk:                gvk.Gvk{Kind: "MyKind"},
+					Gvk:                resid.Gvk{Kind: "MyKind"},
 					Path:               "spec/beeRef/name",
 				},
 			},
 		},
 	}
 
-	expectedTc := &TransformerConfig{
+	expectedTc := &builtinconfig.TransformerConfig{
 		NameReference: nbrs,
 	}
 
-	actualTc, err := LoadConfigFromCRDs(makeLoader(t), []string{"crd.json"})
+	fSys := filesys.MakeFsInMemory()
+	fSys.WriteFile("/testpath/crd.json", []byte(crdContent))
+	ldr, err := loader.NewLoader(loader.RestrictionRootOnly, "/testpath", fSys)
+	if err != nil {
+		t.Fatalf("unexpected error:%v", err)
+	}
+
+	actualTc, err := LoadConfigFromCRDs(ldr, []string{"crd.json"})
 	if err != nil {
 		t.Fatalf("unexpected error:%v", err)
 	}

api/internal/accumulator/namereferencetransformer.go

@@ -0,0 +1,269 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package accumulator
+
+import (
+	"fmt"
+	"log"
+
+	"sigs.k8s.io/kustomize/api/internal/plugins/builtinconfig"
+	"sigs.k8s.io/kustomize/api/resid"
+	"sigs.k8s.io/kustomize/api/resmap"
+	"sigs.k8s.io/kustomize/api/resource"
+	"sigs.k8s.io/kustomize/api/transform"
+)
+
+type nameReferenceTransformer struct {
+	backRefs []builtinconfig.NameBackReferences
+}
+
+var _ resmap.Transformer = &nameReferenceTransformer{}
+
+// newNameReferenceTransformer constructs a nameReferenceTransformer
+// with a given slice of NameBackReferences.
+func newNameReferenceTransformer(br []builtinconfig.NameBackReferences) resmap.Transformer {
+	if br == nil {
+		log.Fatal("backrefs not expected to be nil")
+	}
+	return &nameReferenceTransformer{backRefs: br}
+}
+
+// Transform updates name references in resource A that
+// refer to resource B, given that B's name may have
+// changed.
+//
+// For example, a HorizontalPodAutoscaler (HPA)
+// necessarily refers to a Deployment, the thing that
+// the HPA scales. The Deployment name might change
+// (e.g. prefix added), and the reference in the HPA
+// has to be fixed.
+//
+// In the outer loop over the ResMap below, say we
+// encounter a specific HPA. Then, in scanning backrefs,
+// we encounter an entry like
+//
+//   - kind: Deployment
+//     fieldSpecs:
+//     - kind: HorizontalPodAutoscaler
+//       path: spec/scaleTargetRef/name
+//
+// This entry says that an HPA, via its
+// 'spec/scaleTargetRef/name' field, may refer to a
+// Deployment.  This match to HPA means we may need to
+// modify the value in its 'spec/scaleTargetRef/name'
+// field, by searching for the thing it refers to,
+// and getting its new name.
+//
+// As a filter, and search optimization, we compute a
+// subset of all resources that the HPA could refer to,
+// by excluding objects from other namespaces, and
+// excluding objects that don't have the same prefix-
+// suffix mods as the HPA.
+//
+// We look in this subset for all Deployment objects
+// with a resId that has a Name matching the field value
+// present in the HPA.  If no match do nothing; if more
+// than one match, it's an error.
+//
+// We overwrite the HPA name field with the value found
+// in the Deployment's name field (the name in the raw
+// object - the modified name - not the unmodified name
+// in the Deployment's resId).
+//
+// This process assumes that the name stored in a ResId
+// (the ResMap key) isn't modified by name transformers.
+// Name transformers should only modify the name in the
+// body of the resource object (the value in the ResMap).
+//
+func (o *nameReferenceTransformer) Transform(m resmap.ResMap) error {
+	// TODO: Too much looping, here and in transitive calls.
+	for _, referrer := range m.Resources() {
+		var candidates resmap.ResMap
+		for _, target := range o.backRefs {
+			for _, fSpec := range target.FieldSpecs {
+				if referrer.OrgId().IsSelected(&fSpec.Gvk) {
+					if candidates == nil {
+						candidates = m.SubsetThatCouldBeReferencedByResource(referrer)
+					}
+					err := transform.MutateField(
+						referrer.Map(),
+						fSpec.PathSlice(),
+						fSpec.CreateIfNotPresent,
+						o.getNewNameFunc(
+							// referrer could be an HPA instance,
+							// target could be Gvk for Deployment,
+							// candidate a list of resources "reachable"
+							// from the HPA.
+							referrer, target.Gvk, candidates))
+					if err != nil {
+						return err
+					}
+				}
+			}
+		}
+	}
+	return nil
+}
+
+// selectReferral picks the referral among a subset of candidates.
+// It returns the current name and namespace of the selected candidate.
+// Note that the content of the referricalCandidateSubset slice is most of the time
+// identical to the referralCandidates resmap. Still in some cases, such
+// as ClusterRoleBinding, the subset only contains the resources of a specific
+// namespace.
+func (o *nameReferenceTransformer) selectReferral(
+	oldName string,
+	referrer *resource.Resource,
+	target resid.Gvk,
+	referralCandidates resmap.ResMap,
+	referralCandidateSubset []*resource.Resource) (interface{}, interface{}, error) {
+
+	for _, res := range referralCandidateSubset {
+		id := res.OrgId()
+		if id.IsSelected(&target) && res.GetOriginalName() == oldName {
+			matches := referralCandidates.GetMatchingResourcesByOriginalId(id.Equals)
+			// If there's more than one match, there's no way
+			// to know which one to pick, so emit error.
+			if len(matches) > 1 {
+				return nil, nil, fmt.Errorf(
+					"multiple matches for %s:\n  %v",
+					id, getIds(matches))
+			}
+			// In the resource, note that it is referenced
+			// by the referrer.
+			res.AppendRefBy(referrer.CurId())
+			// Return transformed name of the object,
+			// complete with prefixes, hashes, etc.
+			return res.GetName(), res.GetNamespace(), nil
+		}
+	}
+
+	return oldName, nil, nil
+}
+
+// utility function to replace a simple string by the new name
+func (o *nameReferenceTransformer) getSimpleNameField(
+	oldName string,
+	referrer *resource.Resource,
+	target resid.Gvk,
+	referralCandidates resmap.ResMap,
+	referralCandidateSubset []*resource.Resource) (interface{}, error) {
+
+	newName, _, err := o.selectReferral(oldName, referrer, target,
+		referralCandidates, referralCandidateSubset)
+
+	return newName, err
+}
+
+// utility function to replace name field within a map[string]interface{}
+// and leverage the namespace field.
+func (o *nameReferenceTransformer) getNameAndNsStruct(
+	inMap map[string]interface{},
+	referrer *resource.Resource,
+	target resid.Gvk,
+	referralCandidates resmap.ResMap) (interface{}, error) {
+
+	// Example:
+	if _, ok := inMap["name"]; !ok {
+		return nil, fmt.Errorf(
+			"%#v is expected to contain a name field", inMap)
+	}
+	oldName, ok := inMap["name"].(string)
+	if !ok {
+		return nil, fmt.Errorf(
+			"%#v is expected to contain a name field of type string", oldName)
+	}
+
+	subset := referralCandidates.Resources()
+	if namespacevalue, ok := inMap["namespace"]; ok {
+		namespace := namespacevalue.(string)
+		bynamespace := referralCandidates.GroupedByOriginalNamespace()
+		if _, ok := bynamespace[namespace]; !ok {
+			return inMap, nil
+		}
+		subset = bynamespace[namespace]
+	}
+
+	newname, newnamespace, err := o.selectReferral(oldName, referrer, target,
+		referralCandidates, subset)
+	if err != nil {
+		return nil, err
+	}
+
+	if (newname == oldName) && (newnamespace == nil) {
+		// no candidate found.
+		return inMap, nil
+	}
+
+	inMap["name"] = newname
+	if newnamespace != "" {
+		// We don't want value "" to replace value "default" since
+		// the empty string is handled as a wild card here not default namespace
+		// by kubernetes.
+		inMap["namespace"] = newnamespace
+	}
+	return inMap, nil
+
+}
+
+func (o *nameReferenceTransformer) getNewNameFunc(
+	referrer *resource.Resource,
+	target resid.Gvk,
+	referralCandidates resmap.ResMap) func(in interface{}) (interface{}, error) {
+	return func(in interface{}) (interface{}, error) {
+		switch thing := in.(type) {
+		case string:
+			return o.getSimpleNameField(thing, referrer, target,
+				referralCandidates, referralCandidates.Resources())
+		case map[string]interface{}:
+			// Kind: ValidatingWebhookConfiguration
+			// FieldSpec is webhooks/clientConfig/service
+			return o.getNameAndNsStruct(thing, referrer, target,
+				referralCandidates)
+		case []interface{}:
+			for idx, item := range thing {
+				switch value := item.(type) {
+				case string:
+					// Kind: Role/ClusterRole
+					// FieldSpec is rules.resourceNames
+					newName, err := o.getSimpleNameField(value, referrer, target,
+						referralCandidates, referralCandidates.Resources())
+					if err != nil {
+						return nil, err
+					}
+					thing[idx] = newName
+				case map[string]interface{}:
+					// Kind: RoleBinding/ClusterRoleBinding
+					// FieldSpec is subjects
+					// Note: The corresponding fieldSpec had been changed from
+					// from path: subjects/name to just path: subjects. This is
+					// what get mutatefield to request the mapping of the whole
+					// map containing namespace and name instead of just a simple
+					// string field containing the name
+					newMap, err := o.getNameAndNsStruct(value, referrer, target,
+						referralCandidates)
+					if err != nil {
+						return nil, err
+					}
+					thing[idx] = newMap
+				default:
+					return nil, fmt.Errorf(
+						"%#v is expected to be either a []string or a []map[string]interface{}", in)
+				}
+			}
+			return in, nil
+		default:
+			return nil, fmt.Errorf(
+				"%#v is expected to be either a string or a []interface{}", in)
+		}
+	}
+}
+
+func getIds(rs []*resource.Resource) []string {
+	var result []string
+	for _, r := range rs {
+		result = append(result, r.CurId().String()+"\n")
+	}
+	return result
+}

api/internal/accumulator/refvartransformer.go

@@ -0,0 +1,113 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package accumulator
+
+import (
+	"fmt"
+
+	expansion2 "sigs.k8s.io/kustomize/api/internal/accumulator/expansion"
+
+	"sigs.k8s.io/kustomize/api/resmap"
+	"sigs.k8s.io/kustomize/api/transform"
+	"sigs.k8s.io/kustomize/api/types"
+)
+
+type refVarTransformer struct {
+	varMap            map[string]interface{}
+	replacementCounts map[string]int
+	fieldSpecs        []types.FieldSpec
+	mappingFunc       func(string) interface{}
+}
+
+// newRefVarTransformer returns a new refVarTransformer
+// that replaces $(VAR) style variables with values.
+// The fieldSpecs are the places to look for occurrences of $(VAR).
+func newRefVarTransformer(
+	varMap map[string]interface{}, fs []types.FieldSpec) *refVarTransformer {
+	return &refVarTransformer{
+		varMap:     varMap,
+		fieldSpecs: fs,
+	}
+}
+
+// replaceVars accepts as 'in' a string, or string array, which can have
+// embedded instances of $VAR style variables, e.g. a container command string.
+// The function returns the string with the variables expanded to their final
+// values.
+func (rv *refVarTransformer) replaceVars(in interface{}) (interface{}, error) {
+	switch vt := in.(type) {
+	case []interface{}:
+		var xs []interface{}
+		for _, a := range in.([]interface{}) {
+			xs = append(xs, expansion2.Expand(a.(string), rv.mappingFunc))
+		}
+		return xs, nil
+	case map[string]interface{}:
+		inMap := in.(map[string]interface{})
+		xs := make(map[string]interface{}, len(inMap))
+		for k, v := range inMap {
+			s, ok := v.(string)
+			if !ok {
+				// This field not contain a $(VAR) since it is not
+				// of string type. For instance .spec.replicas: 3 in
+				// a Deployment object
+				xs[k] = v
+			} else {
+				// This field can potentially contains a $(VAR) since it is
+				// of string type. For instance .spec.replicas: $(REPLICAS)
+				// in a Deployment object
+				xs[k] = expansion2.Expand(s, rv.mappingFunc)
+			}
+		}
+		return xs, nil
+	case interface{}:
+		s, ok := in.(string)
+		if !ok {
+			// This field not contain a $(VAR) since it is not of string type.
+			return in, nil
+		}
+		// This field can potentially contain a $(VAR) since it is
+		// of string type.
+		return expansion2.Expand(s, rv.mappingFunc), nil
+	// staticcheck erroneously claims that `case nil`
+	// is unreachable here, so suppressing it.
+	//nolint:staticcheck
+	case nil:
+		return nil, nil
+	default:
+		return "", fmt.Errorf("invalid type encountered %T", vt)
+	}
+}
+
+// UnusedVars returns slice of Var names that were unused
+// after a Transform run.
+func (rv *refVarTransformer) UnusedVars() []string {
+	var unused []string
+	for k := range rv.varMap {
+		_, ok := rv.replacementCounts[k]
+		if !ok {
+			unused = append(unused, k)
+		}
+	}
+	return unused
+}
+
+// Transform replaces $(VAR) style variables with values.
+func (rv *refVarTransformer) Transform(m resmap.ResMap) error {
+	rv.replacementCounts = make(map[string]int)
+	rv.mappingFunc = expansion2.MappingFuncFor(
+		rv.replacementCounts, rv.varMap)
+	for _, res := range m.Resources() {
+		for _, fieldSpec := range rv.fieldSpecs {
+			if res.OrgId().IsSelected(&fieldSpec.Gvk) {
+				if err := transform.MutateField(
+					res.Map(), fieldSpec.PathSlice(),
+					false, rv.replaceVars); err != nil {
+					return err
+				}
+			}
+		}
+	}
+	return nil
+}

api/internal/accumulator/refvartransformer_test.go

@@ -0,0 +1,137 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package accumulator
+
+import (
+	"reflect"
+	"testing"
+
+	"sigs.k8s.io/kustomize/api/k8sdeps/kunstruct"
+	"sigs.k8s.io/kustomize/api/resid"
+	"sigs.k8s.io/kustomize/api/resmap"
+	"sigs.k8s.io/kustomize/api/resource"
+	resmaptest_test "sigs.k8s.io/kustomize/api/testutils/resmaptest"
+	"sigs.k8s.io/kustomize/api/types"
+)
+
+func TestRefVarTransformer(t *testing.T) {
+	type given struct {
+		varMap map[string]interface{}
+		fs     []types.FieldSpec
+		res    resmap.ResMap
+	}
+	type expected struct {
+		res    resmap.ResMap
+		unused []string
+	}
+	testCases := []struct {
+		description string
+		given       given
+		expected    expected
+	}{
+		{
+			description: "var replacement in map[string]",
+			given: given{
+				varMap: map[string]interface{}{
+					"FOO": "replacementForFoo",
+					"BAR": "replacementForBar",
+					"BAZ": int64(5),
+					"BOO": true,
+				},
+				fs: []types.FieldSpec{
+					{Gvk: resid.Gvk{Version: "v1", Kind: "ConfigMap"}, Path: "data/map"},
+					{Gvk: resid.Gvk{Version: "v1", Kind: "ConfigMap"}, Path: "data/slice"},
+					{Gvk: resid.Gvk{Version: "v1", Kind: "ConfigMap"}, Path: "data/interface"},
+					{Gvk: resid.Gvk{Version: "v1", Kind: "ConfigMap"}, Path: "data/nil"},
+					{Gvk: resid.Gvk{Version: "v1", Kind: "ConfigMap"}, Path: "data/num"},
+				},
+				res: resmaptest_test.NewRmBuilder(
+					t, resource.NewFactory(kunstruct.NewKunstructuredFactoryImpl())).
+					Add(map[string]interface{}{
+						"apiVersion": "v1",
+						"kind":       "ConfigMap",
+						"metadata": map[string]interface{}{
+							"name": "cm1",
+						},
+						"data": map[string]interface{}{
+							"map": map[string]interface{}{
+								"item1": "$(FOO)",
+								"item2": "bla",
+								"item3": "$(BAZ)",
+								"item4": "$(BAZ)+$(BAZ)",
+								"item5": "$(BOO)",
+								"item6": "if $(BOO)",
+								"item7": 2019,
+							},
+							"slice": []interface{}{
+								"$(FOO)",
+								"bla",
+								"$(BAZ)",
+								"$(BAZ)+$(BAZ)",
+								"$(BOO)",
+								"if $(BOO)",
+							},
+							"interface": "$(FOO)",
+							"nil":       nil,
+							"num":       2019,
+						}}).ResMap(),
+			},
+			expected: expected{
+				res: resmaptest_test.NewRmBuilder(
+					t, resource.NewFactory(kunstruct.NewKunstructuredFactoryImpl())).
+					Add(map[string]interface{}{
+						"apiVersion": "v1",
+						"kind":       "ConfigMap",
+						"metadata": map[string]interface{}{
+							"name": "cm1",
+						},
+						"data": map[string]interface{}{
+							"map": map[string]interface{}{
+								"item1": "replacementForFoo",
+								"item2": "bla",
+								"item3": int64(5),
+								"item4": "5+5",
+								"item5": true,
+								"item6": "if true",
+								"item7": 2019,
+							},
+							"slice": []interface{}{
+								"replacementForFoo",
+								"bla",
+								int64(5),
+								"5+5",
+								true,
+								"if true",
+							},
+							"interface": "replacementForFoo",
+							"nil":       nil,
+							"num":       2019,
+						}}).ResMap(),
+				unused: []string{"BAR"},
+			},
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.description, func(t *testing.T) {
+			// arrange
+			tr := newRefVarTransformer(tc.given.varMap, tc.given.fs)
+
+			// act
+			err := tr.Transform(tc.given.res)
+
+			// assert
+			if err != nil {
+				t.Errorf("unexpected error: %v", err)
+			}
+
+			a, e := tc.given.res, tc.expected.res
+			if !reflect.DeepEqual(a, e) {
+				err = e.ErrorIfNotEqualLists(a)
+				t.Fatalf("actual doesn't match expected: \nACTUAL:\n%v\nEXPECTED:\n%v\nERR: %v", a, e, err)
+			}
+
+		})
+	}
+}

api/internal/accumulator/resaccumulator.go

@@ -0,0 +1,166 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package accumulator
+
+import (
+	"fmt"
+	"log"
+	"strings"
+
+	"sigs.k8s.io/kustomize/api/internal/plugins/builtinconfig"
+	"sigs.k8s.io/kustomize/api/resid"
+	"sigs.k8s.io/kustomize/api/resmap"
+	"sigs.k8s.io/kustomize/api/types"
+)
+
+// ResAccumulator accumulates resources and the rules
+// used to customize those resources.  It's a ResMap
+// plus stuff needed to modify the ResMap.
+type ResAccumulator struct {
+	resMap  resmap.ResMap
+	tConfig *builtinconfig.TransformerConfig
+	varSet  types.VarSet
+}
+
+func MakeEmptyAccumulator() *ResAccumulator {
+	ra := &ResAccumulator{}
+	ra.resMap = resmap.New()
+	ra.tConfig = &builtinconfig.TransformerConfig{}
+	ra.varSet = types.NewVarSet()
+	return ra
+}
+
+// ResMap returns a copy of the internal resMap.
+func (ra *ResAccumulator) ResMap() resmap.ResMap {
+	return ra.resMap.ShallowCopy()
+}
+
+// Vars returns a copy of underlying vars.
+func (ra *ResAccumulator) Vars() []types.Var {
+	return ra.varSet.AsSlice()
+}
+
+func (ra *ResAccumulator) AppendAll(
+	resources resmap.ResMap) error {
+	return ra.resMap.AppendAll(resources)
+}
+
+func (ra *ResAccumulator) AbsorbAll(
+	resources resmap.ResMap) error {
+	return ra.resMap.AbsorbAll(resources)
+}
+
+func (ra *ResAccumulator) MergeConfig(
+	tConfig *builtinconfig.TransformerConfig) (err error) {
+	ra.tConfig, err = ra.tConfig.Merge(tConfig)
+	return err
+}
+
+func (ra *ResAccumulator) GetTransformerConfig() *builtinconfig.TransformerConfig {
+	return ra.tConfig
+}
+
+func (ra *ResAccumulator) MergeVars(incoming []types.Var) error {
+	for _, v := range incoming {
+		targetId := resid.NewResIdWithNamespace(v.ObjRef.GVK(), v.ObjRef.Name, v.ObjRef.Namespace)
+		idMatcher := targetId.GvknEquals
+		if targetId.Namespace != "" || !targetId.IsNamespaceableKind() {
+			// Preserve backward compatibility. An empty namespace means
+			// wildcard search on the namespace hence we still use GvknEquals
+			idMatcher = targetId.Equals
+		}
+		matched := ra.resMap.GetMatchingResourcesByOriginalId(idMatcher)
+		if len(matched) > 1 {
+			return fmt.Errorf(
+				"found %d resId matches for var %s "+
+					"(unable to disambiguate)",
+				len(matched), v)
+		}
+		if len(matched) == 1 {
+			matched[0].AppendRefVarName(v)
+		}
+	}
+	return ra.varSet.MergeSlice(incoming)
+}
+
+func (ra *ResAccumulator) MergeAccumulator(other *ResAccumulator) (err error) {
+	err = ra.AppendAll(other.resMap)
+	if err != nil {
+		return err
+	}
+	err = ra.MergeConfig(other.tConfig)
+	if err != nil {
+		return err
+	}
+	return ra.varSet.MergeSet(other.varSet)
+}
+
+func (ra *ResAccumulator) findVarValueFromResources(v types.Var) (interface{}, error) {
+	for _, res := range ra.resMap.Resources() {
+		for _, varName := range res.GetRefVarNames() {
+			if varName == v.Name {
+				s, err := res.GetFieldValue(v.FieldRef.FieldPath)
+				if err != nil {
+					return "", fmt.Errorf(
+						"field specified in var '%v' "+
+							"not found in corresponding resource", v)
+				}
+
+				return s, nil
+			}
+		}
+	}
+
+	return "", fmt.Errorf(
+		"var '%v' cannot be mapped to a field "+
+			"in the set of known resources", v)
+}
+
+// makeVarReplacementMap returns a map of Var names to
+// their final values. The values are strings intended
+// for substitution wherever the $(var.Name) occurs.
+func (ra *ResAccumulator) makeVarReplacementMap() (map[string]interface{}, error) {
+	result := map[string]interface{}{}
+	for _, v := range ra.Vars() {
+		s, err := ra.findVarValueFromResources(v)
+		if err != nil {
+			return nil, err
+		}
+
+		result[v.Name] = s
+	}
+
+	return result, nil
+}
+
+func (ra *ResAccumulator) Transform(t resmap.Transformer) error {
+	return t.Transform(ra.resMap)
+}
+
+func (ra *ResAccumulator) ResolveVars() error {
+	replacementMap, err := ra.makeVarReplacementMap()
+	if err != nil {
+		return err
+	}
+	if len(replacementMap) == 0 {
+		return nil
+	}
+	t := newRefVarTransformer(
+		replacementMap, ra.tConfig.VarReference)
+	err = ra.Transform(t)
+	if len(t.UnusedVars()) > 0 {
+		log.Printf(
+			"well-defined vars that were never replaced: %s\n",
+			strings.Join(t.UnusedVars(), ","))
+	}
+	return err
+}
+
+func (ra *ResAccumulator) FixBackReferences() (err error) {
+	if ra.tConfig.NameReference == nil {
+		return nil
+	}
+	return ra.Transform(newNameReferenceTransformer(
+		ra.tConfig.NameReference))
+}

api/internal/accumulator/resaccumulator_test.go

@@ -0,0 +1,417 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package accumulator_test
+
+import (
+	"bytes"
+	"log"
+	"os"
+	"strings"
+	"testing"
+
+	. "sigs.k8s.io/kustomize/api/internal/accumulator"
+	"sigs.k8s.io/kustomize/api/internal/plugins/builtinconfig"
+	"sigs.k8s.io/kustomize/api/k8sdeps/kunstruct"
+	"sigs.k8s.io/kustomize/api/resid"
+	"sigs.k8s.io/kustomize/api/resmap"
+	"sigs.k8s.io/kustomize/api/resource"
+	resmaptest_test "sigs.k8s.io/kustomize/api/testutils/resmaptest"
+	"sigs.k8s.io/kustomize/api/types"
+)
+
+func makeResAccumulator(t *testing.T) (*ResAccumulator, *resource.Factory) {
+	ra := MakeEmptyAccumulator()
+	err := ra.MergeConfig(builtinconfig.MakeDefaultConfig())
+	if err != nil {
+		t.Fatalf("unexpected err: %v", err)
+	}
+	rf := resource.NewFactory(
+		kunstruct.NewKunstructuredFactoryImpl())
+	err = ra.AppendAll(
+		resmaptest_test.NewRmBuilder(t, rf).
+			Add(map[string]interface{}{
+				"apiVersion": "apps/v1",
+				"kind":       "Deployment",
+				"metadata": map[string]interface{}{
+					"name": "deploy1",
+				},
+				"spec": map[string]interface{}{
+					"template": map[string]interface{}{
+						"spec": map[string]interface{}{
+							"containers": []interface{}{
+								map[string]interface{}{
+									"command": []interface{}{
+										"myserver",
+										"--somebackendService $(SERVICE_ONE)",
+										"--yetAnother $(SERVICE_TWO)",
+									},
+								},
+							},
+						},
+					},
+				}}).
+			Add(map[string]interface{}{
+				"apiVersion": "v1",
+				"kind":       "Service",
+				"metadata": map[string]interface{}{
+					"name": "backendOne",
+				}}).
+			Add(map[string]interface{}{
+				"apiVersion": "v1",
+				"kind":       "Service",
+				"metadata": map[string]interface{}{
+					"name": "backendTwo",
+				}}).ResMap())
+	if err != nil {
+		t.Fatalf("unexpected err: %v", err)
+	}
+	return ra, rf
+}
+
+func TestResolveVarsHappy(t *testing.T) {
+	ra, _ := makeResAccumulator(t)
+	err := ra.MergeVars([]types.Var{
+		{
+			Name: "SERVICE_ONE",
+			ObjRef: types.Target{
+				Gvk:  resid.Gvk{Version: "v1", Kind: "Service"},
+				Name: "backendOne"},
+		},
+		{
+			Name: "SERVICE_TWO",
+			ObjRef: types.Target{
+				Gvk:  resid.Gvk{Version: "v1", Kind: "Service"},
+				Name: "backendTwo"},
+		},
+	})
+	if err != nil {
+		t.Fatalf("unexpected err: %v", err)
+	}
+	err = ra.ResolveVars()
+	if err != nil {
+		t.Fatalf("unexpected err: %v", err)
+	}
+	c := getCommand(find("deploy1", ra.ResMap()))
+	if c != "myserver --somebackendService backendOne --yetAnother backendTwo" {
+		t.Fatalf("unexpected command: %s", c)
+	}
+}
+
+func TestResolveVarsOneUnused(t *testing.T) {
+	ra, _ := makeResAccumulator(t)
+	err := ra.MergeVars([]types.Var{
+		{
+			Name: "SERVICE_ONE",
+			ObjRef: types.Target{
+				Gvk:  resid.Gvk{Version: "v1", Kind: "Service"},
+				Name: "backendOne"},
+		},
+		{
+			Name: "SERVICE_UNUSED",
+			ObjRef: types.Target{
+				Gvk:  resid.Gvk{Version: "v1", Kind: "Service"},
+				Name: "backendTwo"},
+		},
+	})
+	if err != nil {
+		t.Fatalf("unexpected err: %v", err)
+	}
+	var buf bytes.Buffer
+	log.SetOutput(&buf)
+	defer func() {
+		log.SetOutput(os.Stderr)
+	}()
+	err = ra.ResolveVars()
+	if err != nil {
+		t.Fatalf("unexpected err: %v", err)
+	}
+	expectLog(t, buf, "well-defined vars that were never replaced: SERVICE_UNUSED")
+	c := getCommand(find("deploy1", ra.ResMap()))
+	if c != "myserver --somebackendService backendOne --yetAnother $(SERVICE_TWO)" {
+		t.Fatalf("unexpected command: %s", c)
+	}
+}
+
+func expectLog(t *testing.T, log bytes.Buffer, expect string) {
+	if !strings.Contains(log.String(), expect) {
+		t.Fatalf("expected log containing '%s', got '%s'", expect, log.String())
+	}
+}
+
+func TestResolveVarsVarNeedsDisambiguation(t *testing.T) {
+	ra, rf := makeResAccumulator(t)
+
+	rm0 := resmap.New()
+	err := rm0.Append(
+		rf.FromMap(
+			map[string]interface{}{
+				"apiVersion": "v1",
+				"kind":       "Service",
+				"metadata": map[string]interface{}{
+					"name":      "backendOne",
+					"namespace": "fooNamespace",
+				},
+			}))
+	if err != nil {
+		t.Fatalf("unexpected err: %v", err)
+	}
+	err = ra.AppendAll(rm0)
+	if err != nil {
+		t.Fatalf("unexpected err: %v", err)
+	}
+
+	err = ra.MergeVars([]types.Var{
+		{
+			Name: "SERVICE_ONE",
+			ObjRef: types.Target{
+				Gvk:  resid.Gvk{Version: "v1", Kind: "Service"},
+				Name: "backendOne",
+			},
+		},
+	})
+	if err == nil {
+		t.Fatalf("expected error")
+	}
+	if !strings.Contains(
+		err.Error(), "unable to disambiguate") {
+		t.Fatalf("unexpected err: %v", err)
+	}
+}
+
+func makeNamespacedConfigMapWithDataProviderValue(
+	namespace string,
+	value string,
+) map[string]interface{} {
+	return map[string]interface{}{
+		"apiVersion": "v1",
+		"kind":       "ConfigMap",
+		"metadata": map[string]interface{}{
+			"name":      "environment",
+			"namespace": namespace,
+		},
+		"data": map[string]interface{}{
+			"provider": value,
+		},
+	}
+}
+
+func makeVarToNamepaceAndPath(
+	name string,
+	namespace string,
+	path string,
+) types.Var {
+	return types.Var{
+		Name: name,
+		ObjRef: types.Target{
+			Gvk:       resid.Gvk{Version: "v1", Kind: "ConfigMap"},
+			Name:      "environment",
+			Namespace: namespace,
+		},
+		FieldRef: types.FieldSelector{FieldPath: path},
+	}
+}
+
+func TestResolveVarConflicts(t *testing.T) {
+	rf := resource.NewFactory(kunstruct.NewKunstructuredFactoryImpl())
+
+	// create configmaps in foo and bar namespaces with `data.provider` values.
+	fooAws := makeNamespacedConfigMapWithDataProviderValue("foo", "aws")
+	barAws := makeNamespacedConfigMapWithDataProviderValue("bar", "aws")
+	barGcp := makeNamespacedConfigMapWithDataProviderValue("bar", "gcp")
+
+	// create two variables with (apparently) conflicting names that point to
+	// fieldpaths that could be generalized.
+	varFoo := makeVarToNamepaceAndPath("PROVIDER", "foo", "data.provider")
+	varBar := makeVarToNamepaceAndPath("PROVIDER", "bar", "data.provider")
+
+	// create accumulators holding apparently conflicting vars that are not
+	// actually in conflict because they point to the same concrete value.
+	rm0 := resmap.New()
+	rm0.Append(rf.FromMap(fooAws))
+	ac0 := MakeEmptyAccumulator()
+	ac0.AppendAll(rm0)
+	ac0.MergeVars([]types.Var{varFoo})
+
+	rm1 := resmap.New()
+	rm1.Append(rf.FromMap(barAws))
+	ac1 := MakeEmptyAccumulator()
+	ac1.AppendAll(rm1)
+	ac1.MergeVars([]types.Var{varBar})
+
+	// validate that two vars of the same name which reference the same concrete
+	// value do not produce a conflict.
+	err := ac0.MergeAccumulator(ac1)
+	if err == nil {
+		t.Fatalf("see bug gh-1600")
+	}
+
+	// create an accumulator will have an actually conflicting value with the
+	// two above (because it contains a variable whose name is used in the other
+	// accumulators AND whose concrete values are different).
+	rm2 := resmap.New()
+	rm2.Append(rf.FromMap(barGcp))
+	ac2 := MakeEmptyAccumulator()
+	ac2.AppendAll(rm2)
+	ac2.MergeVars([]types.Var{varBar})
+	err = ac1.MergeAccumulator(ac2)
+	if err == nil {
+		t.Fatalf("dupe vars w/ different concrete values should conflict")
+	}
+}
+
+func TestResolveVarsGoodResIdBadField(t *testing.T) {
+	ra, _ := makeResAccumulator(t)
+	err := ra.MergeVars([]types.Var{
+		{
+			Name: "SERVICE_ONE",
+			ObjRef: types.Target{
+				Gvk:  resid.Gvk{Version: "v1", Kind: "Service"},
+				Name: "backendOne"},
+			FieldRef: types.FieldSelector{FieldPath: "nope_nope_nope"},
+		},
+	})
+	if err != nil {
+		t.Fatalf("unexpected err: %v", err)
+	}
+	err = ra.ResolveVars()
+	if err == nil {
+		t.Fatalf("expected error")
+	}
+	if !strings.Contains(
+		err.Error(),
+		"not found in corresponding resource") {
+		t.Fatalf("unexpected err: %v", err)
+	}
+}
+
+func TestResolveVarsUnmappableVar(t *testing.T) {
+	ra, _ := makeResAccumulator(t)
+	err := ra.MergeVars([]types.Var{
+		{
+			Name: "SERVICE_THREE",
+			ObjRef: types.Target{
+				Gvk:  resid.Gvk{Version: "v1", Kind: "Service"},
+				Name: "doesNotExist"},
+		},
+	})
+	if err != nil {
+		t.Fatalf("unexpected err: %v", err)
+	}
+	err = ra.ResolveVars()
+	if err == nil {
+		t.Fatalf("expected error")
+	}
+	if !strings.Contains(
+		err.Error(),
+		"cannot be mapped to a field in the set of known resources") {
+		t.Fatalf("unexpected err: %v", err)
+	}
+}
+
+func TestResolveVarsWithNoambiguation(t *testing.T) {
+	ra1, rf := makeResAccumulator(t)
+	err := ra1.MergeVars([]types.Var{
+		{
+			Name: "SERVICE_ONE",
+			ObjRef: types.Target{
+				Gvk:  resid.Gvk{Version: "v1", Kind: "Service"},
+				Name: "backendOne",
+			},
+		},
+	})
+	if err != nil {
+		t.Fatalf("unexpected err: %v", err)
+	}
+
+	// Create another accumulator having a resource with different prefix
+	ra2 := MakeEmptyAccumulator()
+
+	m := resmaptest_test.NewRmBuilder(t, rf).
+		Add(map[string]interface{}{
+			"apiVersion": "apps/v1",
+			"kind":       "Deployment",
+			"metadata": map[string]interface{}{
+				"name": "deploy2",
+			},
+			"spec": map[string]interface{}{
+				"template": map[string]interface{}{
+					"spec": map[string]interface{}{
+						"containers": []interface{}{
+							map[string]interface{}{
+								"command": []interface{}{
+									"myserver",
+									"--somebackendService $(SUB_SERVICE_ONE)",
+								},
+							},
+						},
+					},
+				},
+			}}).
+		Add(map[string]interface{}{
+			"apiVersion": "v1",
+			"kind":       "Service",
+			"metadata": map[string]interface{}{
+				"name": "backendOne",
+			}}).ResMap()
+
+	// Make it seem like this resource
+	// went through a prefix transformer.
+	r := m.GetByIndex(1)
+	r.AddNamePrefix("sub-")
+	r.SetName("sub-backendOne") // original name remains "backendOne"
+
+	err = ra2.AppendAll(m)
+	if err != nil {
+		t.Fatalf("unexpected err: %v", err)
+	}
+
+	err = ra2.MergeVars([]types.Var{
+		{
+			Name: "SUB_SERVICE_ONE",
+			ObjRef: types.Target{
+				Gvk:  resid.Gvk{Version: "v1", Kind: "Service"},
+				Name: "backendOne",
+			},
+		},
+	})
+	if err != nil {
+		t.Fatalf("unexpected err: %v", err)
+	}
+	err = ra1.MergeAccumulator(ra2)
+	if err != nil {
+		t.Fatalf("unexpected err: %v", err)
+	}
+
+	err = ra1.ResolveVars()
+	if err != nil {
+		t.Fatalf("unexpected err: %v", err)
+	}
+}
+
+func find(name string, resMap resmap.ResMap) *resource.Resource {
+	for _, r := range resMap.Resources() {
+		if r.GetName() == name {
+			return r
+		}
+	}
+	return nil
+}
+
+// Assumes arg is a deployment, returns the command of first container.
+func getCommand(r *resource.Resource) string {
+	var m map[string]interface{}
+	var c []interface{}
+	m, _ = r.Map()["spec"].(map[string]interface{})
+	m, _ = m["template"].(map[string]interface{})
+	m, _ = m["spec"].(map[string]interface{})
+	c, _ = m["containers"].([]interface{})
+	m, _ = c[0].(map[string]interface{})
+
+	cmd, _ := m["command"].([]interface{})
+	n := make([]string, len(cmd))
+	for i, v := range cmd {
+		n[i] = v.(string)
+	}
+	return strings.Join(n, " ")
+}

api/internal/crawl/README.md

@@ -0,0 +1,428 @@
+## What is this?
+### In short
+Be the GoDoc.org of k8s configuration files.
+
+### More explicitly
+Support k8s document indexing from open-source configurations in order to make
+it easy for people to learn to use a new feature, explore k8s configs in a
+central hub, and see some metrics about kustomize use.
+
+We want people to be able to support three main classes of queries:
+
+1. Structured document queries: how should I use the following fields
+   - Grace periods: `spec:template:spec:terminationGracePeriod`?
+   - Kustomize inline patch: `patches:patch`?
+
+2. Key value queries: how should I use this more specific use case of a
+   structure configuration.
+   - HorizontalPodAutoScalers: `kind=HorizontalPodAutoScaler`?
+   - Patches on StatefulSets: `patches:target:kind=StatefulSet`?
+
+3. Full text search: search the comments and the document text from any
+   type of k8s config file.
+
+## Road map
+There is a lot that can be added in order to improve the state of this
+application. Some more details along with general thoughts and comments can be
+found in the Roadmap.md file in this directory. This README contains only
+what can be considered as mostly complete and iterable parts of this project.
+
+## Running this project
+Everything is configured using kubernetes, so it should be easy for people to
+spin this up on any k8s cluster. Everything should just work (TM).
+
+The config files live in the `config` directory.
+
+```
+config
+├── base
+│   └── kustomization.yaml
+├── crawler
+│   ├── base
+│   │   ├── github_api_secret.txt
+│   │   └── kustomization.yaml
+│   ├── cronjob
+│   │   ├── cronjob.yaml
+│   │   └── kustomization.yaml
+│   └── job
+│       ├── job.yaml
+│       └── kustomization.yaml
+├── elastic
+│   └── ...
+├── redis
+│   ├── document_keystore
+│   │   ├── kustomization.yaml
+│   │   ├── redis.yaml
+│   │   └── service.yaml
+│   └── http_cache
+│       ├── kustomization.yaml
+│       ├── redis.yaml
+│       └── service.yaml
+├── webapp
+│   ├── backend
+│   │   ├── deployment.yaml
+│   │   ├── kustomization.yaml
+│   │   └── service.yaml
+│   └── frontend
+│       ├── deployment.yaml
+│       ├── kustomization.yaml
+│       └── service.yaml
+└── schema_files
+    └── kustomization_index
+        ├── es_index_mappings.json
+        └── es_index_settings.json
+```
+
+To get everything up and running you have to:
+
+1. Get some instance of elasticsearch working... and configure the
+   configmapGenerator in `config/base` to point to the right endpoint(s). The
+   configurations that need this value to be populated are the following:
+    - `config/crawler/cronjob` to run periodic crawls.
+    - `config/crawler/job` to run crawls on demand.
+    - `config/webapp/backend` to run the search server.
+
+2. Configure the elasticsearch indices:
+```
+kustomize build config/schema_files/kustomization_index | kubectl apply -f -
+```
+This will run a `curl` command that reads json data from a ConfigMap. This will
+setup the schema.  If you want to make more complex modifications to the
+schema, you should refer to the elastic docs to figure out whether the mapping
+can be added to the current index, or whether you will need to copy the
+existing index into a different one with the appropriate mappings. Modifications
+can be made by using the elasticsearch go library and writing a simple program,
+or it can be made with any http command to the appropriate server endpoint from
+within the cluster. Unfortunately I did not have the time to write a few helper
+tools for this. Feel free to contact me if you need help with modifying
+elasticsearch configs, I'm by no means an expert, but I can try to help.
+
+3. (Optional) run the redis http chache for the crawler:
+```
+kubectl apply -k config/redis/http_cache
+```
+  This will create a deployment for the cache, and a service. The crawler should
+  be configured to connect to the `http_cache` if it exists, but you can always
+  check the logs to make sure it connects, and that the identifiers match in the
+  crawler configuration and for the service endpoint.
+
+  The please be aware that the cache does not have a persistent volume.
+
+4. Configure the main redis instance:
+```
+kubectl apply -k config/redis/document_keystore
+```
+  This will create a StatefulSet with a volume of 4GiB for a redis instance.
+
+5. Get an access token from GitHub.
+
+To be able to kindly ask GitHub for it's data on k8s config files, you'll need
+to create an access\_token. From my understanding, this is the only way to do
+these code search queries (without first specifying a repository).
+
+To generate a token, go to your GitHub's account in Settings > Developer
+Settings > Personal access tokens. It should look like this.
+
+![GitHub Token 1](
+https://sigs.k8s.io/kustomize/internal/tools/pictures/github_token.png)
+
+From here you want to generate a new token and  have the following
+configuration:
+
+![GitHub Token 1](
+https://sigs.k8s.io/kustomize/internal/tools/pictures/token_config.png)
+
+If you have uses for any other data from this token, (org data, or something
+else) you can pick and choose, but be careful since it can grant this
+application access to your notifications, etc. However, any such extension
+is explicitly a non-goal and would not be maintained by this project.
+
+6. Launch the crawler:
+```
+kustomize build config/crawler/cronjob | kubectl apply -f -
+```
+This will periodically run the crawler every day according to the cron timing
+rules in the cronjob.yaml file.
+
+Instead, to get the crawler running now, you can run:
+```
+kustomize build config/crawler/cronjob | kubectl apply -f -
+```
+which will launch a non-periodic version of the crawler. It will take a few
+minutes for the crawler to split the search, but then config files should
+start to get populated within 20 minutes. It may take a while to do the
+first crawl, since it has to fetch rate-limited endpoints for each new file it
+finds. It should get significantly faster to update in the future.
+
+5. Launch the search backend
+```
+kustomize build config/webapp/backend | kubectl apply -f -
+```
+
+6. Launch the search frontend
+```
+kustomize build config/webapp/frontend | kubectl apply -f -
+```
+
+## Notes about the components
+
+### Elasticsearch
+I will add a basic working setup soon. I just did the lazy thing and used an
+already packaged solution. Most clouds will provide their own elastic
+environments, however, Elasticsearch is also working on their own
+implementation of a
+![k8s operator](https://www.elastic.co/elasticsearch-kubernetes), which might
+be worth checking out. Please note that it comes with its own license
+agreement.
+
+### Redis
+There are two Redis instances that are used in this application.
+
+One of them is configured to have on disk persistence, so make sure to have
+that set up in your kubernetes cluster. Also note that it is running on a
+single master node (i.e. it does not automatically shard keys to multiple head
+nodes as part of a highly available cluster). Since it's storing a sparse
+graph, I can't imagine this being much of an issue, but it's probably worth
+mentioning.
+
+The other Redis instance is running as a HTTP (RFC 7234) cache for etags from
+GitHub (or any other document store from which we could crawl/index). This one
+does not require full persistent storage on disk. The caching strategy is an
+LRU cache which is probably a good starting point. It might be worth it to
+investigate other cache policies, but I think LRU will work well since
+documents may or may not expire anyway, and the amount of memory allocated for
+keys is fairly large, so eviction of frequently used documents seems unlikely
+anyway.
+
+### Nginx + Angular
+There is a Dockerfile included for generating the container image with Nginx
+(using the default package) and adding all of the supporting compiled angular
+files. Any modifications to the code-base should be compatible with this setup,
+so all that's needed is to rebuild the container image, and possibly modify
+the image tags in the k8s file.
+
+### Supporting Go binaries
+There are a few go binaries that each have their own Dockerfile to build
+containers in which to run them on k8s, namely the crawler and the search
+service. Their configurations are not optimal (read: needs to be cleaned up),
+but they are functional.
+
+## Technical details
+
+### Overall design and imlpementation
+
+There are a few components that are all running together in order to get
+the overall application to work smoothly. This section will provide a brief
+overview of each component with the following sections going into more details.
+
+The overall structure is outlined in the following figure:
+![overview](
+https://sigs.k8s.io/kustomize/internal/tools/pictures/sys_arch.png)
+
+#### Crawler
+The leftmost component consists of a crawler with an http cache of GitHub
+queries does two things, it first looks at the list of documents in
+elasticsearch and tries to update them. In doing so, it maintains a set of
+newly updated files to exclude them from other parts of the crawl.
+
+To find newly added documents, the crawler crawls any new dependencies
+introduced in the document updating step and it also queries GitHub for the
+most recently indexed kustomization.\* files. Each new file will be processed
+for efficient text queries and put into the document index. Any new dependency
+will also incur more crawl operations. Finally, a graphical
+representation of the documents and their dependencies is built in Redis to be
+used for graph algorithms such as PageRank and component analysis.
+
+#### Data library
+There are a few helper libaries for dealing with Elasticsearch, Redis and
+documents. This is not persistent, nor is it centralized. They act as small
+components that help to package common pieces of code. Eventually it may make
+sense to merge all of it together and make a proper persistent model around
+this while providing an external API for document insertion/deletion. But
+that is definitely out of scope in terms of getting this to run. However
+there are limitations with the current model in terms of minimizing the
+API surface for the different components of the application. For now this
+problem is mostly mitigated by having the query server only connected to
+a data node of the Elasticsearch cluster, but the problem of knowing what
+is accessible and what isn't is left to the programmer instead of being
+clearly and explicitly supported by the API.
+
+#### Server
+Uses the data library to communicate with the data store and answer queries.
+Processes the user entered text queries into somewhat optimized elasticsearch
+queries. Provides a few endpoints to get different metrics and to eventually
+allow for registration of remote repositories.
+
+This application has an exposing service in order to allow users of the
+application access to queries and the results.
+
+#### Nginx + Angular
+Communicates directly with the backend server to forward user queries and
+their results. Presents the results on an interface. It's still pretty simple
+looking but it seems usable (to me).
+
+
+### Crawling GitHub
+With the use of API keys, GitHub allows account owners to search for files
+using their API.
+
+The search endpoints allow for the use of metadata search
+that is fairly useful/powerful. For instance they provide a `filename:` keyword
+that permits us to look for `kustomization.yaml`, `kustomization.yml`, etc.
+This enables the fetching of a list of kustomization documents, from which
+we can get the actual content from another endpoint
+(raw.githubusercontent.com).
+
+However, the search API is fairly limited. There is a restriction to the number
+of documents that can be retrieved from this method. One possible way to
+mitigate this would be to periodically query GitHub for results, sorted by the
+last indexed time. This would allow you to collect most documents from this
+point forwards. The downside to this is that it may require a large number of
+requests to their API since you cannot know when new files will be added.
+Furthermore, there is a possibility that you would not be able to get all of
+files either, depending on the velocity of growth.
+
+The approach that was taken to mitigate this is to use the `filesize:` keyword
+and to shard the search space into contiguous buckets of appropriate size in
+order to get all of the documents. This is fairly efficient, since you can find
+a good enough way to shard the documents in
+`lg(max file size) * number of documents / 1000` API queries. Moreover, since
+queries are paginated with at most 100 results per query, this solution is
+competitive with getting the optimal (non-contiguous) sharding of result sets.
+Furthermore, filesize queries can be cached to minimize the total number of
+queries called to the API in order to shard the search space. This is done by
+querying for file size intervals that always start with 0..X and binary
+searching over the `filesize:` space. This will allow you to reuse a lot of
+queries when you're looking for the next range, since it is upper bounded and
+lower bounded to a smaller number of queries within a range that has also been
+queried. I think this is only true because filesizes are power law distributed,
+so searches will typically require less queries as they progress from left to
+right.
+
+However, this method in no way depends on intervals of the form 0..X, as
+the number of documents in the many intervals of the range search could be
+added together to also make this work. This approach just seemed simpler to
+implement, maintain, and debug so it was preferred.
+
+To get an idea of how efficient this method is, to shard the search space of
+7000 documents, it will only take ~90 API range queries which should only take
+a few minutes. While actually fetching the documents and their relevant
+metadata (creation time, etc.) will take several hours. Furthermore, this
+could be made more efficient if a prior distribution is approximated.
+This prior could be scaled to the number of documents that need to be fetched,
+and then finding a shard that has an adequate number of requests, will only
+take a few queries per shard. It could probably be supported in a constant
+number of size queries if the size of each shard is halved which shouldn't
+have terrible performance impact for the retrieval. However, there where
+more pressing things to implement. I might revisit this later.
+
+### Document Indexing and Processing
+In order to support simple text queries the structured documents must be
+processed in some way that makes searching them easy. The current method
+is to recursively traverse the map of configurations to generate each sub-path
+and each key-value pair for the leaf nodes of the recursion tree.
+
+However, note that this means that a document has to be valid yaml/json
+format in order for indexing to happen. The rest of the document is treated
+as mostly text and uses default text settings from Elasticsearch.
+
+What this means is that for the following yaml document:
+
+```yaml
+resources:
+- service.yaml
+- deployment.yaml
+
+configmapGenerator:
+- name: app-configuration
+  files:
+  - config.yaml
+
+patchesJson6902:
+- target:
+    version: v1
+    kind: StatefulSet
+    name: ss-name
+  path: ss-patch.yaml
+- target:
+    version: v1
+    kind: Deployment
+    name: dep-name
+  path: dep-patch.yaml
+```
+
+the following flattened structure would look like:
+```
+{
+  "identifiers": [
+    "resources",
+    "configmapGenerator",
+    "configmapGenerator:name",
+    "configmapGenerator:files",
+    "patchesJson6902",
+    "patchesJson6902:target",
+    "patchesJson6902:target:version",
+    "patchesJson6902:target:kind",
+    "patchesJson6902:target:name",
+    "patchesJson6902:path",
+  ],
+  "values": [
+    "resources=service.yaml",
+    "resources=deployment.yaml",
+    "configmapGenerator:name=app-configuration",
+    "configmapGenerator:files=config.yaml",
+    "patchesJson6902:target:version=v1",
+    "patchesJson6902:target:kind=StatefulSet",
+    "patchesJson6902:target:name=ss-name",
+    "patchesJson6902:path=ss-patch.yaml",
+    "patchesJson6902:target:kind=Deployment",
+    "patchesJson6902:target:name=dep-name",
+    "patchesJson6902:path=dep-patch.yaml",
+  ],
+  ...
+}
+```
+
+Note that unique paths and values are deduplicated.
+
+On the search side, exact queries will be prioritized, but the document paths
+and key=value pairs will also be analyzed with 3-grams to have some amount of
+fuzzy search. The reason that a Levenshtein-Distance was not used instead, is due
+to searching multiple fields at the same time, which is a use case where
+Elasticsearch does not support proper fuzzy searching.
+
+### Document Search
+Given a text query, each token is considered separately. Each token will be fed
+through a handful of analyzers on the Elasticsearch side, and will be compared
+with the reverse document index of each document fields. It will then determine
+the best matching documents. Text ordering is largely insignificant. This makes
+sense for the structured search, but may leave room for improvement for the
+text only search within the document.
+
+Each token _must_ be matched, so each white space character acts as a
+conjunction of individual queries. There are also ways of telling
+Elasticsearch that some things _should_ match, but I think for now it makes
+more sense to leave it as is.
+
+I think this behavior is sufficient to make the search feel fairly intuitive
+while providing support for fairly complex use cases.
+
+### Metrics Computation
+From the each kustomization document that is indexed, we can find it's
+resources that are publicly available. This includes other kustomizations.
+From this, we can build a directed graph of dependencies and reverse
+dependencies.
+
+This opens up the possibility to add a plethora of graph metrics that can
+give the project maintainers feedback and insight into how people are using
+their tools.
+
+Some of these are useful such as getting an idea for how large the dependency
+graphs actually grow in practice, and can be used to find _popular_
+kustomizations within the corpus. This lends itself to implementing PageRank
+to help bubble up popular results as good search results. I unfortunately
+did not have the time to implement the algorithm, but I do plan to revisit
+this sometime soon to add a few good and efficient implementations of useful
+graph algorithms that would be useful to have. See the Roadmap.md for a more
+complete list of features that could be added and how I think they could be
+implemented.

api/internal/crawl/ROADMAP.md

@@ -0,0 +1,176 @@
+# Road map and comments about this work
+
+From working on this project, here is a collection of thoughts and suggestions
+for future improvements. For any questions about this, or to request help do
+not hesitate to contact @damienr74 on GitHub, my email should be listed.
+
+I think this project has the potential for the K8s community to promote best
+practices. If this becomes popular, It could become easier to find
+*subjectively good* configurations. This can act as a way to guide newcomers
+to k8s config features that are easy to maintain, practical, and tested in some
+real world environment. However, a lot of work remains to be made if this is
+to happen. Extracting and ranking semantic-level information from the open
+source configuration files, is definitely not trivial, and will require a lot of
+though and consideration from the experts and the patterns that successful k8s
+project follow. This, is outside of my scope having little to no experience with
+k8s other than working on this project; however, if you have ideas I can
+probably suggest approaches in order to implement it, having worked a lot on
+this project.
+
+### Improving configuration files and container configs
+I did not have a lot of time to refactor the images to use configmaps for
+everything. This is a good thing to improve, should be fairly easy. Another
+thing that could make the user experience of launcing this could be to make all
+of the go utilities be subcommands to the same binary/container image. This
+would reduce the number of things that would have to be rebuilt, in order to get
+it running, and it would make the application (and its components) more self
+contained. (also has some disadvantages, so I'll let someone else decide.
+
+### Adding graph metrics
+From the Redis graph representation, we are able to run a multitude of graph
+algorithms (not all of which are implemented).
+
+The simplest one would be to run kruskal's algorithm to find connected
+components, and to compute graph metrics on each component. Here are some of the
+metrics that may be useful:
+
++ Average size and histograms of the sizes of each components.
+
++ Average size and histograms of the node with the highest in degree (rdeps) of
+  each component.
+
++ Average size and histograms of the number of repositories in a connected
+  component.
+
++ Any other metric that may be helpful to measure the scale of the kustomize
+  import graph.
+
+Another cool thing that may be helpful, would be to output the graph
+representation of deps/rdeps. This should be fairly easy to do with graphviz/dot
+so if anyone really wants this, I (damienr74) should be able to do it. Feel free
+to send me an email or to @ mention me in an issue.
+
+Note: dfs could also be used to find connected components, but I think union
+find is preferable, since the results can be stored and modified very
+efficiently. The only challenging part would be to implement deleting of edges
+and nodes from a component efficiently, but I know it is possible to support
+these operations with a union find structure.
+
+### Implementing PageRank
+The graph is set up to be able to efficiently compute PageRank since the edge
+weights are real valued, and the graph representation is sparse which means that
+it will fit in the memory of a single machine which will make the processing
+much more efficient.
+
+It could also be implemented as a Redis script, but I feel like there's
+something fundamentally wrong with implementing PageRank in lua. :P
+
+### Implement feature tracking
+Each day, when the crawler finds and indexes these structured documents,
+it should insert aggregate data to a separate index. This data could look like the
+following:
+
+```
+{
+  "kind": "kustomization",
+  "added_identifiers": [
+    {
+      "identifier": "some:new:k8s:feature",
+      "addedIn": [
+        "docID1",
+        "docID100",
+        "docID45",
+        ...
+      ],
+    },
+    {
+      "identifier": "another:k8s:feature",
+      "documents": [
+        ...
+      ],
+    },
+    ...
+  ]
+
+  "removed_identifiers": [
+    {
+      "identifier": "some:deprecated:field",
+      "documents": [
+        ...
+      ]
+    }
+  ]
+}
+```
+
+This would make it fairly easy to get deep insight into:
+- the speed at which things can effectively be deprecated.
+- how many people are migrating to current best practices.
+- how many documents get updated frequently/rarely.
+- detailed cross sections of growth/regression over conjunctions of features.
+- a world of possibilities.
+
+This is also something that I would be interested to work on sometime soon, so
+feel free to contact me (damienr74) or ask questions about this.
+
+As needed, it could be a good idea to also aggregate past data with a larger
+granularity. for instance each month, the past 30 days can be aggregated into
+weekish durations, And every year these weekly aggregations can be converted
+into monthly summaries depending on how much data this ends up being, and how
+much you want to pay for the storage of this data.
+
+Another cool way to compress this data would be to dynamically compress this
+data into a logarithmic number of buckets with decreasing granularity. But it
+seems like overkill for the amount of data that we'd likely get.
+
+### The UI probably needs a lot of work
+I'm not much of a UI/UX person and have little to no experience in developing
+these types of applications. If anyone with Angular experience wants to dive in
+and completely restructure the app to make the UI/UX/Code health better that
+would be greatly appreciated.
+
+### Query tuning probably still has to be adjusted
+I'm also not an expert in Elasticsearch. From what I could read in the docs,
+I think I've made sane decisions in converting user queries into meaningful
+Elasticsearch queries, but I'm sure there are a lot of improvements that remain
+to be done in order to get more accurate results.
+
+
+### Some other signals that indicate the presence of a good configuration file
+There are lots of heuristics that could be used to achieve this. Here are a
+couple in no particular order:
+
++ Penalize for the number of yaml `---` document splits. I'm not sure what the
+  general consensus is, but I think it's better to separate them, since it
+  makes git commits less noisy, it's a trivial transformation, and it makes
+  config files smaller. However, I can understand the argument that its somewhat
+  practical to keep an overall view of the configurations together (maybe).
+
++ Penalize the number of unique identifiers in a structured document. I think
+  this makes sense, since we don't want to have someone game the search engine
+  to match documents with every possible path from the k8s docs. PageRank might
+  help with this to some extent, but with a small corpus it would be fairly easy
+  to game.
+
++ Assign weights to the usefulness of certain fields. It would be good to
+  promote documents that use `keyRefFromConfigMap`, liveness probes, etc.
+
+These are the main ones I can think of, but I'm sure there are a *ton* of
+ways to achieve this.
+
+If the corpus gets large enough, we might even be able to use *blockchains*,
+*machine learning*, and maybe even self-driving cars.
+
+### Add more support for indexing of other k8s/kustomize related data
+One thing that jumps to mind is the use of kustomize plugins. They are easy
+to track since they all have an unused global variable: `var KustomizePluggin`
+it would be easy to run the pluginator command and generate godocs for each
+go file with this unique identifier.
+
+For the sake of completeness, here is the full GitHub query that we can use to
+find these:
+`api.github.com/search/code?q=var+KustomizePlugin+extension%3A.go&access_token=access_token`
+
+Godoc will not show much, since most packages will be using package main, but
+using pluginator we can make it a properly named package such that Godoc would
+actually generate the relevant documentation.

api/internal/crawl/backend/search_backend.go

@@ -0,0 +1,195 @@
+package server
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"log"
+	"net/http"
+	"os"
+	"strconv"
+	"strings"
+
+	"github.com/gorilla/mux"
+	"github.com/rs/cors"
+
+	"sigs.k8s.io/kustomize/api/internal/crawl/index"
+)
+
+type kustomizeSearch struct {
+	ctx context.Context
+	// Eventually pIndex *index.PlugginIndex
+	idx    *index.KustomizeIndex
+	router *mux.Router
+	log    *log.Logger
+}
+
+// New server. Creating a server does not launch it. To launch simply:
+//	srv, _ := NewKustomizeSearch(context.Backgroud())
+//	err := srv.Serve()
+//	if err != nil {
+//		// Handle server issues.
+//	}
+//
+// The server has three enpoints, two of which are functional:
+//
+// /search: processes the ?q= parameter for a text query and
+// returns a list of 10 resutls starting from the ?from= value provided,
+// with the default being zero.
+//
+// /metrics: returns overall metrics about the files indexed. Returns
+// timeseries data for kustomization files, and returns breakdown of file
+// counts by their 'kind' fields
+//
+// /register: not implemented, but meant as an endpoint for adding new
+// kustomization files to the corpus.
+func NewKustomizeSearch(ctx context.Context) (*kustomizeSearch, error) {
+	idx, err := index.NewKustomizeIndex(ctx, "kustomize")
+	if err != nil {
+		return nil, err
+	}
+
+	ks := &kustomizeSearch{
+		ctx:    ctx,
+		idx:    idx,
+		router: mux.NewRouter(),
+		log: log.New(os.Stdout, "Kustomize server: ",
+			log.LstdFlags|log.Llongfile|log.LUTC),
+	}
+
+	return ks, nil
+}
+
+// Set up common middleware and the routes for the server.
+func (ks *kustomizeSearch) routes() {
+
+	// Setup middleware.
+	ks.router.Use(func(handler http.Handler) http.Handler {
+		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			w.Header().Set("Content-Type", "application/json")
+			handler.ServeHTTP(w, r)
+		})
+	})
+
+	ks.router.HandleFunc("/liveness", ks.liveness()).Methods(http.MethodGet)
+	ks.router.HandleFunc("/readiness", ks.readiness()).Methods(http.MethodGet)
+	ks.router.HandleFunc("/search", ks.search()).Methods(http.MethodGet)
+	ks.router.HandleFunc("/metrics", ks.metrics()).Methods(http.MethodGet)
+	ks.router.HandleFunc("/register", ks.register()).Methods(http.MethodPost)
+}
+
+// Start listening and serving on the provided port.
+func (ks *kustomizeSearch) Serve(port int) error {
+	ks.routes()
+	handler := cors.Default().Handler(ks.router)
+	s := &http.Server{
+		Addr:    fmt.Sprintf(":%d", port),
+		Handler: handler,
+		// Timeouts/Limits
+	}
+
+	return s.ListenAndServe()
+}
+
+// /liveness endpoint
+func (ks *kustomizeSearch) liveness() http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusOK)
+	}
+}
+
+// /readyness endpoint
+func (ks *kustomizeSearch) readiness() http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		opt := index.KustomizeSearchOptions{}
+		_, err := ks.idx.Search("", opt)
+		if err != nil {
+			http.Error(w,
+				`{ "error": "could not connect to database" }`,
+				http.StatusInternalServerError)
+			return
+		}
+		w.WriteHeader(http.StatusOK)
+	}
+}
+
+// /register endpoint.
+func (ks *kustomizeSearch) register() http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		http.Error(w, "not implemented", http.StatusInternalServerError)
+	}
+}
+
+// /search endpoint.
+func (ks *kustomizeSearch) search() http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		values := r.URL.Query()
+
+		queries := values["q"]
+		ks.log.Println("Query: ", values)
+
+		var from int
+		fromParam := values["from"]
+		if len(fromParam) > 0 {
+			from, _ = strconv.Atoi(fromParam[0])
+			if from < 0 {
+				from = 0
+			}
+		}
+		_, noKinds := values["nokinds"]
+
+		opt := index.KustomizeSearchOptions{
+			SearchOptions: index.SearchOptions{
+				Size: 10,
+				From: from,
+			},
+			KindAggregation: !noKinds,
+		}
+
+		results, err := ks.idx.Search(strings.Join(queries, " "), opt)
+		if err != nil {
+			ks.log.Println("Error: ", err)
+			http.Error(w, fmt.Sprintf(
+				`{ "error": "could not complete the query" }`),
+				http.StatusInternalServerError)
+			return
+		}
+
+		enc := json.NewEncoder(w)
+		setIndent(enc)
+		if err = enc.Encode(results); err != nil {
+			http.Error(w, `{ "error": "failed to send back results" }`,
+				http.StatusInternalServerError)
+			return
+		}
+		return
+	}
+}
+
+// metrics endpoint.
+func (ks *kustomizeSearch) metrics() http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		res, err := ks.idx.Search("", index.KustomizeSearchOptions{
+			KindAggregation:       true,
+			TimeseriesAggregation: true,
+		})
+		if err != nil {
+			http.Error(w, `{ "error": "could not perform the search."}`,
+				http.StatusInternalServerError)
+			return
+		}
+
+		enc := json.NewEncoder(w)
+		setIndent(enc)
+		if err := enc.Encode(res); err != nil {
+			http.Error(w, `{ "error": "could not format return value" }`,
+				http.StatusInternalServerError)
+			return
+		}
+	}
+}
+
+// make json response human readable.
+func setIndent(e *json.Encoder) {
+	e.SetIndent("", "  ")
+}

api/internal/crawl/cmd/backend/Dockerfile

@@ -0,0 +1,14 @@
+FROM golang:1.11 AS build
+
+ARG GO111MODULE=on
+
+WORKDIR /go/src/sigs.k8s.io/kustomize/api/internal/crawl
+COPY . /go/src/sigs.k8s.io/kustomize/api/internal/crawl
+
+RUN go mod download
+RUN CGO_ENABLED=0 go install sigs.k8s.io/kustomize/api/internal/crawl/cmd/backend/
+
+FROM scratch
+COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
+COPY --from=build /go/bin/backend /
+ENTRYPOINT ["/backend"]

api/internal/crawl/cmd/backend/main.go

@@ -0,0 +1,29 @@
+package main
+
+import (
+	"context"
+	"log"
+	"os"
+	server "sigs.k8s.io/kustomize/api/internal/crawl/backend"
+	"strconv"
+)
+
+func main() {
+	portStr := os.Getenv("PORT")
+	port, err := strconv.Atoi(portStr)
+	if portStr == "" || err != nil {
+		log.Fatalf("$PORT(%s) must be set to an integer\n", portStr)
+	}
+
+	ctx := context.Background()
+
+	ks, err := server.NewKustomizeSearch(ctx)
+	if err != nil {
+		log.Fatalf("Error creating kustomize server: %v", ks)
+	}
+
+	err = ks.Serve(port)
+	if err != nil {
+		log.Fatalf("Error while running server: %v", err)
+	}
+}

api/internal/crawl/cmd/crawler/Dockerfile

@@ -0,0 +1,15 @@
+FROM golang:1.11 AS build
+
+ARG GO111MODULE=on
+
+WORKDIR /go/src/sigs.k8s.io/kustomize/api/internal/crawl
+COPY . /go/src/sigs.k8s.io/kustomize//api/internal/crawl
+
+RUN go mod download
+RUN CGO_ENABLED=0 go install -v ./cmd/crawler/crawler.go
+
+FROM scratch
+COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
+COPY --from=build /go/bin/crawler /
+ENTRYPOINT ["/crawler"]
+CMD []

api/internal/crawl/cmd/crawler/crawler.go

@@ -0,0 +1,215 @@
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"log"
+	"net/http"
+	"os"
+	"time"
+
+	"sigs.k8s.io/kustomize/api/internal/crawl/utils"
+
+	"sigs.k8s.io/kustomize/api/internal/crawl/crawler"
+	"sigs.k8s.io/kustomize/api/internal/crawl/crawler/github"
+	"sigs.k8s.io/kustomize/api/internal/crawl/doc"
+	"sigs.k8s.io/kustomize/api/internal/crawl/httpclient"
+	"sigs.k8s.io/kustomize/api/internal/crawl/index"
+
+	"github.com/gomodule/redigo/redis"
+)
+
+const (
+	githubAccessTokenVar = "GITHUB_ACCESS_TOKEN"
+	redisCacheURL        = "REDIS_CACHE_URL"
+	redisKeyURL          = "REDIS_KEY_URL"
+	retryCount           = 3
+)
+
+type CrawlMode int
+
+const (
+	CrawlUnknown CrawlMode = iota
+	// Crawl all the kustomization files in all the repositories of a Github user
+	CrawlUser
+	// Crawl all the kustomization files in a Github repo
+	CrawlRepo
+	// Crawl all the documents in the index
+	CrawlIndex
+	// Crawl all the kustomization files on Github
+	CrawlGithub
+	// Crawl all the documents in the index and crawling all the kustomization files on Github
+	CrawlIndexAndGithub
+)
+
+func NewCrawlMode(s string) CrawlMode {
+	switch s {
+	case "github-user":
+		return CrawlUser
+	case "github-repo":
+		return CrawlRepo
+	case "index+github":
+		return CrawlIndexAndGithub
+	case "index":
+		return CrawlIndex
+	case "github":
+		return CrawlGithub
+	default:
+		return CrawlUnknown
+	}
+}
+
+func main() {
+	indexNamePtr := flag.String(
+		"index", "kustomize", "The name of the ElasticSearch index.")
+	modePtr := flag.String("mode", "index+github",
+		`The crawling mode, which can be one of [github-user, github-repo, index, github, index+github].
+  * github-user: crawl all the kustomization files in all the repositories of a Github user (--github-user must be specified for this mode).
+  * github-repo: crawl all the kustomization files in a Github repository (--github-repo must be specified for this mode).
+  * index: crawl all the documents in the index.
+  * gihub: crawl all the kustomization files on Github.
+  * index+github: crawl all the documents in the index and crawling all the kustomization files on Github.`)
+	githubUserPtr := flag.String("github-user", "",
+		"A github user name (e.g., kubernetes-sigs). This flag is required for the `github-user` mode.")
+	githubRepoPtr := flag.String("github-repo", "",
+		"A github repository name (e.g., kubernetes-sigs/kustomize). This flag is required for the `github-repo` mode.")
+	flag.Parse()
+
+	githubToken := os.Getenv(githubAccessTokenVar)
+	if githubToken == "" {
+		log.Printf("Must set the variable '%s' to make github requests.\n",
+			githubAccessTokenVar)
+		return
+	}
+
+	ctx := context.Background()
+	idx, err := index.NewKustomizeIndex(ctx, *indexNamePtr)
+	if err != nil {
+		log.Printf("Could not create an index: %v\n", err)
+		return
+	}
+
+	cacheURL := os.Getenv(redisCacheURL)
+	cache, err := redis.DialURL(cacheURL)
+	clientCache := &http.Client{}
+	if err != nil {
+		log.Printf("Error: redis could not make a connection: %v\n", err)
+	} else {
+		clientCache = httpclient.NewClient(cache)
+	}
+
+	// docConverter takes in a plain document and processes it for the index.
+	docConverter := func(d *doc.Document) (crawler.CrawledDocument, error) {
+		kdoc := doc.KustomizationDocument{
+			Document: *d,
+		}
+
+		err := kdoc.ParseYAML()
+		return &kdoc, err
+	}
+
+	// Index updates the value in the index.
+	indexFunc := func(cdoc crawler.CrawledDocument, mode index.Mode) error {
+		switch d := cdoc.(type) {
+		case *doc.KustomizationDocument:
+			switch mode {
+			case index.Delete:
+				log.Printf("Deleting: %v", d)
+				return idx.Delete(d.ID())
+			default:
+				log.Printf("Inserting: %v", d)
+				return idx.Put(d.ID(), d)
+			}
+		default:
+			return fmt.Errorf("type %T not supported", d)
+		}
+	}
+
+	// seen tracks the IDs of all the documents in the index and their corresponding file types.
+	// This helps avoid indexing a given document multiple times.
+	seen := utils.NewSeenMap()
+
+	mode := NewCrawlMode(*modePtr)
+
+	ghCrawlerConstructor := func(user, repo string) crawler.Crawler {
+		if user != "" {
+			return github.NewCrawler(githubToken, retryCount, clientCache,
+				github.QueryWith(
+					github.Filename("kustomization.yaml"),
+					github.Filename("kustomization.yml"),
+					github.Filename("kustomization"),
+					github.User(user)),
+			)
+		} else if repo != "" {
+			return github.NewCrawler(githubToken, retryCount, clientCache,
+				github.QueryWith(
+					github.Filename("kustomization.yaml"),
+					github.Filename("kustomization.yml"),
+					github.Filename("kustomization"),
+					github.Repo(repo)),
+			)
+		} else {
+			return github.NewCrawler(githubToken, retryCount, clientCache,
+				github.QueryWith(
+					github.Filename("kustomization.yaml"),
+					github.Filename("kustomization.yml"),
+					github.Filename("kustomization")),
+			)
+		}
+	}
+
+	seedDocs := make(crawler.CrawlSeed, 0)
+
+	// get all the documents in the index
+	getSeedDocsFunc := func() {
+		query := []byte(`{ "query":{ "match_all":{} } }`)
+		it := idx.IterateQuery(query, 10000, 60*time.Second)
+		for it.Next() {
+			for _, hit := range it.Value().Hits.Hits {
+				seedDocs = append(seedDocs, hit.Document.Document.Copy())
+			}
+		}
+		if err := it.Err(); err != nil {
+			log.Fatalf("getSeedDocsFunc Error iterating: %v\n", err)
+		}
+	}
+
+	switch mode {
+	case CrawlIndexAndGithub:
+		getSeedDocsFunc()
+		crawlers := []crawler.Crawler{ghCrawlerConstructor("", "")}
+		crawler.CrawlFromSeed(ctx, seedDocs, crawlers, docConverter, indexFunc, seen)
+		crawler.CrawlGithub(ctx, crawlers, docConverter, indexFunc, seen)
+	case CrawlIndex:
+		getSeedDocsFunc()
+		crawlers := []crawler.Crawler{ghCrawlerConstructor("", "")}
+		crawler.CrawlFromSeed(ctx, seedDocs, crawlers, docConverter, indexFunc, seen)
+	case CrawlGithub:
+		crawlers := []crawler.Crawler{ghCrawlerConstructor("", "")}
+		// add all the documents in the index into seen.
+		// this greatly reduces the time overhead of CrawlGithub.
+		getSeedDocsFunc()
+		for _, d := range seedDocs {
+			seen[d.ID()] = d.FileType
+		}
+		crawler.CrawlGithub(ctx, crawlers, docConverter, indexFunc, seen)
+	case CrawlUser:
+		if *githubUserPtr == "" {
+			flag.Usage()
+			log.Fatalf("Please specify a github user with the github-user flag!")
+		}
+		crawlers := []crawler.Crawler{ghCrawlerConstructor(*githubUserPtr, "")}
+		crawler.CrawlGithub(ctx, crawlers, docConverter, indexFunc, seen)
+	case CrawlRepo:
+		if *githubRepoPtr == "" {
+			flag.Usage()
+			log.Fatalf("Please specify a github repository with the github-repo flag!")
+		}
+		crawlers := []crawler.Crawler{ghCrawlerConstructor("", *githubRepoPtr)}
+		crawler.CrawlGithub(ctx, crawlers, docConverter, indexFunc, seen)
+	case CrawlUnknown:
+		flag.Usage()
+		log.Fatalf("The --mode flag must be one of [github-user, github-repo, index, github, index+github].")
+	}
+}

api/internal/crawl/cmd/kustomize_stats/Dockerfile

@@ -0,0 +1,14 @@
+FROM golang:1.11 AS build
+
+ARG GO111MODULE=on
+
+WORKDIR /go/src/sigs.k8s.io/kustomize/api/internal/crawl
+COPY . /go/src/sigs.k8s.io/kustomize/api/internal/crawl
+
+RUN go mod download
+RUN CGO_ENABLED=0 go install ./cmd/kustomize_stats
+
+FROM scratch
+COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
+COPY --from=build /go/bin/kustomize_stats /
+ENTRYPOINT ["/kustomize_stats"]

api/internal/crawl/cmd/kustomize_stats/main.go

@@ -0,0 +1,249 @@
+package main
+
+import (
+	"context"
+	"crypto/sha256"
+	"flag"
+	"fmt"
+	"log"
+	"sort"
+	"time"
+
+	"sigs.k8s.io/kustomize/api/internal/crawl/doc"
+
+	"sigs.k8s.io/kustomize/api/internal/crawl/index"
+)
+
+// iterateArr adds each item in arr into countMap.
+func iterateArr(arr []string, countMap map[string]int) {
+	for _, item := range arr {
+		if _, ok := countMap[item]; !ok {
+			countMap[item] = 1
+		} else {
+			countMap[item]++
+		}
+	}
+
+}
+
+// SortMapKeyByValueInt takes a map as its input, sorts its keys according to their values
+// in the map, and outputs the sorted keys as a slice.
+func SortMapKeyByValueInt(m map[string]int) []string {
+	keys := make([]string, 0, len(m))
+	for key := range m {
+		keys = append(keys, key)
+	}
+	// sort keys according to their values in the map m
+	sort.Slice(keys, func(i, j int) bool { return m[keys[i]] > m[keys[j]] })
+	return keys
+}
+
+// SortMapKeyByValue takes a map as its input, sorts its keys according to their values
+// in the map, and outputs the sorted keys as a slice.
+func SortMapKeyByValueLen(m map[string][]string) []string {
+	keys := make([]string, 0, len(m))
+	for key := range m {
+		keys = append(keys, key)
+	}
+	// sort keys according to their values in the map m
+	sort.Slice(keys, func(i, j int) bool { return len(m[keys[i]]) > len(m[keys[j]]) })
+	return keys
+}
+
+func GeneratorOrTransformerStats(docs []*doc.KustomizationDocument) {
+	n := len(docs)
+	if n == 0 {
+		return
+	}
+
+	fileType := docs[0].FileType
+	fmt.Printf("There are totally %d %s files.\n", n, fileType)
+
+	GitRepositorySummary(docs, fileType)
+
+	// key of kindToUrls: a string in the KustomizationDocument.Kinds field
+	// value of kindToUrls: a slice of string urls defining a given kind.
+	kindToUrls := make(map[string][]string)
+
+	for _, d := range docs {
+		url := fmt.Sprintf("%s/blob/%s/%s", d.RepositoryURL, d.DefaultBranch, d.FilePath)
+		for _, kind := range d.Kinds {
+			if _, ok := kindToUrls[kind]; !ok {
+				kindToUrls[kind] = []string{url}
+			} else {
+				kindToUrls[kind] = append(kindToUrls[kind], url)
+			}
+		}
+	}
+	fmt.Printf("There are totally %d kinds of %s\n", len(kindToUrls), fileType)
+	sortedKeys := SortMapKeyByValueLen(kindToUrls)
+	for _, k := range sortedKeys {
+		sort.Strings(kindToUrls[k])
+		fmt.Printf("%s kind %s appears %d times\n", fileType, k, len(kindToUrls[k]))
+		for _, url := range kindToUrls[k] {
+			fmt.Printf("%s\n", url)
+		}
+	}
+}
+
+// GitRepositorySummary counts the distribution of docs:
+// 1) how many git repositories are these docs from?
+// 2) how many docs are from each git repository?
+func GitRepositorySummary(docs []*doc.KustomizationDocument, fileType string) {
+	m := make(map[string]int)
+	for _, d := range docs {
+		if _, ok := m[d.RepositoryURL]; ok {
+			m[d.RepositoryURL]++
+		} else {
+			m[d.RepositoryURL] = 1
+		}
+	}
+	sortedKeys := SortMapKeyByValueInt(m)
+	topN := 10
+	i := 0
+	for _, k := range sortedKeys {
+		if i >= topN {
+			break
+		}
+		fmt.Printf("%d %s are from %s\n", m[k], fileType, k)
+		i++
+	}
+}
+
+func main() {
+	topKindsPtr := flag.Int(
+		"kinds", -1,
+		`the number of kubernetes object kinds to be listed according to their popularities.
+By default, all the kinds will be listed.
+If you only want to list the 10 most popular kinds, set the flag to 10.`)
+	topIdentifiersPtr := flag.Int(
+		"identifiers", -1,
+		`the number of identifiers to be listed according to their popularities.
+By default, all the identifiers will be listed.
+If you only want to list the 10 most popular identifiers, set the flag to 10.`)
+	topKustomizeFeaturesPtr := flag.Int(
+		"kustomize-features", -1,
+		`the number of kustomize features to be listed according to their popularities.
+By default, all the features will be listed.
+If you only want to list the 10 most popular features, set the flag to 10.`)
+	indexNamePtr := flag.String(
+		"index", "kustomize", "The name of the ElasticSearch index.")
+	flag.Parse()
+
+	ctx := context.Background()
+	idx, err := index.NewKustomizeIndex(ctx, *indexNamePtr)
+	if err != nil {
+		log.Fatalf("Could not create an index: %v\n", err)
+	}
+
+	// count tracks the number of documents in the index
+	count := 0
+
+	// kustomizationFilecount tracks the number of kustomization files in the index
+	kustomizationFilecount := 0
+
+	kindsMap := make(map[string]int)
+	identifiersMap := make(map[string]int)
+	kustomizeIdentifiersMap := make(map[string]int)
+
+	// ids tracks the unique IDs of the documents in the index
+	ids := make(map[string]struct{})
+
+	// generatorFiles include all the non-kustomization files whose FileType is generator
+	generatorFiles := make([]*doc.KustomizationDocument, 0)
+
+	// transformersFiles include all the non-kustomization files whose FileType is transformer
+	transformersFiles := make([]*doc.KustomizationDocument, 0)
+
+	checksums := make(map[string]int)
+
+	// get all the documents in the index
+	query := []byte(`{ "query":{ "match_all":{} } }`)
+	it := idx.IterateQuery(query, 10000, 60*time.Second)
+	for it.Next() {
+		for _, hit := range it.Value().Hits.Hits {
+			sum := fmt.Sprintf("%x", sha256.Sum256([]byte(hit.Document.DocumentData)))
+			if _, ok := checksums[sum]; ok {
+				checksums[sum]++
+			} else {
+				checksums[sum] = 1
+			}
+
+			// check whether there is any duplicate IDs in the index
+			if _, ok := ids[hit.ID]; !ok {
+				ids[hit.ID] = struct{}{}
+			} else {
+				log.Printf("Found duplicate ID (%s)\n", hit.ID)
+			}
+
+			count++
+			iterateArr(hit.Document.Kinds, kindsMap)
+			iterateArr(hit.Document.Identifiers, identifiersMap)
+
+			if doc.IsKustomizationFile(hit.Document.FilePath) {
+				kustomizationFilecount++
+				iterateArr(hit.Document.Identifiers, kustomizeIdentifiersMap)
+
+			} else {
+				switch hit.Document.FileType {
+				case "generator":
+					generatorFiles = append(generatorFiles, hit.Document.Copy())
+				case "transformer":
+					transformersFiles = append(transformersFiles, hit.Document.Copy())
+				}
+			}
+		}
+	}
+
+	if err := it.Err(); err != nil {
+		log.Fatalf("Error iterating: %v\n", err)
+	}
+
+	sortedKindsMapKeys := SortMapKeyByValueInt(kindsMap)
+	sortedIdentifiersMapKeys := SortMapKeyByValueInt(identifiersMap)
+	sortedKustomizeIdentifiersMapKeys := SortMapKeyByValueInt(kustomizeIdentifiersMap)
+
+	fmt.Printf(`The count of unique document IDs in the kustomize index: %d
+There are %d documents in the kustomize index.
+%d kinds of kubernetes objects are customized:`, len(ids), count, len(kindsMap))
+	fmt.Printf("\n")
+
+	kindCount := 0
+	for _, key := range sortedKindsMapKeys {
+		if *topKindsPtr < 0 || (*topKindsPtr >= 0 && kindCount < *topKindsPtr) {
+			fmt.Printf("\tkind `%s` is customimzed in %d documents\n", key, kindsMap[key])
+			kindCount++
+		}
+	}
+
+	fmt.Printf("%d kinds of identifiers are found:\n", len(identifiersMap))
+	identifierCount := 0
+	for _, key := range sortedIdentifiersMapKeys {
+		if *topIdentifiersPtr < 0 || (*topIdentifiersPtr >= 0 && identifierCount < *topIdentifiersPtr) {
+			fmt.Printf("\tidentifier `%s` appears in %d documents\n", key, identifiersMap[key])
+			identifierCount++
+		}
+	}
+
+	fmt.Printf(`There are %d kustomization files in the kustomize index.
+%d kinds of kustomize features are found:`, kustomizationFilecount, len(kustomizeIdentifiersMap))
+	fmt.Printf("\n")
+	kustomizeFeatureCount := 0
+	for _, key := range sortedKustomizeIdentifiersMapKeys {
+		if *topKustomizeFeaturesPtr < 0 || (*topKustomizeFeaturesPtr >= 0 && kustomizeFeatureCount < *topKustomizeFeaturesPtr) {
+			fmt.Printf("\tfeature `%s` is used in %d documents\n", key, kustomizeIdentifiersMap[key])
+			kustomizeFeatureCount++
+		}
+	}
+
+	GeneratorOrTransformerStats(generatorFiles)
+	GeneratorOrTransformerStats(transformersFiles)
+
+	fmt.Printf("There are total %d checksums of document contents\n", len(checksums))
+	sortedChecksums := SortMapKeyByValueInt(checksums)
+	sortedChecksums = sortedChecksums[:20]
+	fmt.Printf("The top 20 checksums are:\n")
+	for _, key := range sortedChecksums {
+		fmt.Printf("checksum %s apprears %d\n", key, checksums[key])
+	}
+}

api/internal/crawl/cmd/log-parser/README.md

@@ -0,0 +1,8 @@
+This binary takes as its input a json file including GKE logs (which can be
+[exported](https://cloud.google.com/logging/docs/export/configure_export_v2) into 
+[Cloud Storage](https://cloud.google.com/storage/docs/)), 
+and extracts the `textPayload` field of each log entry.
+
+Here is an log entry example:
+
+{"insertId":"1sxuh4jg5lw6w10","labels":{"compute.googleapis.com/resource_name":"gke-crawler2-default-pool-5e55ea05-gzgv","container.googleapis.com/namespace_name":"default","container.googleapis.com/pod_name":"kustomize-stats-5bczg","container.googleapis.com/stream":"stdout"},"logName":"projects/haiyanmeng-gke-dev/logs/kustomize-stats","receiveTimestamp":"2020-01-06T23:33:07.012831742Z","resource":{"labels":{"cluster_name":"crawler2","container_name":"kustomize-stats","instance_id":"8183086081854184383","namespace_id":"default","pod_id":"kustomize-stats-5bczg","project_id":"haiyanmeng-gke-dev","zone":"us-central1-a"},"type":"container"},"severity":"INFO","textPayload":"The kustomize index already exists\n","timestamp":"2020-01-06T23:32:46.628930547Z"}

api/internal/crawl/cmd/log-parser/kustomize-stats-cmd

@@ -0,0 +1,7 @@
+wget <log-file-url> -O log
+go build .
+./log-parser log >out
+cat out | grep "kind \`" | cut -d\` -f2 | tail -n 50
+cat out | grep "kind \`" | awk '{print $6}' | tail -n 50
+cat out | grep "feature \`" | grep -v "\`resources\`" | grep -v -e "\`apiVersion\`" | grep -v -e "\`apiversion\`" | cut -d\` -f2
+cat out | grep "feature \`" | grep -v "\`resources\`" | grep -v -e "\`apiVersion\`" | grep -v -e "\`apiversion\`" | awk '{print $6}'

api/internal/crawl/cmd/log-parser/main.go

@@ -0,0 +1,49 @@
+package main
+
+import (
+	"bufio"
+	"encoding/json"
+	"fmt"
+	"log"
+	"os"
+)
+
+func main() {
+	if len(os.Args) != 2 {
+		log.Fatalf("The usage of the command is: \n\t%s <log-file.json>", os.Args[0])
+	}
+
+	file, err := os.Open(os.Args[1])
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	closeFile := func(file *os.File) {
+		if err := file.Close(); err != nil {
+			log.Fatal(err)
+		}
+	}
+	defer closeFile(file)
+
+	scanner := bufio.NewScanner(file)
+	for scanner.Scan() {
+		line := scanner.Text()
+
+		var entry interface{}
+		if err := json.Unmarshal([]byte(line), &entry); err != nil {
+			log.Printf("failed to unmarshal a log entry: %s\n", line)
+		}
+
+		m := entry.(map[string]interface{})
+		if payload, ok := m["textPayload"]; ok {
+			// use fmt.Printf here instead of log.Printf to avoid the time and code location info the log package provides
+			fmt.Printf("%s", payload)
+		} else {
+			log.Printf("the log entry does not have the `textPayload` field: %s\n", line)
+		}
+	}
+
+	if err := scanner.Err(); err != nil {
+		log.Fatal(err)
+	}
+}

api/internal/crawl/config/base/kustomization.yaml

@@ -0,0 +1,5 @@
+configmapGenerator:
+- name: elasticsearch-config
+  literals:
+  - es-url="http://esbasic-master:9200"
+  - plugin-index-name="plugin"

api/internal/crawl/config/crawler/base/.gitignore

@@ -0,0 +1 @@
+github_api_secret.txt

api/internal/crawl/config/crawler/base/github_api_secret.txt

@@ -0,0 +1,2 @@
+<ADD YOUR GITHUB PERSONAL ACCESS TOKEN HERE WITHOUT A TRAILING NEWLINE>
+Run: printf "<your-token>" > github_api_secret.txt

api/internal/crawl/config/crawler/base/kustomization.yaml

@@ -0,0 +1,15 @@
+resources:
+- ../../base
+
+configmapGenerator:
+- name: crawler-http-cache
+  literals:
+  - redis-cache-url="redis://redis-http-cache:6379"
+- name: redis-keystore
+  literals:
+  - keystore-url="redis://redis-docs-keystore:6379"
+
+secretGenerator:
+- name: github-access-token
+  files:
+  - token=github_api_secret.txt

api/internal/crawl/config/crawler/cronjob/cronjob.yaml

@@ -0,0 +1,34 @@
+apiVersion: batch/v1beta1
+kind: CronJob
+metadata:
+  name: crawler-cronjob
+spec:
+  # run the cronjob at 00:00 every 7 days
+  schedule: "0 0 */7 * *"
+  jobTemplate:
+    spec:
+      template:
+        spec:
+          restartPolicy: OnFailure
+          containers:
+          - name: crawler
+            image: gcr.io/haiyanmeng-gke-dev/crawler:v1
+            command: ["/crawler"]
+            args: ["--mode=index+github", "--github-repo=kubernetes-sigs/kustomize", "--index=kustomize"]
+            imagePullPolicy: Always
+            env:
+            - name: GITHUB_ACCESS_TOKEN
+              valueFrom:
+                secretKeyRef:
+                  name: github-access-token
+                  key: token
+            - name: ELASTICSEARCH_URL
+              valueFrom:
+                configMapKeyRef:
+                  name: elasticsearch-config
+                  key: es-url
+            - name: REDIS_URL
+              valueFrom:
+                configMapKeyRef:
+                  name: crawler-http-cache
+                  key: redis-cache-url

api/internal/crawl/config/crawler/cronjob/kustomization.yaml

@@ -0,0 +1,3 @@
+resources:
+- ../base
+- cronjob.yaml

api/internal/crawl/config/crawler/job/README.md

@@ -0,0 +1,53 @@
+The crawler job can run in one of the following mode:
+
+# Crawling all the documents in the index and crawling all the kustomization files on Github
+
+This is the default setting of the crawler job. The `command` and `args` field
+of the container should be:
+
+```
+        command: ["/crawler"]
+```
+
+Or
+
+```
+        command: ["/crawler"]
+        args: ["--mode=index+github"]
+```
+
+# Crawling all the documents in the index
+
+The `command` and `args` field of the container should be:
+
+```
+        command: ["/crawler"]
+        args: ["--mode=index"]
+```
+
+# Crawling all the kustomization files on Github
+
+The `command` and `args` field of the container should be:
+
+```
+        command: ["/crawler"]
+        args: ["--mode=github"]
+```
+
+# Crawling all the kustomization files in a Github repo
+
+The `command` and `args` field of the container should be like:
+
+```
+        command: ["/crawler"]
+        args: ["--mode=github-repo", "--github-repo=kubernetes-sigs/kustomize"]
+```
+
+# Crawling all the kustomization files in all the repositories of a Github user
+
+The `command` and `args` field of the container should be like:
+
+```
+        command: ["/crawler"]
+        args: ["--github-user", "--github-user=kubernetes-sigs"]
+```

api/internal/crawl/config/crawler/job/job.yaml

@@ -0,0 +1,35 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: crawler
+spec:
+  template:
+    spec:
+      restartPolicy: OnFailure
+      containers:
+      - name: crawler
+        image: gcr.io/haiyanmeng-gke-dev/crawler:v1
+        imagePullPolicy: Always
+        command: ["/crawler"]
+        args: ["--mode=github-repo", "--github-repo=kubernetes-sigs/kustomize", "--index=kustomize"]
+        env:
+        - name: GITHUB_ACCESS_TOKEN
+          valueFrom:
+            secretKeyRef:
+              name: github-access-token
+              key: token
+        - name: ELASTICSEARCH_URL
+          valueFrom:
+            configMapKeyRef:
+              name: elasticsearch-config
+              key: es-url
+        - name: REDIS_CACHE_URL
+          valueFrom:
+            configMapKeyRef:
+              name: crawler-http-cache
+              key: redis-cache-url
+        - name: REDIS_KEY_URL
+          valueFrom:
+            configMapKeyRef:
+              name: redis-keystore
+              key: keystore-url

api/internal/crawl/config/crawler/job/kustomization.yaml

@@ -0,0 +1,3 @@
+resources:
+- ../base
+- job.yaml

api/internal/crawl/config/crawler/kustomize_stats/job.yaml

@@ -0,0 +1,20 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: kustomize-stats
+spec:
+  template:
+    spec:
+      restartPolicy: OnFailure
+      containers:
+      - name: kustomize-stats
+        image: gcr.io/haiyanmeng-gke-dev/kustomize_stats:v1
+        imagePullPolicy: Always
+        command: ["/kustomize_stats"]
+        args: ["--index=kustomize", "--kinds=51", "--identifiers=50", "--kustomize-features=50"]
+        env:
+        - name: ELASTICSEARCH_URL
+          valueFrom:
+            configMapKeyRef:
+              name: elasticsearch-config
+              key: es-url

api/internal/crawl/config/crawler/kustomize_stats/kustomization.yaml

@@ -0,0 +1,3 @@
+resources:
+- ../base
+- job.yaml

api/internal/crawl/config/elastic/esbackup.yaml

@@ -0,0 +1,23 @@
+# ESBackup depends on ESCluster, and is depended by ESSnapshot.
+# Creating `esbackup/kustomize-backbup` will create the `kustomize-backup` snapshot repository.
+# Deleting `esbackup/kustomize-backbup` will delete the `kustomize-backup` snapshot repository.
+# Deleting `esbackup/kustomize-backbup` will NOT delete the snapshots in the `kustomize-backup` snapshot repository, instead makes all the snapshots in the repository inaccessible.
+# Deleting `esbackup/kustomize-backbup` will NOT delete the essnapshot objects depending on it, but will cause those essnapshot objects to be reconciled, which update the status of the essnapshot objects to reflect the fact that the esbackup object is missing.
+# If you delete the `kustomize-backup` snapshot repository directly without deleting `esbackup/kustomize-backbup`, the ESBackup object will not recreate the snapshot repository.
+apiVersion: elasticsearch.cloud.google.com/v1alpha1
+kind: ESBackup
+metadata:
+  name: kustomize-backup
+spec:
+  storage:
+    gcs:
+      # the bucket must exist for the snapshot respository to be created successfully.
+      bucket: kustomize-backup
+      # the path does not need to exist.
+      # If the path does not exist, the controller will create the folder in the GCS bucket.
+      # If the path already exists and includes snapshots, these snapshots can be used.
+      path: kustomize
+      secret:
+        name: kustomizesa
+  escluster:
+    name: esbasic

api/internal/crawl/config/elastic/escluster.yaml

@@ -0,0 +1,51 @@
+# ESCluster is depended by ESBackup and ESRestore.
+apiVersion: elasticsearch.cloud.google.com/v1alpha1
+kind: ESCluster
+metadata:
+  name: esbasic
+spec:
+  plugin:
+   pluginList:
+    - repository-gcs
+    - ingest-user-agent
+    - ingest-geoip
+  # To set `gcpserviceaccount`,
+  # First, create and download a GCP service account into a json file, named `sakey.json` following the instruction:
+  #     https://www.elastic.co/guide/en/elasticsearch/plugins/6.5/repository-gcs-usage.html#repository-gcs-using-service-account
+  # Second, create a secret for the service account using the following command:
+  #     $ kubectl create secret generic kustomizesa --from-file=./sakey.json
+  gcpserviceaccount:
+    name: kustomizesa
+  config:
+    env:
+      example: test
+  nodegroups:
+    - name: di
+      replicas: 2
+      data: true
+      ingest: true
+      config:
+        jvm:
+          - Djava.net.preferIPv4Stack=true
+          - Xms2g
+          - Xmx2g
+        es:
+          path.repo: '["/tmp/es_backup_basic"]'
+      affinity:
+        podAntiAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+          - topologyKey: kubernetes.io/hostname
+            labelSelector:
+              matchLabels:
+                es/nodegroup: di
+      resources:
+        requests:
+          memory: 3Gi
+        limits:
+          memory: 3Gi
+    - name: m
+      replicas: 2
+      master: true
+      config:
+        es:
+          path.repo: '["/tmp/es_backup_basic"]'

api/internal/crawl/config/elastic/esrestore.yaml

@@ -0,0 +1,19 @@
+# ESRestore depends on both ESCluster and ESSnapshot.
+# Creating `esrestore/kustomize-restore` will restore the `kuostmize` index in the `kustomize-snapshot` snapshot to a new index named `kusotmize-restore`.
+# Deleting `esrestore/kustomize-restore` will not delete the restored index.
+# Deleting `esrestore/kustomize-restore` should happen before deleting `essnapshot/kustomize-snapshot`.
+# After the restore is complete, if the `kusotmize-restore` index is deleted manually, the ESRestore object will NOT restore the `kustomize` index to it again.
+# The correct way of using ESRestore is: create a ESRestore object to restore the index; delete the ESRestore object after the restore is complete.
+apiVersion: elasticsearch.cloud.google.com/v1alpha1
+kind: ESRestore
+metadata:
+  name: kustomize-restore
+spec:
+  include_global_state: true
+  ignore_unavailable: true
+  rename_pattern: kustomize
+  rename_replacement: kustomize-restore
+  essnapshot:
+    name: kustomize-snapshot
+  escluster:
+    name: esbasic

api/internal/crawl/config/elastic/essnapshot.yaml

@@ -0,0 +1,23 @@
+# ESSnapshot depends on ESBackup, and is depended by ESRestore.
+# Creating `essnapshot/kustomize-snapshot` will create a snapshot named `kustomize-snapshot` in the `kustomize-backup` snapshot repository.
+# After being created, the `kustomize-snapshot` snapshot will not be automatically updated when the `kuostomize` index is updated.
+# If you delete `essnapshot/kustomize-snapshot` and recreate it, the new snapshot will capture the current status of the `kustomize` index.
+# Deleting `essnapshot/kustomize-snapshot` will delete the snapshot.
+# Deleting `essnapshot/kustomize-snapshot` should happen before deleting `esbackup/kustomize-backup`.
+# If the `kustomize-snapshot` snapshot is deleted directly without deleting `essnapshot/kustomize-snapshot`, the ESSnapshot object will recreate the snapshot.
+# The correct way of using ESSnapshot is: create an ESSnapshot object to create a snapshot, keep the ESSnapshot object until the snapshot is no longer needed.
+# To update the snapshot to capture the latest version of the index, you can either:
+#   1) delete the snapshot, and wait for the ESSnapshot object to recreate the snapshot;
+#   2) delete the ESSnapshot object, and recreate the ESSnapshot object.
+apiVersion: elasticsearch.cloud.google.com/v1alpha1
+kind: ESSnapshot
+metadata:
+  name: kustomize-snapshot
+spec:
+  # indices are optional. If not specified all indices are selected.
+  indices:
+  - kustomize
+  include_global_state: true
+  ignore_unavailable: true
+  esbackup:
+    name: kustomize-backup

api/internal/crawl/config/elastic/service.yaml

@@ -0,0 +1,16 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: elasticsearch
+spec:
+  selector:
+    custom-resource: v1alpha1.ESCluster
+    custom-resource-name: esbasic
+    custom-resource-namespace: default
+    es/data: "true"
+    using: escluster.Cluster
+  ports:
+  - protocol: "TCP"
+    port: 9200
+  type: LoadBalancer
+  loadBalancerIP: ""

api/internal/crawl/config/redis/document_keystore/kustomization.yaml

@@ -0,0 +1,7 @@
+resources:
+- redis.yaml
+- service.yaml
+
+commonLabels:
+  app: redis
+  tier: document-keystore

api/internal/crawl/config/redis/document_keystore/redis.yaml

@@ -0,0 +1,37 @@
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: redis-docs-keystore
+spec:
+  serviceName: "redis-docs-keystore"
+  template:
+    spec:
+      containers:
+      - name: redis
+        image: redis:5-alpine
+        imagePullPolicy: Always
+        args:
+          - "--save"
+          - "900"
+          - "1"
+          - "--save"
+          - "30"
+          - "100"
+          - "--appendonly"
+          - "yes"
+        ports:
+        - name: redis-docs-port
+          containerPort: 6379
+        volumeMounts:
+        - mountPath: /data
+          name: redis-docs-keystore-data
+      restartPolicy: Always
+  volumeClaimTemplates:
+  - metadata:
+      name: redis-docs-keystore-data
+    spec:
+      accessModes:
+      - ReadWriteOnce
+      resources:
+        requests:
+          storage: 4Gi

api/internal/crawl/config/redis/document_keystore/service.yaml

@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: redis-docs-keystore
+spec:
+  clusterIP: None
+  ports:
+  - protocol: "TCP"
+    port: 6379
+    targetPort: redis-docs-port

api/internal/crawl/config/redis/http_cache/kustomization.yaml

@@ -0,0 +1,7 @@
+resources:
+- redis.yaml
+- service.yaml
+
+commonLabels:
+  app: redis
+  tier: http-cache

api/internal/crawl/config/redis/http_cache/redis.yaml

@@ -0,0 +1,16 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: redis-http-cache
+spec:
+  template:
+    spec:
+      containers:
+      - name: redis
+        image: redis:5-alpine
+        imagePullPolicy: Always
+        # see redis.io/topics/lru-cache for other policy options.
+        args: ["--maxmemory", "1gb", "--maxmemory-policy", "allkeys-lru"]
+        ports:
+        - name: http-cache-port
+          containerPort: 6379

api/internal/crawl/config/redis/http_cache/service.yaml

@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: redis-http-cache
+spec:
+  clusterIP: None
+  ports:
+  - protocol: "TCP"
+    port: 6379
+    targetPort: http-cache-port

api/internal/crawl/config/webapp/backend/deployment.yaml

@@ -0,0 +1,35 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: kustomize-search
+spec:
+  selector:
+    matchLabels:
+      app: kustomize-search
+      tier: backend
+  replicas: 1
+  template:
+    metadata:
+      labels:
+        app: kustomize-search
+        tier: backend
+    spec:
+      containers:
+      - name: kustomize-search
+        image: gcr.io/kustomize-search/backend:latest
+        imagePullPolicy: Always
+        livenessProbe:
+          httpGet:
+            path: /liveness
+            port: backend-port
+        ports:
+        - name: backend-port
+          containerPort: 8080
+        env:
+        - name: ELASTICSEARCH_URL
+          valueFrom:
+            configMapKeyRef:
+              name: elasticsearch-config
+              key: es-url
+        - name: PORT
+          value: "8080"

api/internal/crawl/config/webapp/backend/kustomization.yaml

@@ -0,0 +1,4 @@
+resources:
+- ../../base
+- deployment.yaml
+- service.yaml

api/internal/crawl/config/webapp/backend/service.yaml

@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: kustomize-search
+spec:
+  selector:
+    app: kustomize-search
+    tier: backend
+  ports:
+  - protocol: "TCP"
+    port: 80
+    targetPort: backend-port
+  type: LoadBalancer
+  loadBalancerIP: ""

api/internal/crawl/config/webapp/frontend/deployment.yaml

@@ -0,0 +1,23 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: kustomize-search-ui
+spec:
+  selector:
+    matchLabels:
+      app: kustomize-search
+      tier: frontend
+  replicas: 1
+  template:
+    metadata:
+      labels:
+        app: kustomize-search
+        tier: frontend
+    spec:
+      containers:
+      - name: frontend
+        image: gcr.io/kustomize-search/frontend:latest
+        imagePullPolicy: Always
+        ports:
+        - name: frontend-port
+          containerPort: 80

api/internal/crawl/config/webapp/frontend/kustomization.yaml

@@ -0,0 +1,4 @@
+resources:
+- ../../base
+- deployment.yaml
+- service.yaml

api/internal/crawl/config/webapp/frontend/service.yaml

@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: kustomize-search-ui
+spec:
+  selector:
+    app: kustomize-search
+    tier: frontend
+  ports:
+  - protocol: "TCP"
+    port: 80
+    targetPort: frontend-port
+  type: LoadBalancer
+  loadBalancerIP: ""

api/internal/crawl/crawler/crawler.go

@@ -0,0 +1,344 @@
+// Package crawler provides helper methods and defines an interface for lauching
+// source repository crawlers that retrieve files from a source and forwards
+// to a channel for indexing and retrieval.
+package crawler
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"os"
+	"sync"
+
+	"sigs.k8s.io/kustomize/api/internal/crawl/utils"
+
+	"sigs.k8s.io/kustomize/api/internal/crawl/index"
+
+	_ "github.com/gomodule/redigo/redis"
+
+	"sigs.k8s.io/kustomize/api/internal/crawl/doc"
+)
+
+var (
+	logger = log.New(os.Stdout, "Crawler: ", log.LstdFlags|log.LUTC|log.Llongfile)
+)
+
+// Crawler forwards documents from source repositories to index and store them
+// for searching. Each crawler is responsible for querying it's source of
+// information, and forwarding files that have not been seen before or that need
+// updating.
+type Crawler interface {
+	// Crawl returns when it is done processing. This method does not take
+	// ownership of the channel. The channel is write only, and it
+	// designates where the crawler should forward the documents.
+	Crawl(ctx context.Context, output chan<- CrawledDocument, seen utils.SeenMap) error
+
+	// Get the document data given the FilePath, Repo, and Ref/Tag/Branch.
+	FetchDocument(context.Context, *doc.Document) error
+	// Write to the document what the created time is.
+	SetCreated(context.Context, *doc.Document) error
+
+	SetDefaultBranch(*doc.Document)
+
+	Match(*doc.Document) bool
+}
+
+type CrawledDocument interface {
+	ID() string
+	GetDocument() *doc.Document
+	// Get all the Documents directly referred in a Document.
+	// For a Document representing a non-kustomization file, an empty slice will be returned.
+	// For a Document representing a kustomization file:
+	// the `includeResources` parameter determines whether the documents referred in the `resources` field are returned or not;
+	// the `includeTransformers` parameter determines whether the documents referred in the `transformers` field are returned or not;
+	// the `includeGenerators` parameter determines whether the documents referred in the `generators` field are returned or not.
+	GetResources(includeResources, includeTransformers, includeGenerators bool) ([]*doc.Document, error)
+	WasCached() bool
+}
+
+type CrawlSeed []*doc.Document
+
+type IndexFunc func(CrawledDocument, index.Mode) error
+type Converter func(*doc.Document) (CrawledDocument, error)
+
+func logIfErr(err error) {
+	if err == nil {
+		return
+	}
+	logger.Println("error: ", err)
+}
+
+func findMatch(d *doc.Document, crawlers []Crawler) Crawler {
+	for _, crawl := range crawlers {
+		if crawl.Match(d) {
+			return crawl
+		}
+	}
+	return nil
+}
+
+func addBranches(cdoc CrawledDocument, match Crawler, indx IndexFunc,
+	seen utils.SeenMap, stack *CrawlSeed) {
+
+	seen.Set(cdoc.ID(), cdoc.GetDocument().FileType)
+
+	match.SetDefaultBranch(cdoc.GetDocument())
+
+	// Insert into index
+	if err := indx(cdoc, index.InsertOrUpdate); err != nil {
+		logger.Printf("Failed to insert or update doc(%s): %v",
+			cdoc.GetDocument().Path(), err)
+		return
+	}
+
+	deps, err := cdoc.GetResources(true, true, true)
+	if err != nil {
+		logger.Println(err)
+		return
+	}
+
+	for _, dep := range deps {
+		if seen.Seen(dep.ID()) && seen.Value(dep.ID()) == dep.FileType {
+			continue
+		}
+		*stack = append(*stack, dep)
+	}
+}
+
+func doCrawl(ctx context.Context, docsPtr *CrawlSeed, crawlers []Crawler, conv Converter, indx IndexFunc,
+	seen utils.SeenMap, stack *CrawlSeed, refreshDoc bool, updateFileType bool) {
+
+	UpdatedDocCount := 0
+	seenDocCount := 0
+	cachedDocCount := 0
+	findMatchErrCount := 0
+	FetchDocumentErrCount := 0
+	SetCreatedErrCount := 0
+	convErrCount := 0
+	deleteDocCount := 0
+	crawledDocCount := 0
+
+	// During the execution of the for loop, more Documents may be added into (*docsPtr).
+	for len(*docsPtr) > 0 {
+		// get the last Document in (*docPtr), which will be crawled in this iteration.
+		tail := (*docsPtr)[len(*docsPtr)-1]
+
+		// remove the last Document in (*docPtr)
+		*docsPtr = (*docsPtr)[:(len(*docsPtr) - 1)]
+
+		crawledDocCount++
+		logger.Printf("Crawling doc %d: %s", crawledDocCount, tail.Path())
+
+		if seen.Seen(tail.ID()) {
+			if !updateFileType || seen.Value(tail.ID()) == tail.FileType {
+				logger.Printf("this doc has been seen before")
+				seenDocCount++
+				continue
+			}
+		}
+
+		if tail.WasCached() {
+			logger.Printf("doc(%s) is cached already", tail.Path())
+			cachedDocCount++
+			continue
+		}
+
+		match := findMatch(tail, crawlers)
+		if match == nil {
+			logIfErr(fmt.Errorf("%v could not match any crawler", tail))
+			findMatchErrCount++
+			continue
+		}
+
+		if tail.User == "" {
+			tail.User = doc.UserName(tail.RepositoryURL)
+		}
+
+		// If the Document represents a kustomization root, FetchDcoument will change
+		// the `filePath` field of the Document by adding `kustomization.yaml` or
+		// `kustomization.yml` or `kustomization` into the the field.
+		// Therefore, it is necessary to add the ID of the Document into seen before
+		// calling FetchDocument. Otherwise, the binary may enter into an infinite loop
+		// if a kustomization file points to its kustmozation root in its `resources` or
+		// `bases` field.
+		seen.Set(tail.ID(), tail.FileType)
+
+		if refreshDoc || tail.DefaultBranch == "" {
+			match.SetDefaultBranch(tail)
+		}
+
+		if refreshDoc || tail.DocumentData == "" {
+			if err := match.FetchDocument(ctx, tail); err != nil {
+				logger.Printf("FetchDocument failed on doc(%s): %v", tail.Path(), err)
+				FetchDocumentErrCount++
+				// delete the document from the index
+				cdoc := &doc.KustomizationDocument{
+					Document: *tail,
+				}
+				seen.Set(cdoc.ID(), tail.FileType)
+				if err := indx(cdoc, index.Delete); err != nil {
+					logger.Printf("Failed to delete doc(%s): %v", cdoc.Path(), err)
+				}
+				deleteDocCount++
+				continue
+			}
+		}
+
+		if refreshDoc || tail.CreationTime == nil {
+			if err := match.SetCreated(ctx, tail); err != nil {
+				logger.Printf("SetCreated failed on doc(%s): %v", tail.Path(), err)
+				SetCreatedErrCount++
+			}
+		}
+
+		cdoc, err := conv(tail)
+		// If conv returns an error, cdoc can still be added into the index so that
+		// cdoc.Document can be searched.
+		if err != nil {
+			logger.Printf("conv failed on doc(%s): %v", tail.Path(), err)
+			convErrCount++
+		}
+
+		UpdatedDocCount++
+		addBranches(cdoc, match, indx, seen, stack)
+	}
+	logger.Printf("Summary of doCrawl:\n")
+	logger.Printf("\t%d documents were updated\n", UpdatedDocCount)
+	logger.Printf("\t%d documents were seen by the crawler already and skipped\n", seenDocCount)
+	logger.Printf("\t%d documents were cached already and skipped\n", cachedDocCount)
+	logger.Printf("\t%d documents didn't have a matching crawler and skipped\n", findMatchErrCount)
+	logger.Printf("\t%d documents cannot be fetched, %d out of them are deleted\n",
+		FetchDocumentErrCount, deleteDocCount)
+	logger.Printf("\t%d documents cannot update its creation time but still were inserted or updated in the index\n", SetCreatedErrCount)
+	logger.Printf("\t%d documents cannot be converted but still were inserted or updated in the index\n", convErrCount)
+}
+
+// CrawlFromSeed updates all the documents in seed, and crawls all the new
+// documents referred in the seed.
+func CrawlFromSeed(ctx context.Context, seed CrawlSeed, crawlers []Crawler,
+	conv Converter, indx IndexFunc, seen utils.SeenMap) {
+
+	// stack tracks the documents directly referred in other documents.
+	stack := make(CrawlSeed, 0)
+
+	// Exploit seed to update bulk of corpus.
+	logger.Printf("updating %d documents from seed\n", len(seed))
+	// each unique document in seed will be crawled once.
+	doCrawl(ctx, &seed, crawlers, conv, indx, seen, &stack, true, false)
+
+	// Traverse any new documents added while updating corpus.
+	logger.Printf("crawling %d new documents found in the seed\n", len(stack))
+	// While crawling each document in stack, the documents directly referred in the document
+	// will be added into stack.
+	// After this statement is done, stack will become empty.
+	doCrawl(ctx, &stack, crawlers, conv, indx, seen, &stack, false, true)
+}
+
+// CrawlGithubRunner is a blocking function and only returns once all of the
+// crawlers are finished with execution.
+//
+// This function uses the output channel to forward kustomization documents
+// from a list of crawlers. The output is to be consumed by a database/search
+// indexer for later retrieval.
+//
+// The return value is an array of errors in which each index represents the
+// index of the crawler that emitted the error. Although the errors themselves
+// can be nil, the array will always be exactly the size of the crawlers array.
+//
+// CrawlGithubRunner takes in a seed, which represents the documents stored in an
+// index somewhere. The document data is not required to be populated. If there
+// are many documents, this is preferable. The order of iteration over the seed
+// is not guaranteed, but the CrawlGithub does guarantee that every element
+// from the seed will be processed before any other documents from the
+// crawlers.
+func CrawlGithubRunner(ctx context.Context, output chan<- CrawledDocument,
+	crawlers []Crawler, seen utils.SeenMap) []error {
+
+	errs := make([]error, len(crawlers))
+	wg := sync.WaitGroup{}
+
+	for i, crawler := range crawlers {
+		// Crawler implementations get their own channels to prevent a
+		// crawler from closing the main output channel.
+		docs := make(chan CrawledDocument)
+		wg.Add(2)
+
+		// Forward all of the documents from this crawler's channel to
+		// the main output channel.
+		go func(docs <-chan CrawledDocument) {
+			defer wg.Done()
+			for d := range docs {
+				output <- d
+			}
+		}(docs)
+
+		// Run this crawler and capture its returned error.
+		go func(idx int, crawler Crawler,
+			docs chan<- CrawledDocument) {
+
+			defer func() {
+				wg.Done()
+				if r := recover(); r != nil {
+					errs[idx] = fmt.Errorf(
+						"%+v panicked: %v, additional error %v",
+						crawler, r, errs[idx],
+					)
+				}
+			}()
+			defer close(docs)
+			errs[idx] = crawler.Crawl(ctx, docs, seen)
+		}(i, crawler, docs) // Copies the index and the crawler
+	}
+
+	wg.Wait()
+	return errs
+}
+
+// CrawlGithub crawls all the kustomization files on Github.
+func CrawlGithub(ctx context.Context, crawlers []Crawler, conv Converter,
+	indx IndexFunc, seen utils.SeenMap) {
+	// stack tracks the documents directly referred in other documents.
+	stack := make(CrawlSeed, 0)
+
+	// ch is channel where all the crawlers sends the crawled documents to.
+	ch := make(chan CrawledDocument, 1<<10)
+
+	wg := sync.WaitGroup{}
+
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		docCount := 0
+		for cdoc := range ch {
+			docCount++
+			logger.Printf("Processing doc %d found on Github", docCount)
+			// all the docs here are kustomization files found by querying Github, and
+			// their `FileType` fields all should be empty.
+			if seen.Seen(cdoc.ID()) {
+				logger.Printf("the doc has been seen before")
+				continue
+			}
+			match := findMatch(cdoc.GetDocument(), crawlers)
+			if match == nil {
+				logIfErr(fmt.Errorf(
+					"%v could not match any crawler", cdoc))
+				continue
+			}
+			addBranches(cdoc, match, indx, seen, &stack)
+		}
+	}()
+
+	logger.Println("processing the documents found from crawling github")
+	if errs := CrawlGithubRunner(ctx, ch, crawlers, seen); errs != nil {
+		for _, err := range errs {
+			logIfErr(err)
+		}
+	}
+	close(ch)
+	wg.Wait()
+
+	// Handle deps of newly discovered documents.
+	logger.Printf("crawling the %d new documents referred by other documents",
+		len(stack))
+	doCrawl(ctx, &stack, crawlers, conv, indx, seen, &stack, false, true)
+}

api/internal/crawl/crawler/crawler_test.go

@@ -0,0 +1,343 @@
+package crawler
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"log"
+	"reflect"
+	"sort"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"sigs.k8s.io/kustomize/api/internal/crawl/utils"
+
+	"sigs.k8s.io/kustomize/api/internal/crawl/index"
+
+	"sigs.k8s.io/kustomize/api/internal/crawl/doc"
+	"sigs.k8s.io/kustomize/api/konfig"
+)
+
+const (
+	kustomizeRepo = "https://github.com/kubernetes-sigs/kustomize"
+)
+
+// Simple crawler that forwards it's list of documents to a provided channel and
+// returns it's error to the caller.
+type testCrawler struct {
+	matchPrefix string
+	err         error
+	docs        []doc.KustomizationDocument
+	lukp        map[string]int
+}
+
+func (c testCrawler) Match(d *doc.Document) bool {
+	return d != nil
+}
+
+func (c testCrawler) SetDefaultBranch(d *doc.Document) {}
+
+func (c testCrawler) FetchDocument(_ context.Context, d *doc.Document) error {
+	if i, ok := c.lukp[d.ID()]; ok {
+		d.DocumentData = c.docs[i].DocumentData
+		return nil
+	}
+	for _, suffix := range konfig.RecognizedKustomizationFileNames() {
+		savedFilePath := d.FilePath
+		d.FilePath += "/" + suffix
+		i, ok := c.lukp[d.ID()]
+		if !ok {
+			d.FilePath = savedFilePath
+			continue
+		}
+		d.DocumentData = c.docs[i].DocumentData
+		return nil
+	}
+	return fmt.Errorf("document %v does not exist for matcher: %s",
+		d, c.matchPrefix)
+}
+
+func (c testCrawler) SetCreated(_ context.Context, d *doc.Document) error {
+	d.CreationTime = &time.Time{}
+	return nil
+}
+
+func newCrawler(matchPrefix string, err error,
+	docs []doc.KustomizationDocument) testCrawler {
+	c := testCrawler{
+		matchPrefix: matchPrefix,
+		err:         err,
+		docs:        docs,
+		lukp:        make(map[string]int),
+	}
+	for i, d := range docs {
+		c.lukp[d.ID()] = i
+	}
+	return c
+}
+
+// Crawl implements the Crawler interface for testing.
+func (c testCrawler) Crawl(_ context.Context,
+	output chan<- CrawledDocument, _ utils.SeenMap) error {
+
+	for i, d := range c.docs {
+		isResource := true
+		for _, suffix := range konfig.RecognizedKustomizationFileNames() {
+			if strings.HasSuffix(d.FilePath, suffix) {
+				isResource = false
+				break
+			}
+		}
+		if isResource {
+			continue
+		}
+		output <- &c.docs[i]
+	}
+	return c.err
+}
+
+// Used to make sure that we're comparing documents in order. This is needed
+// since these documents will be sent concurrently.
+type sortableDocs []doc.KustomizationDocument
+
+func (s sortableDocs) Less(i, j int) bool {
+	return s[i].FilePath < s[j].FilePath
+}
+
+func (s sortableDocs) Swap(i, j int) {
+	s[i], s[j] = s[j], s[i]
+}
+
+func (s sortableDocs) Len() int {
+	return len(s)
+}
+
+func TestCrawlGithubRunner(t *testing.T) {
+	log.Println("testing CrawlGithubRunner")
+	tests := []struct {
+		tc   []Crawler
+		errs []error
+		docs sortableDocs
+	}{
+		{
+			tc: []Crawler{
+				testCrawler{
+					docs: []doc.KustomizationDocument{
+						{Document: doc.Document{
+							FilePath: "crawler1/doc1/kustomization.yaml",
+						}},
+						{Document: doc.Document{
+							FilePath: "crawler1/doc2/kustomization.yaml",
+						}},
+						{Document: doc.Document{
+							FilePath: "crawler1/doc3/kustomization.yaml",
+						}},
+					},
+				},
+				testCrawler{err: errors.New("crawler2")},
+				testCrawler{},
+				testCrawler{
+					docs: []doc.KustomizationDocument{
+						{Document: doc.Document{
+							FilePath: "crawler4/doc1/kustomization.yaml",
+						}},
+						{Document: doc.Document{
+							FilePath: "crawler4/doc2/kustomization.yaml",
+						}},
+					},
+					err: errors.New("crawler4"),
+				},
+			},
+			errs: []error{
+				nil,
+				errors.New("crawler2"),
+				nil,
+				errors.New("crawler4"),
+			},
+			docs: sortableDocs{
+				{Document: doc.Document{
+					FilePath: "crawler1/doc1/kustomization.yaml",
+				}},
+				{Document: doc.Document{
+					FilePath: "crawler1/doc2/kustomization.yaml",
+				}},
+				{Document: doc.Document{
+					FilePath: "crawler1/doc3/kustomization.yaml",
+				}},
+				{Document: doc.Document{
+					FilePath: "crawler4/doc1/kustomization.yaml",
+				}},
+				{Document: doc.Document{
+					FilePath: "crawler4/doc2/kustomization.yaml",
+				}},
+			},
+		},
+	}
+
+	for _, test := range tests {
+		output := make(chan CrawledDocument)
+		wg := sync.WaitGroup{}
+		wg.Add(1)
+
+		// Run the Crawler runner with a list of crawlers.
+		go func() {
+			defer close(output)
+			defer wg.Done()
+
+			seen := utils.NewSeenMap()
+			errs := CrawlGithubRunner(context.Background(),
+				output, test.tc, seen)
+
+			// Check that errors are returned as they should be.
+			if !reflect.DeepEqual(errs, test.errs) {
+				t.Errorf("Expected errs (%v) to equal (%v)",
+					errs, test.errs)
+			}
+
+		}()
+
+		// Iterate over the output channel of Crawler runner.
+		returned := make(sortableDocs, 0, len(test.docs))
+		for o := range output {
+			d, ok := o.(*doc.KustomizationDocument)
+			if !ok || d == nil {
+				t.Errorf("%T not expected type (%T)",
+					o, d)
+			}
+			returned = append(returned, *d)
+		}
+
+		// Check that all documents are received.
+		sort.Sort(returned)
+		if !reflect.DeepEqual(returned, test.docs) {
+			t.Errorf("Expected docs (%v) to equal (%v)\n",
+				returned, test.docs)
+		}
+
+		wg.Wait()
+	}
+}
+
+func TestCrawlFromSeed(t *testing.T) {
+	log.Println("testing CrawlFromSeed")
+
+	tests := []struct {
+		seed    CrawlSeed
+		matcher string
+		corpus  []doc.KustomizationDocument
+	}{
+		{
+			seed: CrawlSeed{
+				{
+					RepositoryURL: kustomizeRepo,
+					FilePath:      "examples/helloWorld/kustomization.yaml",
+				},
+				{
+					RepositoryURL: kustomizeRepo,
+					FilePath:      "examples/other/kustomization.yaml",
+				},
+			},
+			matcher: kustomizeRepo,
+			corpus: []doc.KustomizationDocument{
+				// Visited from the seed, will be ignored in the crawl.
+				{Document: doc.Document{
+					RepositoryURL: kustomizeRepo,
+					FilePath:      "examples/helloWorld/kustomization.yaml",
+					DocumentData: `
+resources:
+- deployment.yaml
+`,
+				}},
+				// Also visited from the seed as a relative resource.
+				{Document: doc.Document{
+					RepositoryURL: kustomizeRepo,
+					FilePath:      "examples/helloWorld/deployment.yaml",
+					DocumentData: `
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: hello
+`,
+				}},
+				// Visited from the seed. Has a remote import.
+				{Document: doc.Document{
+					RepositoryURL: kustomizeRepo,
+					FilePath:      "examples/other/kustomization.yaml",
+					DocumentData: `
+resources:
+- https://github.com/kubernetes-sigs/kustomize/examples/other/overlay
+- service.yaml
+`,
+				}},
+				// Imported as a base from the seed.
+				{Document: doc.Document{
+					RepositoryURL: kustomizeRepo,
+					FilePath:      "examples/other/overlay/kustomization.yaml",
+					DocumentData: `
+resources:
+- https://github.com/kubernetes-sigs/kustomize/examples/seedcrawl1
+- https://github.com/kubernetes-sigs/kustomize/examples/seedcrawl2
+`,
+				}},
+				// Imported as a resource from the seed.
+				{Document: doc.Document{
+					RepositoryURL: kustomizeRepo,
+					FilePath:      "examples/other/service.yaml",
+				}},
+				// Visited from crawling seed.
+				{Document: doc.Document{
+					RepositoryURL: kustomizeRepo,
+					FilePath:      "examples/seedcrawl1/kustomization.yml",
+				}},
+				// Visited from crawling seed.
+				{Document: doc.Document{
+					RepositoryURL: kustomizeRepo,
+					FilePath:      "examples/seedcrawl2/kustomization.yaml",
+					DocumentData: `
+resources:
+- ../base
+- job.yaml
+`,
+				}},
+				// Visited from crawling seed.
+				{Document: doc.Document{
+					RepositoryURL: kustomizeRepo,
+					FilePath:      "examples/base/kustomization.yml",
+				}},
+				// Visited from crawling seed imported as resource.
+				{Document: doc.Document{
+					RepositoryURL: kustomizeRepo,
+					FilePath:      "examples/seedcrawl2/job.yaml",
+				}},
+			},
+		},
+	}
+
+	for _, tc := range tests {
+		cr := newCrawler(tc.matcher, nil, tc.corpus)
+		visited := make(map[string]int)
+		CrawlFromSeed(context.Background(), tc.seed, []Crawler{cr},
+			func(d *doc.Document) (CrawledDocument, error) {
+				return &doc.KustomizationDocument{
+					Document: *d,
+				}, nil
+			},
+			func(d CrawledDocument, mode index.Mode) error {
+				visited[d.ID()]++
+				return nil
+			},
+			utils.NewSeenMap(),
+		)
+		if lv, lc := len(visited), len(tc.corpus); lv != lc {
+			t.Errorf("error: %d of %d documents visited.", lv, lc)
+			t.Errorf("\nvisited (%v)\nexpected (%v).", visited, cr.lukp)
+		}
+		for id, cnt := range visited {
+			if cnt != 1 {
+				t.Errorf("%s not visited once (%d)", id, cnt)
+			}
+		}
+	}
+}

api/internal/crawl/crawler/github/crawler.go

@@ -0,0 +1,759 @@
+// Package github implements the crawler.Crawler interface, getting data
+// from the Github search API.
+package github
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"math"
+	"net/http"
+	"os"
+	"regexp"
+	"strconv"
+	"strings"
+	"time"
+
+	"sigs.k8s.io/kustomize/api/internal/crawl/utils"
+
+	"sigs.k8s.io/kustomize/api/internal/crawl/crawler"
+	"sigs.k8s.io/kustomize/api/internal/crawl/doc"
+	"sigs.k8s.io/kustomize/api/internal/crawl/httpclient"
+	"sigs.k8s.io/kustomize/api/internal/git"
+	"sigs.k8s.io/kustomize/api/konfig"
+)
+
+var logger = log.New(os.Stdout, "Github Crawler: ",
+	log.LstdFlags|log.LUTC|log.Llongfile)
+
+// Implements crawler.Crawler.
+type githubCrawler struct {
+	client GhClient
+	query  Query
+	// branchMap maps github repositories to their default branches
+	branchMap map[string]string
+}
+
+type GhClient struct {
+	RequestConfig
+	retryCount  uint64
+	client      *http.Client
+	accessToken string
+}
+
+func NewCrawler(accessToken string, retryCount uint64, client *http.Client,
+	query Query) githubCrawler {
+
+	return githubCrawler{
+		client: GhClient{
+			retryCount: retryCount,
+			client:     client,
+			RequestConfig: RequestConfig{
+				perPage: githubMaxPageSize,
+			},
+			accessToken: accessToken,
+		},
+		query:     query,
+		branchMap: map[string]string{},
+	}
+}
+
+func (gc githubCrawler) SetDefaultBranch(d *doc.Document) {
+	url := gc.client.ReposRequest(d.RepositoryFullName())
+	defaultBranch, err := gc.client.GetDefaultBranch(url, d.RepositoryURL, gc.branchMap)
+	if err != nil {
+		logger.Printf(
+			"(error: %v) setting default_branch to master\n", err)
+		defaultBranch = "master"
+	}
+	d.DefaultBranch = defaultBranch
+	gc.branchMap[d.RepositoryURL] = d.DefaultBranch
+}
+
+func (gc githubCrawler) DefaultBranch(repo string) string {
+	return gc.branchMap[repo]
+}
+
+// Implements crawler.Crawler.
+func (gc githubCrawler) Crawl(ctx context.Context,
+	output chan<- crawler.CrawledDocument, seen utils.SeenMap) error {
+
+	ranges := []RangeWithin{
+		RangeWithin{
+		start: uint64(0),
+		end:   githubMaxFileSize,
+	},
+	}
+
+	errs := make(multiError, 0)
+	for len(ranges) > 0 {
+		tailRange := ranges[len(ranges) - 1]
+		ranges = ranges[:(len(ranges) - 1)]
+		reProcessQueryRanges, err := gc.CrawlSingleRange(ctx, output, seen, tailRange.start, tailRange.end)
+		if err != nil {
+			errs = append(errs, err)
+		}
+		ranges = append(ranges, reProcessQueryRanges...)
+	}
+
+	if len(errs) > 0 {
+		return errs
+	}
+	return nil
+}
+
+func (gc githubCrawler) CrawlSingleRange(ctx context.Context,
+	output chan<- crawler.CrawledDocument, seen utils.SeenMap,
+	lowerBound, upperBound uint64) ([]RangeWithin, error) {
+
+	log.Printf("CrawlSingleRange [%d, %d]", lowerBound, upperBound)
+
+	noETagClient := GhClient{
+		RequestConfig: gc.client.RequestConfig,
+		client:        &http.Client{Timeout: gc.client.client.Timeout},
+		retryCount:    gc.client.retryCount,
+		accessToken:   gc.client.accessToken,
+	}
+
+	var reProcessQueryRanges []RangeWithin
+
+	var ranges []string
+	var err error
+	// Since Github returns a max of 1000 results per query, we can use
+	// multiple queries that split the search space into chunks of at most
+	// 1000 files to get all of the data.
+	for i := 0; i < 5; i++ {
+		ranges, err = FindRangesForRepoSearch(newCache(noETagClient, gc.query),
+			lowerBound, upperBound)
+		if err == nil {
+			logger.Printf("FindRangesForRepoSearch succeeded after %d retries", i)
+			break
+		} else {
+			time.Sleep(time.Minute)
+		}
+	}
+	if err != nil {
+		return reProcessQueryRanges, fmt.Errorf("could not split %v into ranges, %v\n",
+			gc.query, err)
+	}
+
+	logger.Println("ranges: ", ranges)
+
+	// Query each range for files.
+	errs := make(multiError, 0)
+	queryResult := RangeQueryResult{}
+	for _, query := range ranges {
+		reProcessQuery, rangeResult, err := processQuery(ctx, gc.client, query, output, seen, gc.branchMap)
+		if err != nil {
+			errs = append(errs, err)
+		}
+		queryResult.Add(rangeResult)
+		if reProcessQuery {
+			reProcessQueryRanges = append(reProcessQueryRanges, RangeSizes(query))
+		}
+	}
+
+	logger.Printf("Summary of Crawl: %s", queryResult.String())
+
+	if len(errs) > 0 {
+		return reProcessQueryRanges, errs
+	}
+
+	return reProcessQueryRanges, nil
+}
+
+// FetchDocument first tries to fetch the document with d.FilePath. If it fails,
+// it will try to add each string in konfig.RecognizedKustomizationFileNames() to
+// d.FilePath, and try to fetch the document again.
+func (gc githubCrawler) FetchDocument(_ context.Context, d *doc.Document) error {
+	repoURL := d.RepositoryURL + "/" + d.FilePath + "?ref=" + d.DefaultBranch
+	repoSpec, err := git.NewRepoSpecFromUrl(repoURL)
+	if err != nil {
+		return fmt.Errorf("invalid repospec: %v", err)
+	}
+
+	url := "https://raw.githubusercontent.com/" + repoSpec.OrgRepo +
+		"/" + repoSpec.Ref + "/" + repoSpec.Path
+
+	handle := func(resp *http.Response, err error, path string) error {
+		if resp == nil {
+			return fmt.Errorf("empty http response (url: %s; path: %s), error: %v",
+				url, path, err)
+		}
+		if err == nil && resp.StatusCode == http.StatusOK {
+			d.IsSame = httpclient.FromCache(resp.Header)
+			defer CloseResponseBody(resp)
+			data, err := ioutil.ReadAll(resp.Body)
+			if err != nil {
+				return err
+			}
+			d.DocumentData = string(data)
+			d.FilePath = d.FilePath + path
+			return nil
+		}
+
+		return err
+	}
+	resp, errGetRawUserContent := gc.client.GetRawUserContent(url)
+	if err := handle(resp, errGetRawUserContent, ""); err == nil {
+		return nil
+	}
+
+	for _, file := range konfig.RecognizedKustomizationFileNames() {
+		resp, errGetRawUserContent = gc.client.GetRawUserContent(url + "/" + file)
+		if err = handle(resp, errGetRawUserContent, "/"+file); err == nil {
+			return nil
+		}
+	}
+	return fmt.Errorf("file not found: %s, error: %v", url, err)
+}
+
+func (gc githubCrawler) SetCreated(_ context.Context, d *doc.Document) error {
+	fs := GhFileSpec{
+		Path: d.FilePath,
+		Repository: GitRepository{
+			FullName: d.RepositoryFullName(),
+		},
+	}
+	creationTime, err := gc.client.GetFileCreationTime(fs)
+	if err != nil {
+		return err
+	}
+	d.CreationTime = &creationTime
+	return nil
+}
+
+func (gc githubCrawler) Match(d *doc.Document) bool {
+	url := d.RepositoryURL + "/" + d.FilePath + "?ref=" + "/" +
+		d.DefaultBranch
+	repoSpec, err := git.NewRepoSpecFromUrl(url)
+	if err != nil {
+		return false
+	}
+
+	return strings.Contains(repoSpec.Host, "github.com")
+}
+
+type RangeQueryResult struct {
+	totalDocCnt uint64
+	seenDocCnt  uint64
+	newDocCnt   uint64
+	errorCnt    uint64
+}
+
+func (r *RangeQueryResult) Add(other RangeQueryResult) {
+	r.totalDocCnt += other.totalDocCnt
+	r.newDocCnt += other.newDocCnt
+	r.seenDocCnt += other.seenDocCnt
+	r.errorCnt += other.errorCnt
+}
+
+func (r *RangeQueryResult) String() string {
+	return fmt.Sprintf("got %d files from API. "+
+		"%d have been seen before. %d are new and sent to the output channel."+
+		" %d have kustomizationResultAdapter errors.",
+		r.totalDocCnt, r.seenDocCnt, r.newDocCnt, r.errorCnt)
+}
+
+// processQuery follows all of the pages in a query, and updates/adds the
+// documents from the crawl to the datastore/index.
+func processQuery(ctx context.Context, gcl GhClient, query string,
+	output chan<- crawler.CrawledDocument, seen utils.SeenMap,
+	branchMap map[string]string) (bool, RangeQueryResult, error) {
+
+	queryPages := make(chan GhResponseInfo)
+
+	go func() {
+		// Forward the document metadata to the retrieval channel.
+		// This separation allows for concurrent requests for the code
+		// search, and the retrieval portions of the API.
+		err := gcl.ForwardPaginatedQuery(ctx, query, queryPages)
+		if err != nil {
+			// TODO(damienr74) handle this error with redis?
+			logger.Println(err)
+		}
+		close(queryPages)
+	}()
+
+	reProcessQuery := false
+
+	errs := make(multiError, 0)
+	result := RangeQueryResult{}
+	pageID := 1
+	for page := range queryPages {
+		if page.Error != nil {
+			errs = append(errs, page.Error)
+			continue
+		}
+		pageResult := RangeQueryResult{}
+		for _, file := range page.Parsed.Items {
+			k, err := kustomizationResultAdapter(gcl, file, seen, branchMap)
+			if err != nil {
+				logger.Printf("kustomizationResultAdapter failed: %v", err)
+				errs = append(errs, err)
+				pageResult.errorCnt++
+			}
+			if k != nil {
+				pageResult.newDocCnt++
+				output <- k
+			} else {
+				pageResult.seenDocCnt++
+			}
+			pageResult.totalDocCnt++
+		}
+
+		logger.Printf("processQuery [TotalCount %d - page %d]: %s",
+			page.Parsed.TotalCount, pageID, pageResult.String())
+		result.Add(pageResult)
+
+		pageID++
+
+		if page.Parsed.TotalCount > githubMaxResultsPerQuery {
+			reProcessQuery = true
+		}
+	}
+
+	logger.Printf("Summary of processQuery: %s", result.String())
+
+	return reProcessQuery, result, errs
+}
+
+func kustomizationResultAdapter(gcl GhClient, k GhFileSpec, seen utils.SeenMap,
+	branchMap map[string]string) (crawler.CrawledDocument, error) {
+	url := gcl.ReposRequest(k.Repository.FullName)
+	defaultBranch, err := gcl.GetDefaultBranch(url, k.Repository.URL, branchMap)
+	if err != nil {
+		logger.Printf(
+			"(error: %v) setting default_branch to master\n", err)
+		defaultBranch = "master"
+	}
+
+	// document here is a kustomization file found by querying Github, whose
+	// `FileType` field should be empty.
+	document := doc.Document{
+		FilePath:      k.Path,
+		DefaultBranch: defaultBranch,
+		RepositoryURL: k.Repository.URL,
+		User:          doc.UserName(k.Repository.URL),
+	}
+
+	if seen.Seen(document.ID()) {
+		return nil, nil
+	}
+
+	data, err := gcl.GetFileData(k)
+	if err != nil {
+		return nil, err
+	}
+
+	d := doc.KustomizationDocument{
+		Document: doc.Document{
+			DocumentData:  string(data),
+			FilePath:      k.Path,
+			DefaultBranch: defaultBranch,
+			RepositoryURL: k.Repository.URL,
+			User:          doc.UserName(k.Repository.URL),
+		},
+	}
+	creationTime, err := gcl.GetFileCreationTime(k)
+	if err != nil {
+		logger.Printf("GetFileCreationTime failed: %v", err)
+		return &d, err
+	}
+	d.CreationTime = &creationTime
+
+	if err := d.ParseYAML(); err != nil {
+		logger.Printf("ParseYAML failed: %v", err)
+		return &d, err
+	}
+
+	return &d, nil
+}
+
+// ForwardPaginatedQuery follows the links to the next pages and performs all of
+// the queries for a given search query, relaying the data from each request
+// back to an output channel.
+func (gcl GhClient) ForwardPaginatedQuery(ctx context.Context, query string,
+	output chan<- GhResponseInfo) error {
+
+	logger.Println("querying: ", query)
+	response := gcl.parseGithubResponseWithRetry(query)
+
+	if response.Error != nil {
+		return response.Error
+	}
+
+	output <- response
+
+	for response.LastURL != "" && response.NextURL != "" {
+		select {
+		case <-ctx.Done():
+			return nil
+		default:
+			response = gcl.parseGithubResponseWithRetry(response.NextURL)
+			if response.Error != nil {
+				return response.Error
+			}
+
+			output <- response
+		}
+	}
+
+	return nil
+}
+
+// GetFileData gets the bytes from a file.
+func (gcl GhClient) GetFileData(k GhFileSpec) ([]byte, error) {
+
+	url := gcl.ContentsRequest(k.Repository.FullName, k.Path)
+
+	resp, err := gcl.GetReposData(url)
+	if err != nil {
+		return nil, fmt.Errorf("%+v: could not get '%s' metadata: %v",
+			k, url, err)
+	}
+
+	data, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return nil, fmt.Errorf("%+v: could not read '%s' metadata: %v",
+			k, url, err)
+	}
+
+	if err := resp.Body.Close(); err != nil {
+		return nil, err
+	}
+
+	type githubContentRawURL struct {
+		DownloadURL string `json:"download_url,omitempty"`
+	}
+	var rawURL githubContentRawURL
+	err = json.Unmarshal(data, &rawURL)
+	if err != nil {
+		return nil, fmt.Errorf(
+			"%+v: could not get 'download_url' from '%s' response: %v",
+			k, data, err)
+	}
+
+	resp, err = gcl.GetRawUserContent(rawURL.DownloadURL)
+	if err != nil {
+		return nil, fmt.Errorf("%+v: could not fetch file raw data '%s': %v",
+			k, rawURL.DownloadURL, err)
+	}
+
+	defer CloseResponseBody(resp)
+	data, err = ioutil.ReadAll(resp.Body)
+	return data, err
+}
+
+func CloseResponseBody(resp *http.Response) {
+	if err := resp.Body.Close(); err != nil {
+		log.Printf("failed to close response body: %v", err)
+	}
+}
+
+// GetDefaultBranch gets the default branch of a github repository.
+// m is a map which maps a github repository to its default branch.
+// If repo is already in m, the default branch for url will be obtained from m;
+// otherwise, a query will be made to github to obtain the default branch.
+func (gcl GhClient) GetDefaultBranch(url, repo string, m map[string]string) (string, error) {
+	if v, ok := m[repo]; ok {
+		return v, nil
+	}
+
+	resp, err := gcl.GetReposData(url)
+	if err != nil {
+		return "", fmt.Errorf(
+			"'%s' could not get default_branch: %v", url, err)
+	}
+	defer CloseResponseBody(resp)
+	data, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return "", fmt.Errorf(
+			"could not read default_branch: %v", err)
+	}
+
+	type defaultBranch struct {
+		DefaultBranch string `json:"default_branch,omitempty"`
+	}
+	var branch defaultBranch
+	err = json.Unmarshal(data, &branch)
+	if err != nil {
+		return "", fmt.Errorf(
+			"default_branch json malformed: %v", err)
+	}
+
+	return branch.DefaultBranch, nil
+}
+
+// GetFileCreationTime gets the earliest date of a file.
+func (gcl GhClient) GetFileCreationTime(
+	k GhFileSpec) (time.Time, error) {
+
+	url := gcl.CommitsRequest(k.Repository.FullName, k.Path)
+
+	defaultTime := time.Now()
+
+	resp, err := gcl.GetReposData(url)
+	if err != nil {
+		return defaultTime, fmt.Errorf(
+			"%+v: '%s' could not get metadata: %v", k, url, err)
+	}
+
+	type DateSpec struct {
+		Commit struct {
+			Author struct {
+				Date string `json:"date,omitempty"`
+			} `json:"author,omitempty"`
+		} `json:"commit,omitempty"`
+	}
+
+	_, lastURL := parseGithubLinkFormat(resp.Header.Get("link"))
+	if lastURL != "" {
+		resp, err = gcl.GetReposData(lastURL)
+		if err != nil {
+			return defaultTime, fmt.Errorf(
+				"%+v: '%s' could not get metadata: %v",
+				k, lastURL, err)
+		}
+	}
+
+	defer CloseResponseBody(resp)
+	data, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return defaultTime, fmt.Errorf(
+			"%+v: failed to read metadata: %v", k, err)
+	}
+	var earliestDate []DateSpec
+	err = json.Unmarshal(data, &earliestDate)
+	size := len(earliestDate)
+	if err != nil || size == 0 {
+		return defaultTime, fmt.Errorf(
+			"%+v: server response '%s' not in expected format: %v",
+			k, data, err)
+	}
+
+	return time.Parse(time.RFC3339, earliestDate[size-1].Commit.Author.Date)
+}
+
+// TODO(damienr74) change the tickers to actually check api rate limits, reset
+// times, and throttle requests dynamically based off of current utilization,
+// instead of hardcoding the documented values, these calls are not quota'd.
+// This is now especially important, since caching the API requests will reduce
+// API quota use (so we can actually make more requests in the allotted time
+// period).
+//
+// See https://developer.github.com/v3/rate_limit/ for details.
+var (
+	searchRateTicker  = time.NewTicker(time.Second * 2)
+	contentRateTicker = time.NewTicker(time.Second * 1)
+)
+
+func throttleSearchAPI() {
+	<-searchRateTicker.C
+}
+
+func throttleRepoAPI() {
+	<-contentRateTicker.C
+}
+
+type multiError []error
+
+func (e multiError) Error() string {
+	size := len(e) + 2
+	strs := make([]string, size)
+	strs[0] = "Errors ["
+	for i, err := range e {
+		strs[i+1] = "\t" + err.Error()
+	}
+	strs[size-1] = "]"
+	return strings.Join(strs, "\n")
+}
+
+type GitRepository struct {
+	API      string `json:"url,omitempty"`
+	URL      string `json:"html_url,omitempty"`
+	FullName string `json:"full_name,omitempty"`
+}
+
+type GhFileSpec struct {
+	Path       string        `json:"path,omitempty"`
+	Repository GitRepository `json:"repository,omitempty"`
+}
+
+type githubResponse struct {
+	// MaxUint is reserved as a sentinel value.
+	// This is the number of files that match the query.
+	TotalCount uint64 `json:"total_count,omitempty"`
+
+	IncompleteResults bool `json:"incomplete_results,omitempty"`
+
+	// Github representation of a file.
+	Items []GhFileSpec `json:"items,omitempty"`
+}
+
+type GhResponseInfo struct {
+	*http.Response
+	Parsed  *githubResponse
+	Error   error
+	NextURL string
+	LastURL string
+}
+
+func parseGithubLinkFormat(links string) (string, string) {
+	const (
+		linkNext    = "next"
+		linkLast    = "last"
+		linkInfoURL = 1
+		linkInfoRel = 2
+	)
+
+	next, last := "", ""
+	linkInfo := regexp.MustCompile(`<(.*)>.*; rel="(last|next)"`)
+
+	for _, link := range strings.Split(links, ",") {
+		linkParse := linkInfo.FindStringSubmatch(link)
+		if len(linkParse) != 3 {
+			continue
+		}
+
+		url := linkParse[linkInfoURL]
+		switch linkParse[linkInfoRel] {
+		case linkNext:
+			next = url
+		case linkLast:
+			last = url
+		default:
+		}
+	}
+
+	return next, last
+}
+
+func (gcl GhClient) parseGithubResponseWithRetry(getRequest string) GhResponseInfo {
+	resp := gcl.parseGithubResponse(getRequest)
+	retries := 0
+	for resp.Parsed.IncompleteResults {
+		resp = gcl.parseGithubResponse(getRequest)
+		retries++
+	}
+	log.Printf("The result of query(%s) is complete after %d retries", getRequest, retries)
+	return resp
+}
+
+func (gcl GhClient) parseGithubResponse(getRequest string) GhResponseInfo {
+	resp, err := gcl.SearchGithubAPI(getRequest)
+	requestInfo := GhResponseInfo{
+		Response: resp,
+		Error:    err,
+		Parsed:   nil,
+	}
+
+	if err != nil || resp == nil {
+		return requestInfo
+	}
+
+	var data []byte
+	defer CloseResponseBody(resp)
+	data, requestInfo.Error = ioutil.ReadAll(resp.Body)
+	if requestInfo.Error != nil {
+		return requestInfo
+	}
+
+	if resp.StatusCode != http.StatusOK {
+		logger.Println("query: ", getRequest)
+		logger.Println("status not OK at the source")
+		logger.Println("header dump", resp.Header)
+		logger.Println("body dump", string(data))
+		requestInfo.Error = fmt.Errorf("request rejected, status '%s'",
+			resp.Status)
+		return requestInfo
+	}
+
+	requestInfo.NextURL, requestInfo.LastURL =
+		parseGithubLinkFormat(resp.Header.Get("link"))
+
+	resultCount := githubResponse{
+		TotalCount: math.MaxUint64,
+	}
+	requestInfo.Error = json.Unmarshal(data, &resultCount)
+	if requestInfo.Error != nil {
+		return requestInfo
+	}
+
+	requestInfo.Parsed = &resultCount
+
+	return requestInfo
+
+}
+
+// SearchGithubAPI performs a search query and handles rate limitting for
+// the 'search/code?' endpoint as well as timed retries in the case of abuse
+// prevention.
+func (gcl GhClient) SearchGithubAPI(query string) (*http.Response, error) {
+	throttleSearchAPI()
+	return gcl.getWithRetry(query)
+}
+
+// GetReposData performs a search query and handles rate limitting for
+// the '/repos' endpoint as well as timed retries in the case of abuse
+// prevention.
+func (gcl GhClient) GetReposData(query string) (*http.Response, error) {
+	throttleRepoAPI()
+	return gcl.getWithRetry(query)
+}
+
+// User content (file contents) is not API rate limited, so there's no use in
+// throttling this call.
+func (gcl GhClient) GetRawUserContent(query string) (*http.Response, error) {
+	return gcl.getWithRetry(query)
+}
+
+func (gcl GhClient) Do(query string) (*http.Response, error) {
+	req, err := http.NewRequest("GET", query, nil)
+	if err != nil {
+		return nil, err
+	}
+	req.Header.Add("Authorization", fmt.Sprintf("token %s", gcl.accessToken))
+
+	// gcl.client.Do: a non-2xx status code doesn't cause an error.
+	// See https://golang.org/pkg/net/http/#Client.Do for more info.
+	resp, err := gcl.client.Do(req)
+	if resp != nil && resp.StatusCode != http.StatusOK {
+		err = fmt.Errorf("GhClient.Do(%s) failed with response code: %d",
+			query, resp.StatusCode)
+	}
+	return resp, err
+}
+
+func (gcl GhClient) getWithRetry(
+	query string) (resp *http.Response, err error) {
+
+	resp, err = gcl.Do(query)
+
+	retryCount := gcl.retryCount
+
+	for resp != nil && resp.StatusCode == http.StatusForbidden && retryCount > 0 {
+		retryTime := resp.Header.Get("Retry-After")
+		i, errAtoi := strconv.Atoi(retryTime)
+		if errAtoi != nil {
+			return resp, fmt.Errorf(
+				"query '%s' forbidden without 'Retry-After'", query)
+		}
+		logger.Printf(
+			"status forbidden, retring %d more times\n", retryCount)
+
+		logger.Printf("waiting %d seconds before retrying\n", i)
+		time.Sleep(time.Second * time.Duration(i))
+		retryCount--
+		resp, err = gcl.Do(query)
+	}
+
+	if err != nil {
+		return resp, fmt.Errorf("query '%s' could not be processed, %v",
+			query, err)
+	}
+
+	return resp, err
+}

api/internal/crawl/crawler/github/queries.go

@@ -0,0 +1,227 @@
+package github
+
+import (
+	"fmt"
+	"net/url"
+	"strings"
+)
+
+const (
+	perPageArg = "per_page"
+)
+
+const githubMaxPageSize = 100
+
+// Implementation detail, not important to external API.
+type queryField struct {
+	name  string
+	value interface{}
+}
+
+// Formats a query field.
+func (qf queryField) String() string {
+	var value string
+	switch v := qf.value.(type) {
+	case string:
+		value = v
+	case rangeFormatter:
+		value = v.RangeString()
+	default:
+		value = fmt.Sprint(v)
+	}
+
+	if qf.name == "" {
+		return value
+	}
+	return fmt.Sprint(qf.name, ":", value)
+}
+
+// Example of formating a query:
+// QueryWith(
+//	Filename("kustomization.yaml"),
+//	Filesize(RangeWithin{64, 192}),
+//	Keyword("copyright"),
+//	Keyword("2019"),
+// ).String()
+//
+// Outputs "q=filename:kustomization.yaml+size:64..192+copyright+2018" which
+// would search for files that have [64, 192] bytes (inclusive range) and that
+// contain the keywords 'copyright' and '2019' somewhere in the file.
+type Query []queryField
+
+func QueryWith(qfs ...queryField) Query {
+	return qfs
+}
+
+func (q Query) String() string {
+	strs := make([]string, 0, len(q))
+	for _, elem := range q {
+		str := elem.String()
+		if str == "" {
+			continue
+		}
+		strs = append(strs, str)
+	}
+
+	query := strings.Join(strs, "+")
+	if query == "" {
+		return query
+	}
+	return "q=" + query
+}
+
+// Keyword takes a single word, and formats it according to the Github API.
+func Keyword(k string) queryField {
+	return queryField{value: k}
+}
+
+// Filesize takes a rangeFormatter and formats it according to the Github API.
+func Filesize(r rangeFormatter) queryField {
+	return queryField{name: "size", value: r}
+}
+
+// Filename takes a filename and formats it according to the Github API.
+func Filename(f string) queryField {
+	return queryField{name: "filename", value: f}
+}
+
+// Path takes a filepath and formats it according to the Github API.
+func Path(p string) queryField {
+	return queryField{name: "path", value: p}
+}
+
+// Repo takes a repository (i.e., kubernetes-sigs/kustomize) and formats
+// it according to the Github API.
+func Repo(r string) queryField {
+	return queryField{name: "repo", value: r}
+}
+
+// Path takes a github username and formats it according to the Github API.
+func User(u string) queryField {
+	return queryField{name: "user", value: u}
+}
+
+// RequestConfig stores common variables that must be present for the queries.
+// - CodeSearchRequests: ask Github to check the code indices given a query.
+// - ContentsRequests: ask Github where to download a resource given a repo and a
+// file path.
+// - CommitsRequests: asks Github to list commits made one a file. Useful to
+// determine the date of a file.
+type RequestConfig struct {
+	perPage uint64
+}
+
+// CodeSearchRequestWith given a list of query parameters that specify the
+// (patial) query, returns a request object with the (parital) query. Must call
+// the URL method to get the string value of the URL. See request.CopyWith, to
+// understand why the request object is useful.
+func (rc RequestConfig) CodeSearchRequestWith(query Query) request {
+	vals := url.Values{
+		"sort":  []string{"indexed"},
+		"order": []string{"desc"},
+	}
+	req := rc.makeRequest("search/code", query, vals)
+	return req
+}
+
+// ContentsRequest given the repo name, and the filepath returns a formatted
+// query for the Github API to find the dowload information of this filepath.
+func (rc RequestConfig) ContentsRequest(fullRepoName, path string) string {
+	uri := fmt.Sprintf("repos/%s/contents/%s", fullRepoName, path)
+	return rc.makeRequest(uri, Query{}, url.Values{}).URL()
+}
+
+func (rc RequestConfig) ReposRequest(fullRepoName string) string {
+	uri := fmt.Sprintf("repos/%s", fullRepoName)
+	return rc.makeRequest(uri, Query{}, url.Values{}).URL()
+}
+
+// CommitsRequest given the repo name, and a filepath returns a formatted query
+// for the Github API to find the commits that affect this file.
+func (rc RequestConfig) CommitsRequest(fullRepoName, path string) string {
+	uri := fmt.Sprintf("repos/%s/commits", fullRepoName)
+	vals := url.Values{
+		"path": []string{path},
+	}
+	return rc.makeRequest(uri, Query{}, vals).URL()
+}
+
+func (rc RequestConfig) makeRequest(path string, query Query, vals url.Values) request {
+	vals.Set(perPageArg, fmt.Sprint(rc.perPage))
+
+	return request{
+		url: url.URL{
+			Scheme: "https",
+			Host:   "api.github.com",
+			Path:   path,
+		},
+		vals:  vals,
+		query: query,
+	}
+}
+
+type request struct {
+	url   url.URL
+	vals  url.Values
+	query Query
+}
+
+// CopyWith copies the requests and adds the extra query parameters. It is useful
+// for dynamically adding sizes to a filename only query without modifying it.
+func (r request) CopyWith(queryParams ...queryField) request {
+	cpy := r
+	cpy.query = append(cpy.query, queryParams...)
+	return cpy
+}
+
+// URL encodes the variables and the URL representation into a string.
+func (r request) URL() string {
+	// Github does not handle URL encoding properly in its API for the
+	// q='...', so the query parameter is added without any encoding
+	// manually.
+	encoded := r.vals.Encode()
+	query := r.query.String()
+	sep := "&"
+	if query == "" {
+		sep = ""
+	}
+	if encoded == "" && query != "" {
+		sep = "?"
+	}
+	r.url.RawQuery = query + sep + encoded
+	return r.url.String()
+}
+
+// Allows to define a range of numbers and print it in the github range
+// query format https://help.github.com/en/articles/understanding-the-search-syntax.
+type rangeFormatter interface {
+	RangeString() string
+}
+
+// RangeLessThan is a range of values strictly less than (<) size.
+type RangeLessThan struct {
+	size uint64
+}
+
+func (r RangeLessThan) RangeString() string {
+	return fmt.Sprintf("<%d", r.size)
+}
+
+// RangeLessThan is a range of values strictly greater than (>) size.
+type RangeGreaterThan struct {
+	size uint64
+}
+
+func (r RangeGreaterThan) RangeString() string {
+	return fmt.Sprintf(">%d", r.size)
+}
+
+// RangeWithin is an inclusive range from start to end.
+type RangeWithin struct {
+	start uint64
+	end   uint64
+}
+
+func (r RangeWithin) RangeString() string {
+	return fmt.Sprintf("%d..%d", r.start, r.end)
+}

api/internal/crawl/crawler/github/queries_test.go

@@ -0,0 +1,140 @@
+package github
+
+import (
+	"testing"
+)
+
+func TestQueryFields(t *testing.T) {
+	testCases := []struct {
+		formatter queryField
+		expected  string
+	}{
+		{
+			formatter: Keyword("keyword"),
+			expected:  "keyword",
+		},
+		{
+			formatter: Filesize(RangeLessThan{23}),
+			expected:  "size:<23",
+		},
+		{
+			formatter: Filesize(RangeWithin{24, 64}),
+			expected:  "size:24..64",
+		},
+		{
+			formatter: Filesize(RangeGreaterThan{64}),
+			expected:  "size:>64",
+		},
+		{
+			formatter: Path("some/path/to/file"),
+			expected:  "path:some/path/to/file",
+		},
+		{
+			formatter: Filename("kustomization.yaml"),
+			expected:  "filename:kustomization.yaml",
+		},
+	}
+
+	for _, test := range testCases {
+		if result := test.formatter.String(); result != test.expected {
+			t.Errorf("got (%#v = %s), expected %s", test.formatter, result, test.expected)
+		}
+	}
+}
+
+func TestQueryType(t *testing.T) {
+	testCases := []struct {
+		query    Query
+		expected string
+	}{
+		{
+			query: QueryWith(
+				Filesize(RangeWithin{24, 64}),
+				Filename("kustomization.yaml"),
+				Keyword("keyword1"),
+				Keyword("keyword2"),
+				Repo("user1/repo1"),
+				User("user1"),
+			),
+			expected: "q=size:24..64+filename:kustomization.yaml+keyword1+keyword2+" +
+				"repo:user1/repo1+user:user1",
+		},
+	}
+
+	for _, test := range testCases {
+		if queryStr := test.query.String(); queryStr != test.expected {
+			t.Errorf("got (%#v = %s), expected %s", test.query, queryStr, test.expected)
+		}
+
+	}
+}
+
+func TestGithubSearchQuery(t *testing.T) {
+	const (
+		perPage = 100
+	)
+
+	testCases := []struct {
+		rc                    RequestConfig
+		codeQuery             Query
+		fullRepoName          string
+		path                  string
+		expectedCodeQuery     string
+		expectedContentsQuery string
+		expectedCommitsQuery  string
+	}{
+		{
+			rc: RequestConfig{
+				perPage: perPage,
+			},
+			codeQuery: Query{
+				Filename("kustomization.yaml"),
+				Filesize(RangeWithin{64, 128}),
+			},
+			fullRepoName: "kubernetes-sigs/kustomize",
+			path:         "examples/helloWorld/kustomization.yaml",
+
+			expectedCodeQuery: "https://api.github.com/search/code?" +
+				"q=filename:kustomization.yaml+size:64..128&order=desc&per_page=100&sort=indexed",
+
+			expectedContentsQuery: "https://api.github.com/repos/kubernetes-sigs/kustomize/contents/" +
+				"examples/helloWorld/kustomization.yaml?per_page=100",
+
+			expectedCommitsQuery: "https://api.github.com/repos/kubernetes-sigs/kustomize/commits?" +
+				"path=examples%2FhelloWorld%2Fkustomization.yaml&per_page=100",
+		},
+		{
+			rc: RequestConfig{
+				perPage: perPage,
+			},
+			codeQuery: Query{
+				Filename("kustomization.yaml"),
+				Filesize(RangeWithin{64, 128}),
+			},
+			fullRepoName: "kubernetes-sigs/kustomize",
+			path:         "examples 1/helloWorld/kustomization.yaml",
+
+			expectedCodeQuery: "https://api.github.com/search/code?" +
+				"q=filename:kustomization.yaml+size:64..128&order=desc&per_page=100&sort=indexed",
+
+			expectedContentsQuery: "https://api.github.com/repos/kubernetes-sigs/kustomize/contents/" +
+				"examples%201/helloWorld/kustomization.yaml?per_page=100",
+
+			expectedCommitsQuery: "https://api.github.com/repos/kubernetes-sigs/kustomize/commits?" +
+				"path=examples+1%2FhelloWorld%2Fkustomization.yaml&per_page=100",
+		},
+	}
+
+	for _, test := range testCases {
+		if result := test.rc.CodeSearchRequestWith(test.codeQuery).URL(); result != test.expectedCodeQuery {
+			t.Errorf("Got code query: %s, expected %s", result, test.expectedCodeQuery)
+		}
+
+		if result := test.rc.ContentsRequest(test.fullRepoName, test.path); result != test.expectedContentsQuery {
+			t.Errorf("Got contents query: %s, expected %s", result, test.expectedContentsQuery)
+		}
+		if result := test.rc.CommitsRequest(test.fullRepoName, test.path); result != test.expectedCommitsQuery {
+			t.Errorf("Got commits query: %s, expected %s", result, test.expectedCommitsQuery)
+		}
+	}
+}

api/internal/crawl/crawler/github/split_search_ranges.go

@@ -0,0 +1,379 @@
+package github
+
+// GitHub only returns at most 1000 results per search query,
+// this is problematic if you want to retrieve all the results for a given
+// search query. However, GitHub allows you to specify as much as you want per
+// query to make things more specific. Specifically for files, GitHub allows
+// you to specify their sizes with range queries. This is very convenient
+// since it allows us to split the search into disjoint sets/shards of results
+// from the different file size ranges.
+//
+// Some important factors to consider:
+//
+// -  These queries are rate limited by the API to roughly once query every two
+//    seconds.
+//
+// -  The search space for file sizes is in bytes, from 0B to < 512KiB (this is
+//    a huge search space that cannot be probed linearly in a timely manner if
+//    granularity is to be expected).
+//
+// -  If you have K files there will likely be ~K/1000 sets that you have find
+//    from this search space in order to get all of the results.
+//
+// -  If you have O(K) sets it is unlikely that they are all of the same size,
+//    since (most files are power law distributed). That means that the range
+//    might be significantly smaller for 1000 small files, than it is for
+//    1000 large files.
+//
+// -  This method is a best effort approach. There are some limitations to what
+//    it can and can't do, so please note the following:
+//
+//    +  There may very well be a filesize that has more than 1000 results.
+//       this method cannot help in this case. However, requerying over time
+//       (days/weeks/months) while sorting by last indexed values may be
+//       sufficient to eventually get all of the results.
+//
+//    +  It's possible that the github API returns inconsistent counts. This
+//       is problematic in most cases, since it can cause many issues if the
+//       case is not handled properly. For instance, if you requested the
+//       number of files of an interval from size:0..64 and get that there
+//       are 900 results, you may query at size:0..96 and get that there
+//       are 800 results. To guarantee that this approach completes and does
+//       not get into a query loop over the same intervals, it will retry a few
+//       times and take the largest of the results or the largest previously
+//       queried value from another range (in this case, the implementation
+//       could decide that size:0..96 must have 900) results. This makes the
+//       approach best effort even if there are no single file sizes of over
+//       1000 results.
+//
+//
+// The approach that was taken to solve this problem is the following:
+//
+// 1. Determine the total number of results by querying from the lower bound
+//    to the upper bound (size:0..max). If there are less than 1000 files,
+//    return a single range of values (size:0..max) since all results can be
+//    retrieved.
+//
+// 2. Otherwise, set a target number of files to be 1000.
+//
+// 3. Binary search for the range from 0..r that provides a file count that is
+//    less than or equal to the target. Once this value is found, store the
+//    upper bound of range (r). If r is the same as the previous value, (or 0)
+//    increase r by one (this guarantees progress, but will miss out on some
+//    results).
+//
+// 4. Increase the target by 1000.
+//
+// 5. Repeat steps 3 and 4 until the target is at or exceeds the total number
+//    of files.
+//
+//
+// In general there are other ways to get all of the files from GitHub. In
+// some cases it would be sufficient to just get the files that are being
+// updated/indexed by github periodically to update the corpus, so this
+// complicated approach does not have to be run every time. However, for
+// some searches, there may be too many results on a time interval to do
+// this simple update search limited to only 1000 results.
+//
+// There is also a more sophisticated approach that may yield better
+// performance:
+// -  Perform this search once and create a prior distribution of file sizes.
+//    Each time you want to retrieve the results of the query, scale the
+//    prior of expected ranges to the current number of files. From each
+//    expected range of 1000 files, perform a exponential search to find the
+//    lower bound of the range. This would likely reduce the total number
+//    of queries by a significant amount since it would only have to search
+//    for a small set of values around each likely range boundary.
+//
+// However, actually retrieving the files will be the bottleneck operation
+// since the number of queries to find the ranges will be close to:
+//   log2(maxFileSize) * totalResults / 1000 ~= totalResults / 50
+// whereas the number of queries to actually get all of the search results
+// are close to:
+//   apiCallsPerResult * 10(pages) * 100(resultsPerPage) * totalResults / 1000
+//   = apiCallsPerResult * totalResults.
+//
+// So it could very well take apiCallsPerResult * 50 times longer to actually
+// fetch the results (assuming the quotas for the API calls are the same as the
+// search API), than it does to perform these range searches.
+
+import (
+	"fmt"
+	"math/bits"
+	"strconv"
+	"strings"
+)
+
+// Files cannot be more than 2^19 bytes, according to
+// https://help.github.com/en/articles/searching-code#considerations-for-code-search
+const (
+	githubMaxFileSize        = uint64(1 << 19)
+	githubMaxResultsPerQuery = uint64(1000)
+)
+
+// Interface instead of struct for testing purposes.
+// Not expecting to have multiple implementations.
+type cachedSearch interface {
+	CountResults(uint64, uint64) (uint64, error)
+	RequestString(filesize rangeFormatter) string
+}
+
+// cachedSearch is a simple data structure that maps the upper bound (r) of a
+// range from 0 to r to the number of files that have between 0 and r files
+// (inclusive). It also guarantees that the counts are monotonically increasing
+// (not strict) as the value for r increases, by looking at the maximal
+// previous file count for the value that precedes r in the cache.
+//
+// It uses a bit trick to be more efficient in detecting
+// inconsistencies in the returned data from the Github API.
+// Therefore, the cache expects a search to always start at 0, and
+// it expects the max file size to be a power of 2. If this is to be changed
+// there are a few considerations to keep in mind:
+//
+// 1. The cache is only efficient if the queries can be reused, so if
+//    the first chunk of files lives in the range 0..x, continuing the
+//    search for the next chunk from x+1..max (while asymptotically sane)
+//    may actually be less efficient since the cache is essentially reset
+//    at every interval. This leads to a larger number of requests in
+//    practice, and requests are what's expensive (rate limits).
+//
+// 2. The github API is not perfectly monotonic.. (this is somewhat
+//    problematic). The current cache implementation looks at the
+//    predecessor entry to find out if the current value is monotonic.
+//    This is where the bit trick is used, since each step in the binary
+//    search is adding or omitting to add a decreasing power of 2 to the query
+//    value, we can remove the least significant set bit to find the
+//    predecessor in constant time. Ultimately since the search is rate
+//    limited, we could also easily afford to compute this in linear time
+//    by iterating over cached values. So this trick is not crucial to the
+//    cache's performance.
+type githubCachedSearch struct {
+	cache       map[uint64]uint64
+	gcl         GhClient
+	baseRequest request
+}
+
+func newCache(client GhClient, query Query) githubCachedSearch {
+	return githubCachedSearch{
+		cache: map[uint64]uint64{
+			0: 0,
+		},
+		gcl:         client,
+		baseRequest: client.CodeSearchRequestWith(query),
+	}
+}
+
+func (c githubCachedSearch) CountResults(lowerBound, upperBound uint64) (uint64, error) {
+	count, cached := c.cache[upperBound]
+	if cached {
+		return count, nil
+	}
+
+	sizeRange := RangeWithin{lowerBound, upperBound}
+	rangeRequest := c.RequestString(sizeRange)
+
+	result := c.gcl.parseGithubResponseWithRetry(rangeRequest)
+	if result.Error != nil {
+		return count, result.Error
+	}
+
+	// As range search uses powers of 2 for binary search, the previously
+	// cached value is easy to find by removing the least significant set
+	// bit from the current upperBound, since each step of the search adds
+	// least significant set bit.
+	//
+	// Finding the predecessor could also be implemented by iterating over
+	// the map to find the largest key that is smaller than upperBound if
+	// this approach deemed too complex.
+	trail := bits.TrailingZeros64(upperBound)
+	prev := uint64(0)
+	if trail != 64 {
+		prev = upperBound - (1 << uint64(trail))
+	}
+
+	// Sometimes the github API is not monotonically increasing, or ouputs
+	// an erroneous value of 0, or 1. This logic makes sure that it was not
+	// erroneous, and that the sequence continues to be monotonic by setting
+	// the current query count to match the previous value. which at least
+	// guarantees that the range search terminates.
+	//
+	// On the other hand, if files are added, then we way loose out on some
+	// files in a reviously completed range, but these files should be there
+	// the next time the crawler runs, so this is not really problematic.
+	retryMonotonicCount := 4
+	for result.Parsed.TotalCount < c.cache[prev] {
+		logger.Printf(
+			"Retrying query... current lower bound: %d, got: %d\n",
+			c.cache[prev], result.Parsed.TotalCount)
+
+		result = c.gcl.parseGithubResponseWithRetry(rangeRequest)
+		if result.Error != nil {
+			return count, result.Error
+		}
+
+		retryMonotonicCount--
+		if retryMonotonicCount <= 0 {
+			result.Parsed.TotalCount = c.cache[prev]
+			logger.Println(
+				"Retries for monotonic check exceeded,",
+				" setting value to match predecessor")
+		}
+	}
+
+	count = result.Parsed.TotalCount
+	logger.Printf("Caching new query %s, with count %d (incomplete_results: %v)\n",
+		sizeRange.RangeString(), count, result.Parsed.IncompleteResults)
+	c.cache[upperBound] = count
+	return count, nil
+}
+
+func (c githubCachedSearch) RequestString(filesize rangeFormatter) string {
+	return c.baseRequest.CopyWith(Filesize(filesize)).URL()
+}
+
+// Outputs a (possibly incomplete) list of ranges to query to find most search
+// results as permissible by the search github search API. Github search only
+// allows 1,000 results per query (paginated).
+// Source: https://developer.github.com/v3/search/
+//
+// This leaves the possibility of having file sizes with more than 1000 results,
+// This would mean that the search as it is could not find all files. If queries
+// are sorted by last indexed, and retrieved on regular intervals, it should be
+// sufficient to get most if not all documents.
+func FindRangesForRepoSearch(cache cachedSearch, lowerBound, upperBound uint64) ([]string, error) {
+	totalFiles, err := cache.CountResults(lowerBound, upperBound)
+	if err != nil {
+		return nil, err
+	}
+	logger.Println("total kustomization files: ", totalFiles)
+
+	if githubMaxResultsPerQuery >= totalFiles {
+		return []string{
+			cache.RequestString(RangeWithin{lowerBound, upperBound}),
+		}, nil
+	}
+
+	// Find all the ranges of file sizes such that all files are queryable
+	// using the Github API. This does not compute an optimal ranges, since
+	// the number of queries needed to get the information required to
+	// compute an optimal range is expected to be much larger than the
+	// number of queries performed this way.
+	//
+	// The number of ranges is k = (number of files)/1000, and finding a
+	// range is logarithmic in the max file size (n = filesize). This means
+	// that preprocessing takes O(k * lg n) queries to find the ranges with
+	// a binary search over file sizes.
+	//
+	// My intuition is that this approach is competitive to a perfectly
+	// optimal solution, but I didn't actually take the time to do a
+	// rigorous proof. Intuitively, since files sizes are typically power
+	// law distibuted the binary search will be very skewed towards the
+	// smaller file ranges. This means that in practice this approach will
+	// make fewer than (#files/1000)*(log(n) = 19) queries for
+	// preprocessing, since it reuses a lot of the queries in the denser
+	// ranges. Furthermore, because of the distribution, it should be very
+	// easy to find ranges that are very close to the upper bound, up to
+	// the limiting factor of having no more than 1000 files accessible per
+	// range.
+	filesAccessible := uint64(0)
+	sizes := make([]uint64, 0)
+	sizes = append(sizes, lowerBound)
+	for filesAccessible < totalFiles {
+		target := filesAccessible + githubMaxResultsPerQuery
+		if target >= totalFiles {
+			break
+		}
+
+		logger.Printf("%d accessible files, next target = %d\n",
+			filesAccessible, target)
+
+		size, err := FindFileSize(cache, target, lowerBound, upperBound)
+		if err != nil {
+			return nil, err
+		}
+
+		// If there are more than 1000 files in the next bucket, we must
+		// advance anyway and lose out on some files :(.
+		if l := len(sizes); l > 0 && sizes[l-1] == size {
+			size++
+		}
+
+		nextAccessible, err := cache.CountResults(lowerBound, size)
+		if err != nil {
+			return nil, fmt.Errorf(
+				"cache should be populated at %d already, got %v",
+				size, err)
+		}
+		if nextAccessible < filesAccessible {
+			return nil, fmt.Errorf(
+				"number of results dropped from %d to %d within range search",
+				filesAccessible, nextAccessible)
+		}
+
+		filesAccessible = nextAccessible
+		if nextAccessible < totalFiles {
+			sizes = append(sizes, size)
+		}
+	}
+	sizes = append(sizes, upperBound)
+	return formatFilesizeRanges(cache, sizes), nil
+}
+
+// FindFileSize finds the filesize range from [lowerBound, return value] that has
+// the largest file count that is smaller than or equal to
+// githubMaxResultsPerQuery. It is important to note that this returned value
+// could already be in a previous range if the next file size has more than 1000
+// results. It is left to the caller to handle this bit of logic and guarantee
+// forward progession in this case.
+func FindFileSize(
+	cache cachedSearch, targetFileCount, lowerBound, upperBound uint64) (uint64, error) {
+
+	// Binary search for file sizes that make up the next <=1000 element
+	// chunk.
+	cur := lowerBound
+	increase := (upperBound - lowerBound) / 2
+
+	for increase > 0 {
+		mid := cur + increase
+
+		count, err := cache.CountResults(lowerBound, mid)
+		if err != nil {
+			return count, err
+		}
+
+		if count <= targetFileCount {
+			cur = mid
+		}
+
+		if count == targetFileCount {
+			break
+		}
+
+		increase /= 2
+	}
+
+	return cur, nil
+}
+
+func formatFilesizeRanges(cache cachedSearch, sizes []uint64) []string {
+	n := len(sizes)
+	if n < 2 {
+		return []string{}
+	}
+
+	ranges := make([]string, 0, n-1)
+	ranges = append(ranges, cache.RequestString(RangeWithin{sizes[0], sizes[1]}))
+	for i := 1; i < n-1; i++ {
+		ranges = append(ranges, cache.RequestString(RangeWithin{sizes[i] + 1, sizes[i+1]}))
+	}
+	return ranges
+}
+
+func RangeSizes(s string) RangeWithin {
+	start := strings.Index(s, "+size:") + len("+size:")
+	end := strings.Index(s, "&")
+	ranges := strings.Split(s[start:end], "..")
+	lowerBound, _ := strconv.ParseUint(ranges[0], 10, 64)
+	upperBound, _ := strconv.ParseUint(ranges[1], 10, 64)
+	return RangeWithin{lowerBound, upperBound}
+}

api/internal/crawl/crawler/github/split_search_ranges_test.go

@@ -0,0 +1,101 @@
+package github
+
+import (
+	"fmt"
+	"log"
+	"reflect"
+	"testing"
+)
+
+type testCachedSearch struct {
+	cache map[uint64]uint64
+}
+
+func (c testCachedSearch) CountResults(lowerBound, upperBound uint64) (uint64, error) {
+	log.Printf("CountResults(%05x)\n", upperBound)
+	count, ok := c.cache[upperBound]
+	if !ok {
+		return count, fmt.Errorf("cache not set at %x", upperBound)
+	}
+	return count, nil
+}
+
+func (c testCachedSearch) RequestString(filesize rangeFormatter) string {
+	return filesize.RangeString()
+}
+
+// TODO(damienr74) make tests easier to write.. I'm thinking I can make the test
+// cache take in a list of (filesize, count) pairs and it can populate the cache
+// without relying on how the implementation will create queries. This was only
+// a quick and dirty test to make sure that modifications are not going to break
+// the functionality.
+func TestRangeSplitting(t *testing.T) {
+	// Keys follow the binary search depending on whether or not the range
+	// is too small/large to find close to optimal filesize ranges. This
+	// test is heavily tied to the fact that the search is using powers of two
+	// to make progress in the search (hence the use of hexadecimal values).
+	cache := testCachedSearch{
+		map[uint64]uint64{
+			0x80000: 5000,
+			0x40000: 5000,
+			0x20000: 5000,
+			0x10000: 5000,
+			0x08000: 5000,
+			0x04000: 5000,
+			0x02000: 5000,
+			0x01000: 5000,
+			0x00fff: 3950,
+			0x00ffe: 3950,
+			0x00ffc: 3950,
+			0x00ff8: 3950,
+			0x00ff0: 3950,
+			0x00fe0: 3950,
+			0x00fc0: 3950,
+			0x00f80: 3950,
+			0x00f00: 3950,
+			0x00e00: 3950,
+			0x00c00: 3950,
+			0x00800: 3950,
+			0x00400: 3950,
+			0x00200: 3688,
+			0x00180: 3028,
+			0x00100: 2999,
+			0x000c0: 2448,
+			0x00080: 1999,
+			0x00070: 1600,
+			0x0006c: 1003,
+			0x0006b: 1001,
+			0x0006a: 999,
+			0x00068: 999,
+			0x00060: 999,
+			0x00040: 999,
+			0x00000: 0,
+		},
+	}
+
+	requests, err := FindRangesForRepoSearch(cache, 0, 524288)
+	if err != nil {
+		t.Errorf("Error while finding ranges: %v", err)
+	}
+	expected := []string{
+		"0..106",       // cache.RequestString(RangeWithin{0x00, 0x6a}),
+		"107..128",     // cache.RequestString(RangeWithin{0x6b, 0x80}),
+		"129..256",     // cache.RequestString(RangeWithin{0x81, 0x100}),
+		"257..4095",    // cache.RequestString(RangeWithin{0x101, 0xfff}),
+		"4096..524288", // cache.RequestString(RangeWithin{0x1000, 0x80000}),
+	}
+
+	if !reflect.DeepEqual(requests, expected) {
+		t.Errorf("Expected requests (%v) to equal (%v)", requests, expected)
+	}
+}
+
+func TestRangeSizes(t *testing.T) {
+	s := "https://api.github.com/search/code?q=filename:kustomization.yaml+filename:kustomization.yml" +
+		"+filename:kustomization+size:2365..10000&order=desc&per_page=100&sort=indexed"
+	returnedResult := RangeSizes(s)
+	expectedResult := RangeWithin{uint64(2365), uint64(10000)}
+	if !reflect.DeepEqual(returnedResult, expectedResult) {
+		t.Errorf("RangeSizes expected (%v), got (%v)",expectedResult, returnedResult)
+	}
+}

api/internal/crawl/doc/doc.go

@@ -0,0 +1,267 @@
+package doc
+
+import (
+	"fmt"
+	"log"
+	"path/filepath"
+	"sort"
+	"strings"
+
+	"sigs.k8s.io/kustomize/api/k8sdeps/kunstruct"
+	"sigs.k8s.io/kustomize/api/konfig"
+	"sigs.k8s.io/kustomize/api/types"
+	"sigs.k8s.io/yaml"
+)
+
+var fileReader = kunstruct.NewKunstructuredFactoryImpl()
+
+// This document is meant to be used at the elasticsearch document type.
+// Fields are serialized as-is to elasticsearch, where indices are built
+// to facilitate text search queries. Identifiers, Values, FilePath,
+// RepositoryURL and DocumentData are meant to be searched for text queries
+// directly, while the other fields can either be used as a filter, or as
+// additional metadata displayed in the UI.
+//
+// The fields of the document and their purpose are listed below:
+// - DocumentData contains the contents of the kustomization file.
+// - Kinds Represents the kubernetes Kinds that are in this file.
+// - Identifiers are a list of (partial and full) identifier paths that can be
+//   found by users. Each part of a path is delimited by ":" e.g. spec:replicas.
+// - Values are a list of identifier paths and their values that can be found by
+//   search queries. The path is delimited by ":" and the value follows the "="
+//   symbol e.g. spec:replicas=4.
+// - FilePath is the path of the file.
+// - RepositoryURL is the URL of the source repository.
+// - CreationTime is the time at which the file was created.
+//
+// Representing each Identifier and Value as a flat string representation
+// facilitates the use of complex text search features from elasticsearch such
+// as fuzzy searching, regex, wildcards, etc.
+type KustomizationDocument struct {
+	Document
+	Kinds       []string `json:"kinds,omitempty"`
+	Identifiers []string `json:"identifiers,omitempty"`
+	Values      []string `json:"values,omitempty"`
+}
+
+type set map[string]struct{}
+
+func (doc *KustomizationDocument) Copy() *KustomizationDocument {
+	return &KustomizationDocument{
+		Document:    *(doc.Document.Copy()),
+		Kinds:       doc.Kinds,
+		Identifiers: doc.Identifiers,
+		Values:      doc.Values,
+	}
+}
+
+func (doc *KustomizationDocument) String() string {
+	return fmt.Sprintf("%s %s %s %v %v %v len(identifiers):%v  len(values):%v",
+		doc.RepositoryURL, doc.FilePath, doc.DefaultBranch, doc.CreationTime,
+		doc.IsSame, doc.Kinds, len(doc.Identifiers), len(doc.Values))
+}
+
+// IsKustomizationFile determines whether a file path is a kustomization file
+func IsKustomizationFile(path string) bool {
+	basename := filepath.Base(path)
+	for _, name := range konfig.RecognizedKustomizationFileNames() {
+		if basename == name {
+			return true
+		}
+	}
+	return false
+}
+
+// Implements the CrawlerDocument interface.
+func (doc *KustomizationDocument) GetResources(
+	includeResources, includeTransformers, includeGenerators bool) ([]*Document, error) {
+	if !IsKustomizationFile(doc.FilePath) {
+		return []*Document{}, nil
+	}
+
+	content := []byte(doc.DocumentData)
+	content, err := FixKustomizationPreUnmarshallingNonFatal(content)
+	if err != nil {
+		return nil, fmt.Errorf("could not fix kustomize file: %v", err)
+	}
+
+	var k types.Kustomization
+	err = yaml.Unmarshal(content, &k)
+	if err != nil {
+		return nil, fmt.Errorf(
+			"could not parse kustomization: %v", err)
+	}
+	k.FixKustomizationPostUnmarshalling()
+
+	res := make([]*Document, 0)
+
+	if includeResources {
+		resourceDocs := doc.CollectDocuments(k.Resources, "resource")
+		res = append(res, resourceDocs...)
+	}
+
+	if includeGenerators {
+		generatorDocs := doc.CollectDocuments(k.Generators, "generator")
+		res = append(res, generatorDocs...)
+	}
+
+	if includeTransformers {
+		transformerDocs := doc.CollectDocuments(k.Transformers, "transformer")
+		res = append(res, transformerDocs...)
+	}
+
+	return res, nil
+}
+
+// CollectDocuments construct a Document for each path in paths, and return
+// a slice of Document pointers.
+func (doc *KustomizationDocument) CollectDocuments(
+	paths []string, fileType string) []*Document {
+	docs := make([]*Document, 0, len(paths))
+	for _, r := range paths {
+		if strings.TrimSpace(r) == "" {
+			continue
+		}
+		next, err := doc.Document.FromRelativePath(r)
+		if err != nil {
+			log.Printf("CollectDocuments error: %v\n", err)
+			continue
+		}
+		next.FileType = fileType
+		docs = append(docs, &next)
+	}
+	return docs
+}
+
+func (doc *KustomizationDocument) readBytes() ([]map[string]interface{}, error) {
+	data := []byte(doc.DocumentData)
+
+	for _, suffix := range konfig.RecognizedKustomizationFileNames() {
+		if !strings.HasSuffix(doc.FilePath, "/"+suffix) {
+			continue
+		}
+		var config map[string]interface{}
+		err := yaml.Unmarshal(data, &config)
+		if err != nil {
+			return nil, fmt.Errorf(
+				"unable to parse kustomization: %v", err)
+		}
+		return []map[string]interface{}{config}, nil
+	}
+
+	configs := make([]map[string]interface{}, 0)
+	ks, err := fileReader.SliceFromBytes(data)
+	if err != nil {
+		return nil, fmt.Errorf("unable to parse resource: %v", err)
+	}
+	for _, k := range ks {
+		configs = append(configs, k.Map())
+	}
+
+	return configs, nil
+}
+
+// ParseYAML parses doc.Document and sets the following fields of doc:
+// Kinds, Values, Identifiers.
+func (doc *KustomizationDocument) ParseYAML() error {
+	doc.Identifiers = make([]string, 0)
+	doc.Values = make([]string, 0)
+	doc.Kinds = make([]string, 0, 1)
+
+	identifierSet := make(set)
+	valueSet := make(set)
+	kindSet := make(set)
+
+	getKind := func(m map[string]interface{}) string {
+		const defaultStr = "Kustomization"
+		kind, ok := m["kind"]
+		if !ok {
+			return defaultStr
+		}
+		if str, ok := kind.(string); ok && str != "" {
+			return str
+		}
+		return defaultStr
+	}
+
+	ks, err := doc.readBytes()
+	if err != nil {
+		return err
+	}
+
+	for _, contents := range ks {
+		kindSet[getKind(contents)] = struct{}{}
+		createFlatStructure(identifierSet, valueSet, contents)
+	}
+
+	for val := range kindSet {
+		doc.Kinds = append(doc.Kinds, val)
+	}
+
+	for val := range valueSet {
+		doc.Values = append(doc.Values, val)
+	}
+
+	for key := range identifierSet {
+		doc.Identifiers = append(doc.Identifiers, key)
+	}
+
+	// Without sorting these fields, every time when the string order in these fields changes,
+	// the document in the index will be updated.
+	// Sorting these fields are necessary to avoid a document being updated unnecessarily.
+	sort.Strings(doc.Kinds)
+	sort.Strings(doc.Values)
+	sort.Strings(doc.Identifiers)
+
+	return nil
+}
+
+func createFlatStructure(identifierSet set, valueSet set, contents map[string]interface{}) {
+	type Map struct {
+		data   map[string]interface{}
+		prefix string
+	}
+
+	toVisit := []Map{
+		{
+			data:   contents,
+			prefix: "",
+		},
+	}
+
+	for i := 0; i < len(toVisit); i++ {
+		visiting := toVisit[i]
+		for k, v := range visiting.data {
+			identifier := fmt.Sprintf("%s:%s", visiting.prefix, k)
+			// noop after the first iteration.
+			identifier = strings.TrimLeft(identifier, ":")
+
+			// Recursive function traverses structure to find
+			// identifiers and values. These later get formatted
+			// into doc.Identifiers and doc.Values respectively.
+			var traverseStructure func(interface{})
+			traverseStructure = func(arg interface{}) {
+				switch value := arg.(type) {
+				case map[string]interface{}:
+					toVisit = append(toVisit, Map{
+						data:   value,
+						prefix: identifier,
+					})
+				case []interface{}:
+					for _, val := range value {
+						traverseStructure(val)
+					}
+				case interface{}:
+					esc := fmt.Sprintf("%v", value)
+
+					valuePath := fmt.Sprintf("%s=%v",
+						identifier, esc)
+					valueSet[valuePath] = struct{}{}
+				}
+			}
+			traverseStructure(v)
+
+			identifierSet[identifier] = struct{}{}
+		}
+	}
+}

api/internal/crawl/doc/doc_test.go

@@ -0,0 +1,379 @@
+package doc
+
+import (
+	"reflect"
+	"sort"
+	"strings"
+	"testing"
+)
+
+func TestParseYAML(t *testing.T) {
+	testCases := []struct {
+		identifiers []string
+		values      []string
+		kinds       []string
+		filepath    string
+		yaml        string
+	}{
+		{
+			identifiers: []string{
+				"namePrefix",
+				"metadata",
+				"metadata:name",
+				"kind",
+			},
+			values: []string{
+				"kind=",
+				"namePrefix=dev-",
+				"metadata:name=app",
+			},
+			kinds: []string{
+				"Kustomization",
+			},
+			filepath: "some/path/to/kustomization.yaml",
+			yaml: `
+namePrefix: dev-
+metadata:
+  name: app
+kind: ""
+`,
+		},
+		{
+			identifiers: []string{
+				"namePrefix",
+				"metadata",
+				"metadata:name",
+				"metadata:spec",
+				"metadata:spec:replicas",
+				"kind",
+				"replicas",
+				"replicas:name",
+				"replicas:count",
+				"resource",
+			},
+			values: []string{
+				"namePrefix=dev-",
+				"metadata:name=n1",
+				"metadata:spec:replicas=3",
+				"kind=Kustomization",
+				"replicas:name=n1",
+				"replicas:name=n2",
+				"replicas:count=3",
+				"resource=file1.yaml",
+				"resource=file2.yaml",
+			},
+			kinds: []string{
+				"Kustomization",
+			},
+			filepath: "./kustomization.yaml",
+			yaml: `
+namePrefix: dev-
+# map of map
+metadata:
+  name: n1
+  spec:
+    replicas: 3
+kind: Kustomization
+
+#list of map
+replicas:
+- name: n1
+  count: 3
+- name: n2
+  count: 3
+
+# list
+resource:
+- file1.yaml
+- file2.yaml
+`,
+		},
+		{
+			identifiers: []string{
+				"kind",
+				"metadata",
+				"metadata:name",
+			},
+			values: []string{
+				"kind=Deployment",
+				"kind=Service",
+				"kind=Custom",
+				"metadata:name=app",
+				"metadata:name=app-service",
+				"metadata:name=app-crd",
+			},
+			kinds: []string{
+				"Deployment",
+				"Service",
+				"Custom",
+			},
+			filepath: "resources.yaml",
+			yaml: `
+---
+kind: Deployment
+metadata:
+  name: app
+---
+kind: Service
+metadata:
+  name: app-service
+---
+kind: Custom
+metadata:
+  name: app-crd
+`,
+		},
+		{
+			identifiers: []string{
+				"kind",
+				"metadata",
+				"metadata:name",
+			},
+			values: []string{
+				"kind=Deployment",
+				"kind=Service",
+				"metadata:name=app1",
+				"metadata:name=app2",
+			},
+			kinds: []string{
+				"Deployment",
+				"Service",
+			},
+			filepath: "resources.yaml",
+			yaml: `
+---
+kind: Deployment
+metadata:
+  name: app1
+---
+kind: Deployment
+metadata:
+  name: app2
+---
+kind: Service
+metadata:
+  name: app1
+`,
+		},
+	}
+
+	for _, test := range testCases {
+		doc := KustomizationDocument{
+			Document: Document{
+				DocumentData: test.yaml,
+				FilePath:     test.filepath,
+			},
+		}
+
+		err := doc.ParseYAML()
+		if err != nil {
+			t.Errorf("Document error error: %s", err)
+		}
+
+		cmpStrings := func(got, expected []string, label string) {
+			sort.Strings(got)
+			sort.Strings(expected)
+
+			if !reflect.DeepEqual(got, expected) {
+				t.Errorf("Expected %s (%v) to be equal to (%v)\n",
+					label,
+					strings.Join(got, ","),
+					strings.Join(expected, ","))
+			}
+
+		}
+
+		cmpStrings(doc.Identifiers, test.identifiers, "identifiers")
+		cmpStrings(doc.Values, test.values, "values")
+		cmpStrings(doc.Kinds, test.kinds, "kinds")
+	}
+}
+
+type TestStructForGetResources struct {
+	doc       KustomizationDocument
+	resources []*Document
+}
+
+func TestGetResources(t *testing.T) {
+	tests := []TestStructForGetResources{
+		{
+			doc: KustomizationDocument{
+				Document: Document{
+					RepositoryURL: "sigs.k8s.io/kustomize",
+					FilePath:      "some/path/to/kdir/kustomization.yaml",
+					DocumentData: `
+bases:
+- ../base
+- ../otherbase
+
+resources:
+- file.yaml
+- https://github.com/kubernetes-sigs/kustomize/examples/helloWorld?ref=v3.1.0
+`},
+			},
+			resources: []*Document{
+				{
+					RepositoryURL: "sigs.k8s.io/kustomize",
+					FilePath:      "some/path/to/base",
+					FileType:      "resource",
+					User:          "sigs.k8s.io",
+				},
+				{
+					RepositoryURL: "sigs.k8s.io/kustomize",
+					FilePath:      "some/path/to/otherbase",
+					FileType:      "resource",
+					User:          "sigs.k8s.io",
+				},
+				{
+					RepositoryURL: "sigs.k8s.io/kustomize",
+					FilePath:      "some/path/to/kdir/file.yaml",
+					FileType:      "resource",
+					User:          "sigs.k8s.io",
+				},
+				{
+					RepositoryURL: "https://github.com/kubernetes-sigs/kustomize",
+					FilePath:      "examples/helloWorld",
+					DefaultBranch: "v3.1.0",
+					FileType:      "resource",
+					User:          "kubernetes-sigs",
+				},
+			},
+		},
+		{
+			doc: KustomizationDocument{
+				Document: Document{
+					RepositoryURL: "https://github.com/some/repo",
+					FilePath:      "some/resource.yaml",
+					DocumentData: `
+bases:
+- ../base
+- ../overlay
+
+resources:
+- https://github.com/kubernetes-sigs/kustomize/examples/helloWorld?ref=v3.1.0
+- some/file.yaml
+`,
+				},
+			},
+			resources: []*Document{},
+		},
+	}
+	runTest(t, tests, true, false, false)
+}
+
+func runTest(t *testing.T, tests []TestStructForGetResources, includeResources, includeTransformers, includeGenerators bool) {
+	for _, test := range tests {
+		res, err := test.doc.GetResources(includeResources, includeTransformers, includeGenerators)
+		if err != nil {
+			t.Errorf("Unexpected error: %v\n", err)
+			continue
+		}
+		if len(test.resources) != len(res) {
+			t.Errorf("Number of resources does not match.")
+			continue
+		}
+		cmp := func(docs []*Document) func(i, j int) bool {
+			return func(i, j int) bool {
+				if docs[i].RepositoryURL != docs[j].RepositoryURL {
+					return docs[i].RepositoryURL <
+						docs[j].RepositoryURL
+				}
+
+				if docs[i].FilePath != docs[j].FilePath {
+					return docs[i].FilePath <
+						docs[j].FilePath
+				}
+
+				return docs[i].DefaultBranch < docs[j].DefaultBranch
+			}
+		}
+		sort.Slice(test.resources, cmp(test.resources))
+		sort.Slice(res, cmp(res))
+		for i, r := range test.resources {
+			if !reflect.DeepEqual(res[i], r) {
+				t.Errorf("Expected '%+v' to equal '%+v'\n",
+					res[i], r)
+			}
+		}
+	}
+}
+
+func TestGetResourcesAndGenerators(t *testing.T) {
+	tests := []TestStructForGetResources{
+		{
+			doc: KustomizationDocument{
+				Document: Document{
+					RepositoryURL: "sigs.k8s.io/kustomize",
+					FilePath:      "some/path/to/kdir/kustomization.yaml",
+					DocumentData: `
+resources:
+- file.yaml
+
+generators:
+- gen.yaml
+
+transformers:
+- tr.yaml
+`},
+			},
+			resources: []*Document{
+				{
+					RepositoryURL: "sigs.k8s.io/kustomize",
+					FilePath:      "some/path/to/kdir/gen.yaml",
+					FileType:      "generator",
+					User:          "sigs.k8s.io",
+				},
+				{
+					RepositoryURL: "sigs.k8s.io/kustomize",
+					FilePath:      "some/path/to/kdir/file.yaml",
+					FileType:      "resource",
+					User:          "sigs.k8s.io",
+				},
+			},
+		},
+	}
+	runTest(t, tests, true, false, true)
+}
+
+func TestGetResourcesAndGeneratorsAndTransformers(t *testing.T) {
+	tests := []TestStructForGetResources{
+		{
+			doc: KustomizationDocument{
+				Document: Document{
+					RepositoryURL: "sigs.k8s.io/kustomize",
+					FilePath:      "some/path/to/kdir/kustomization.yaml",
+					DocumentData: `
+resources:
+- file.yaml
+
+generators:
+- gen.yaml
+
+transformers:
+- tr.yaml
+`},
+			},
+			resources: []*Document{
+				{
+					RepositoryURL: "sigs.k8s.io/kustomize",
+					FilePath:      "some/path/to/kdir/tr.yaml",
+					FileType:      "transformer",
+					User:          "sigs.k8s.io",
+				},
+				{
+					RepositoryURL: "sigs.k8s.io/kustomize",
+					FilePath:      "some/path/to/kdir/gen.yaml",
+					FileType:      "generator",
+					User:          "sigs.k8s.io",
+				},
+				{
+					RepositoryURL: "sigs.k8s.io/kustomize",
+					FilePath:      "some/path/to/kdir/file.yaml",
+					FileType:      "resource",
+					User:          "sigs.k8s.io",
+				},
+			},
+		},
+	}
+	runTest(t, tests, true, true, true)
+}