Drain the top level internal.

api/internal/accumulator/expansion/expand.go

@@ -0,0 +1,121 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+// Package expansion provides functions find and replace $(FOO) style variables in strings.
+package expansion
+
+import (
+	"bytes"
+	"fmt"
+)
+
+const (
+	operator        = '$'
+	referenceOpener = '('
+	referenceCloser = ')'
+)
+
+// syntaxWrap returns the input string wrapped by the expansion syntax.
+func syntaxWrap(input string) string {
+	return string(operator) + string(referenceOpener) + input + string(referenceCloser)
+}
+
+// MappingFuncFor returns a mapping function for use with Expand that
+// implements the expansion semantics defined in the expansion spec; it
+// returns the input string wrapped in the expansion syntax if no mapping
+// for the input is found.
+func MappingFuncFor(
+	counts map[string]int,
+	context ...map[string]interface{}) func(string) interface{} {
+	return func(input string) interface{} {
+		for _, vars := range context {
+			val, ok := vars[input]
+			if ok {
+				counts[input]++
+				switch typedV := val.(type) {
+				case string, int64, float64, bool:
+					return typedV
+				default:
+					return syntaxWrap(input)
+				}
+			}
+		}
+		return syntaxWrap(input)
+	}
+}
+
+// Expand replaces variable references in the input string according to
+// the expansion spec using the given mapping function to resolve the
+// values of variables.
+func Expand(input string, mapping func(string) interface{}) interface{} {
+	var buf bytes.Buffer
+	checkpoint := 0
+	for cursor := 0; cursor < len(input); cursor++ {
+		if input[cursor] == operator && cursor+1 < len(input) {
+			// Copy the portion of the input string since the last
+			// checkpoint into the buffer
+			buf.WriteString(input[checkpoint:cursor])
+
+			// Attempt to read the variable name as defined by the
+			// syntax from the input string
+			read, isVar, advance := tryReadVariableName(input[cursor+1:])
+
+			if isVar {
+				// We were able to read a variable name correctly;
+				// apply the mapping to the variable name and copy the
+				// bytes into the buffer
+				mapped := mapping(read)
+				if input == syntaxWrap(read) {
+					// Preserve the type of variable
+					return mapped
+				}
+
+				// Variable is used in a middle of a string
+				buf.WriteString(fmt.Sprintf("%v", mapped))
+			} else {
+				// Not a variable name; copy the read bytes into the buffer
+				buf.WriteString(read)
+			}
+
+			// Advance the cursor in the input string to account for
+			// bytes consumed to read the variable name expression
+			cursor += advance
+
+			// Advance the checkpoint in the input string
+			checkpoint = cursor + 1
+		}
+	}
+
+	// Return the buffer and any remaining unwritten bytes in the
+	// input string.
+	return buf.String() + input[checkpoint:]
+}
+
+// tryReadVariableName attempts to read a variable name from the input
+// string and returns the content read from the input, whether that content
+// represents a variable name to perform mapping on, and the number of bytes
+// consumed in the input string.
+//
+// The input string is assumed not to contain the initial operator.
+func tryReadVariableName(input string) (string, bool, int) {
+	switch input[0] {
+	case operator:
+		// Escaped operator; return it.
+		return input[0:1], false, 1
+	case referenceOpener:
+		// Scan to expression closer
+		for i := 1; i < len(input); i++ {
+			if input[i] == referenceCloser {
+				return input[1:i], true, i + 1
+			}
+		}
+
+		// Incomplete reference; return it.
+		return string(operator) + string(referenceOpener), false, 1
+	default:
+		// Not the beginning of an expression, ie, an operator
+		// that doesn't begin an expression.  Return the operator
+		// and the first rune in the string.
+		return string(operator) + string(input[0]), false, 1
+	}
+}

api/internal/accumulator/expansion/expand_test.go

@@ -0,0 +1,365 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package expansion_test
+
+import (
+	"fmt"
+	"testing"
+
+	. "sigs.k8s.io/kustomize/v3/api/internal/accumulator/expansion"
+)
+
+type expected struct {
+	count  int
+	edited string
+}
+
+func TestMapReference(t *testing.T) {
+	type env struct {
+		Name  string
+		Value interface{}
+	}
+	envs := []env{
+		{
+			Name:  "FOO",
+			Value: "bar",
+		},
+		{
+			Name:  "ZOO",
+			Value: "$(FOO)-1",
+		},
+		{
+			Name:  "BLU",
+			Value: "$(ZOO)-2",
+		},
+		{
+			Name:  "INT",
+			Value: 2,
+		},
+		{
+			Name:  "ZINT",
+			Value: "$(INT)",
+		},
+		{
+			Name:  "BOOL",
+			Value: true,
+		},
+		{
+			Name:  "ZBOOL",
+			Value: "$(BOOL)",
+		},
+	}
+
+	declaredEnv := map[string]interface{}{
+		"FOO":   "bar",
+		"ZOO":   "$(FOO)-1",
+		"BLU":   "$(ZOO)-2",
+		"INT":   "2",
+		"ZINT":  "$(INT)",
+		"BOOL":  "true",
+		"ZBOOL": "$(BOOL)",
+	}
+
+	counts := make(map[string]int)
+	mapping := MappingFuncFor(counts, declaredEnv)
+
+	for _, env := range envs {
+		declaredEnv[env.Name] = Expand(fmt.Sprintf("%v", env.Value), mapping)
+	}
+
+	expectedEnv := map[string]expected{
+		"FOO":   {count: 1, edited: "bar"},
+		"ZOO":   {count: 1, edited: "bar-1"},
+		"BLU":   {count: 0, edited: "bar-1-2"},
+		"INT":   {count: 1, edited: "2"},
+		"ZINT":  {count: 0, edited: "2"},
+		"BOOL":  {count: 1, edited: "true"},
+		"ZBOOL": {count: 0, edited: "true"},
+	}
+
+	for k, v := range expectedEnv {
+		if e, a := v, declaredEnv[k]; e.edited != a || e.count != counts[k] {
+			t.Errorf("Expected %v count=%d, got %v count=%d",
+				e.edited, e.count, a, counts[k])
+		} else {
+			delete(declaredEnv, k)
+		}
+	}
+
+	if len(declaredEnv) != 0 {
+		t.Errorf("Unexpected keys in declared env: %v", declaredEnv)
+	}
+}
+
+func TestMapping(t *testing.T) {
+	context := map[string]interface{}{
+		"VAR_A":     "A",
+		"VAR_B":     "B",
+		"VAR_C":     "C",
+		"VAR_REF":   "$(VAR_A)",
+		"VAR_EMPTY": "",
+	}
+	doExpansionTest(t, context)
+}
+
+func TestMappingDual(t *testing.T) {
+	context := map[string]interface{}{
+		"VAR_A":     "A",
+		"VAR_EMPTY": "",
+	}
+	context2 := map[string]interface{}{
+		"VAR_B":   "B",
+		"VAR_C":   "C",
+		"VAR_REF": "$(VAR_A)",
+	}
+
+	doExpansionTest(t, context, context2)
+}
+
+func doExpansionTest(t *testing.T, context ...map[string]interface{}) {
+	cases := []struct {
+		name     string
+		input    string
+		expected string
+		counts   map[string]int
+	}{
+		{
+			name:     "whole string",
+			input:    "$(VAR_A)",
+			expected: "A",
+			counts:   map[string]int{"VAR_A": 1},
+		},
+		{
+			name:     "repeat",
+			input:    "$(VAR_A)-$(VAR_A)",
+			expected: "A-A",
+			counts:   map[string]int{"VAR_A": 2},
+		},
+		{
+			name:     "multiple repeats",
+			input:    "$(VAR_A)-$(VAR_B)-$(VAR_B)-$(VAR_B)-$(VAR_A)",
+			expected: "A-B-B-B-A",
+			counts:   map[string]int{"VAR_A": 2, "VAR_B": 3},
+		},
+		{
+			name:     "beginning",
+			input:    "$(VAR_A)-1",
+			expected: "A-1",
+			counts:   map[string]int{"VAR_A": 1},
+		},
+		{
+			name:     "middle",
+			input:    "___$(VAR_B)___",
+			expected: "___B___",
+			counts:   map[string]int{"VAR_B": 1},
+		},
+		{
+			name:     "end",
+			input:    "___$(VAR_C)",
+			expected: "___C",
+			counts:   map[string]int{"VAR_C": 1},
+		},
+		{
+			name:     "compound",
+			input:    "$(VAR_A)_$(VAR_B)_$(VAR_C)",
+			expected: "A_B_C",
+			counts:   map[string]int{"VAR_A": 1, "VAR_B": 1, "VAR_C": 1},
+		},
+		{
+			name:     "escape & expand",
+			input:    "$$(VAR_B)_$(VAR_A)",
+			expected: "$(VAR_B)_A",
+			counts:   map[string]int{"VAR_A": 1},
+		},
+		{
+			name:     "compound escape",
+			input:    "$$(VAR_A)_$$(VAR_B)",
+			expected: "$(VAR_A)_$(VAR_B)",
+		},
+		{
+			name:     "mixed in escapes",
+			input:    "f000-$$VAR_A",
+			expected: "f000-$VAR_A",
+		},
+		{
+			name:     "backslash escape ignored",
+			input:    "foo\\$(VAR_C)bar",
+			expected: "foo\\Cbar",
+			counts:   map[string]int{"VAR_C": 1},
+		},
+		{
+			name:     "backslash escape ignored",
+			input:    "foo\\\\$(VAR_C)bar",
+			expected: "foo\\\\Cbar",
+			counts:   map[string]int{"VAR_C": 1},
+		},
+		{
+			name:     "lots of backslashes",
+			input:    "foo\\\\\\\\$(VAR_A)bar",
+			expected: "foo\\\\\\\\Abar",
+			counts:   map[string]int{"VAR_A": 1},
+		},
+		{
+			name:     "nested var references",
+			input:    "$(VAR_A$(VAR_B))",
+			expected: "$(VAR_A$(VAR_B))",
+		},
+		{
+			name:     "nested var references second type",
+			input:    "$(VAR_A$(VAR_B)",
+			expected: "$(VAR_A$(VAR_B)",
+		},
+		{
+			name:     "value is a reference",
+			input:    "$(VAR_REF)",
+			expected: "$(VAR_A)",
+			counts:   map[string]int{"VAR_REF": 1},
+		},
+		{
+			name:     "value is a reference x 2",
+			input:    "%%$(VAR_REF)--$(VAR_REF)%%",
+			expected: "%%$(VAR_A)--$(VAR_A)%%",
+			counts:   map[string]int{"VAR_REF": 2},
+		},
+		{
+			name:     "empty var",
+			input:    "foo$(VAR_EMPTY)bar",
+			expected: "foobar",
+			counts:   map[string]int{"VAR_EMPTY": 1},
+		},
+		{
+			name:     "unterminated expression",
+			input:    "foo$(VAR_Awhoops!",
+			expected: "foo$(VAR_Awhoops!",
+		},
+		{
+			name:     "expression without operator",
+			input:    "f00__(VAR_A)__",
+			expected: "f00__(VAR_A)__",
+		},
+		{
+			name:     "shell special vars pass through",
+			input:    "$?_boo_$!",
+			expected: "$?_boo_$!",
+		},
+		{
+			name:     "bare operators are ignored",
+			input:    "$VAR_A",
+			expected: "$VAR_A",
+		},
+		{
+			name:     "undefined vars are passed through",
+			input:    "$(VAR_DNE)",
+			expected: "$(VAR_DNE)",
+		},
+		{
+			name:     "multiple (even) operators, var undefined",
+			input:    "$$$$$$(BIG_MONEY)",
+			expected: "$$$(BIG_MONEY)",
+		},
+		{
+			name:     "multiple (even) operators, var defined",
+			input:    "$$$$$$(VAR_A)",
+			expected: "$$$(VAR_A)",
+		},
+		{
+			name:     "multiple (odd) operators, var undefined",
+			input:    "$$$$$$$(GOOD_ODDS)",
+			expected: "$$$$(GOOD_ODDS)",
+		},
+		{
+			name:     "multiple (odd) operators, var defined",
+			input:    "$$$$$$$(VAR_A)",
+			expected: "$$$A",
+			counts:   map[string]int{"VAR_A": 1},
+		},
+		{
+			name:     "missing open expression",
+			input:    "$VAR_A)",
+			expected: "$VAR_A)",
+		},
+		{
+			name:     "shell syntax ignored",
+			input:    "${VAR_A}",
+			expected: "${VAR_A}",
+		},
+		{
+			name:     "trailing incomplete expression not consumed",
+			input:    "$(VAR_B)_______$(A",
+			expected: "B_______$(A",
+			counts:   map[string]int{"VAR_B": 1},
+		},
+		{
+			name:     "trailing incomplete expression, no content, is not consumed",
+			input:    "$(VAR_C)_______$(",
+			expected: "C_______$(",
+			counts:   map[string]int{"VAR_C": 1},
+		},
+		{
+			name:     "operator at end of input string is preserved",
+			input:    "$(VAR_A)foobarzab$",
+			expected: "Afoobarzab$",
+			counts:   map[string]int{"VAR_A": 1},
+		},
+		{
+			name:     "shell escaped incomplete expr",
+			input:    "foo-\\$(VAR_A",
+			expected: "foo-\\$(VAR_A",
+		},
+		{
+			name:     "lots of $( in middle",
+			input:    "--$($($($($--",
+			expected: "--$($($($($--",
+		},
+		{
+			name:     "lots of $( in beginning",
+			input:    "$($($($($--foo$(",
+			expected: "$($($($($--foo$(",
+		},
+		{
+			name:     "lots of $( at end",
+			input:    "foo0--$($($($(",
+			expected: "foo0--$($($($(",
+		},
+		{
+			name:     "escaped operators in variable names are not escaped",
+			input:    "$(foo$$var)",
+			expected: "$(foo$$var)",
+		},
+		{
+			name:     "newline not expanded",
+			input:    "\n",
+			expected: "\n",
+		},
+	}
+
+	for _, tc := range cases {
+		counts := make(map[string]int)
+		mapping := MappingFuncFor(counts, context...)
+		expanded := Expand(fmt.Sprintf("%v", tc.input), mapping)
+		if e, a := tc.expected, expanded; e != a {
+			t.Errorf("%v: expected %q, got %q", tc.name, e, a)
+		}
+		if len(counts) != len(tc.counts) {
+			t.Errorf("%v: len(counts)=%d != len(tc.counts)=%d",
+				tc.name, len(counts), len(tc.counts))
+		}
+		if len(tc.counts) > 0 {
+			for k, expectedCount := range tc.counts {
+				c, ok := counts[k]
+				if ok {
+					if c != expectedCount {
+						t.Errorf(
+							"%v: k=%s, expected count %d, got %d",
+							tc.name, k, expectedCount, c)
+					}
+				} else {
+					t.Errorf(
+						"%v: k=%s, expected count %d, got zero",
+						tc.name, k, expectedCount)
+				}
+			}
+		}
+	}
+}

api/internal/accumulator/loadconfigfromcrds.go

@@ -0,0 +1,190 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package accumulator
+
+import (
+	"encoding/json"
+	"strings"
+
+	"github.com/go-openapi/spec"
+	"github.com/pkg/errors"
+	"k8s.io/kube-openapi/pkg/common"
+	"sigs.k8s.io/kustomize/v3/api/builtinconfig"
+	"sigs.k8s.io/kustomize/v3/api/ifc"
+	"sigs.k8s.io/kustomize/v3/api/resid"
+	"sigs.k8s.io/kustomize/v3/api/types"
+	"sigs.k8s.io/yaml"
+)
+
+type myProperties map[string]spec.Schema
+type nameToApiMap map[string]common.OpenAPIDefinition
+
+// LoadConfigFromCRDs parse CRD schemas from paths into a TransformerConfig
+func LoadConfigFromCRDs(
+	ldr ifc.Loader, paths []string) (*builtinconfig.TransformerConfig, error) {
+	tc := builtinconfig.MakeEmptyConfig()
+	for _, path := range paths {
+		content, err := ldr.Load(path)
+		if err != nil {
+			return nil, err
+		}
+		m, err := makeNameToApiMap(content)
+		if err != nil {
+			return nil, errors.Wrapf(err, "unable to parse open API definition from '%s'", path)
+		}
+		otherTc, err := makeConfigFromApiMap(m)
+		if err != nil {
+			return nil, err
+		}
+		tc, err = tc.Merge(otherTc)
+		if err != nil {
+			return nil, err
+		}
+	}
+	return tc, nil
+}
+
+func makeNameToApiMap(content []byte) (result nameToApiMap, err error) {
+	if content[0] == '{' {
+		err = json.Unmarshal(content, &result)
+	} else {
+		err = yaml.Unmarshal(content, &result)
+	}
+	return
+}
+
+func makeConfigFromApiMap(m nameToApiMap) (*builtinconfig.TransformerConfig, error) {
+	result := builtinconfig.MakeEmptyConfig()
+	for name, api := range m {
+		if !looksLikeAk8sType(api.Schema.SchemaProps.Properties) {
+			continue
+		}
+		tc := builtinconfig.MakeEmptyConfig()
+		err := loadCrdIntoConfig(
+			tc, makeGvkFromTypeName(name), m, name, []string{})
+		if err != nil {
+			return result, err
+		}
+		result, err = result.Merge(tc)
+		if err != nil {
+			return result, err
+		}
+	}
+	return result, nil
+}
+
+// TODO: Get Group and Version for CRD from the
+// openAPI definition once
+// "x-kubernetes-group-version-kind" is available in CRD
+func makeGvkFromTypeName(n string) resid.Gvk {
+	names := strings.Split(n, ".")
+	kind := names[len(names)-1]
+	return resid.Gvk{Kind: kind}
+}
+
+func looksLikeAk8sType(properties myProperties) bool {
+	_, ok := properties["kind"]
+	if !ok {
+		return false
+	}
+	_, ok = properties["apiVersion"]
+	if !ok {
+		return false
+	}
+	_, ok = properties["metadata"]
+	if !ok {
+		return false
+	}
+	return true
+}
+
+const (
+	// "x-kubernetes-annotation": ""
+	xAnnotation = "x-kubernetes-annotation"
+
+	// "x-kubernetes-label-selector": ""
+	xLabelSelector = "x-kubernetes-label-selector"
+
+	// "x-kubernetes-identity": ""
+	xIdentity = "x-kubernetes-identity"
+
+	// "x-kubernetes-object-ref-api-version": <apiVersion name>
+	xVersion = "x-kubernetes-object-ref-api-version"
+
+	// "x-kubernetes-object-ref-kind": <kind name>
+	xKind = "x-kubernetes-object-ref-kind"
+
+	// "x-kubernetes-object-ref-name-key": "name"
+	// default is "name"
+	xNameKey = "x-kubernetes-object-ref-name-key"
+)
+
+// loadCrdIntoConfig loads a CRD spec into a TransformerConfig
+func loadCrdIntoConfig(
+	theConfig *builtinconfig.TransformerConfig, theGvk resid.Gvk, theMap nameToApiMap,
+	typeName string, path []string) (err error) {
+	api, ok := theMap[typeName]
+	if !ok {
+		return nil
+	}
+	for propName, property := range api.Schema.SchemaProps.Properties {
+		_, annotate := property.Extensions.GetString(xAnnotation)
+		if annotate {
+			err = theConfig.AddAnnotationFieldSpec(
+				makeFs(theGvk, append(path, propName)))
+			if err != nil {
+				return
+			}
+		}
+		_, label := property.Extensions.GetString(xLabelSelector)
+		if label {
+			err = theConfig.AddLabelFieldSpec(
+				makeFs(theGvk, append(path, propName)))
+			if err != nil {
+				return
+			}
+		}
+		_, identity := property.Extensions.GetString(xIdentity)
+		if identity {
+			err = theConfig.AddPrefixFieldSpec(
+				makeFs(theGvk, append(path, propName)))
+			if err != nil {
+				return
+			}
+		}
+		version, ok := property.Extensions.GetString(xVersion)
+		if ok {
+			kind, ok := property.Extensions.GetString(xKind)
+			if ok {
+				nameKey, ok := property.Extensions.GetString(xNameKey)
+				if !ok {
+					nameKey = "name"
+				}
+				err = theConfig.AddNamereferenceFieldSpec(
+					builtinconfig.NameBackReferences{
+						Gvk: resid.Gvk{Kind: kind, Version: version},
+						FieldSpecs: []types.FieldSpec{
+							makeFs(theGvk, append(path, propName, nameKey))},
+					})
+				if err != nil {
+					return
+				}
+			}
+		}
+		if property.Ref.GetURL() != nil {
+			loadCrdIntoConfig(
+				theConfig, theGvk, theMap,
+				property.Ref.String(), append(path, propName))
+		}
+	}
+	return nil
+}
+
+func makeFs(in resid.Gvk, path []string) types.FieldSpec {
+	return types.FieldSpec{
+		CreateIfNotPresent: false,
+		Gvk:                in,
+		Path:               strings.Join(path, "/"),
+	}
+}

api/internal/accumulator/loadconfigfromcrds_test.go

@@ -0,0 +1,182 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package accumulator_test
+
+import (
+	"reflect"
+	"testing"
+
+	"sigs.k8s.io/kustomize/v3/api/builtinconfig"
+	"sigs.k8s.io/kustomize/v3/api/ifc"
+	. "sigs.k8s.io/kustomize/v3/api/internal/accumulator"
+	"sigs.k8s.io/kustomize/v3/api/internal/loadertest"
+	"sigs.k8s.io/kustomize/v3/api/resid"
+	"sigs.k8s.io/kustomize/v3/api/types"
+)
+
+// This defines two CRD's:  Bee and MyKind.
+//
+// Bee is boring, it's spec has no dependencies.
+//
+// MyKind, however, has a spec that contains
+// a Bee and a (k8s native) Secret.
+const (
+	crdContent = `
+{
+	"github.com/example/pkg/apis/jingfang/v1beta1.Bee": {
+		"Schema": {
+			"description": "Bee",
+			"properties": {
+				"apiVersion": {
+					"description": "APIVersion defines the versioned schema of this representation of an object. Servers should convert 
+recognized schemas to the latest internal value, and may reject unrecognized values. 
+More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources",
+					"type": "string"
+				},
+				"kind": {
+					"description": "Kind is a string value representing the REST resource this object represents. Servers may infer
+this from the endpoint the client submits requests to. Cannot be updated. In CamelCase.
+More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds",
+					"type": "string"
+				},
+				"metadata": {
+					"$ref": "k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta"
+				},
+				"spec": {
+					"$ref": "github.com/example/pkg/apis/jingfang/v1beta1.BeeSpec"
+				},
+				"status": {
+					"$ref": "github.com/example/pkg/apis/jingfang/v1beta1.BeeStatus"
+				}
+			}
+		},
+		"Dependencies": [
+			"github.com/example/pkg/apis/jingfang/v1beta1.BeeSpec",
+			"github.com/example/pkg/apis/jingfang/v1beta1.BeeStatus",
+			"k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta"
+		]
+	},
+	"github.com/example/pkg/apis/jingfang/v1beta1.BeeSpec": {
+		"Schema": {
+			"description": "BeeSpec defines the desired state of Bee"
+		},
+		"Dependencies": []
+	},
+	"github.com/example/pkg/apis/jingfang/v1beta1.BeeStatus": {
+		"Schema": {
+			"description": "BeeStatus defines the observed state of Bee"
+		},
+		"Dependencies": []
+	},
+	"github.com/example/pkg/apis/jingfang/v1beta1.MyKind": {
+		"Schema": {
+			"description": "MyKind",
+			"properties": {
+				"apiVersion": {
+					"description": "APIVersion defines the versioned schema of this representation of an object.
+Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values.
+More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources",
+					"type": "string"
+				},
+				"kind": {
+					"description": "Kind is a string value representing the REST resource this object represents.
+Servers may infer this from the endpoint the client submits requests to. Cannot be updated.
+In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds",
+					"type": "string"
+				},
+				"metadata": {
+					"$ref": "k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta"
+				},
+				"spec": {
+					"$ref": "github.com/example/pkg/apis/jingfang/v1beta1.MyKindSpec"
+				},
+				"status": {
+					"$ref": "github.com/example/pkg/apis/jingfang/v1beta1.MyKindStatus"
+				}
+			}
+		},
+		"Dependencies": [
+			"github.com/example/pkg/apis/jingfang/v1beta1.MyKindSpec",
+			"github.com/example/pkg/apis/jingfang/v1beta1.MyKindStatus",
+			"k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta"
+		]
+	},
+	"github.com/example/pkg/apis/jingfang/v1beta1.MyKindSpec": {
+		"Schema": {
+			"description": "MyKindSpec defines the desired state of MyKind",
+			"properties": {
+				"beeRef": {
+					"x-kubernetes-object-ref-api-version": "v1beta1",
+					"x-kubernetes-object-ref-kind": "Bee",
+					"$ref": "github.com/example/pkg/apis/jingfang/v1beta1.Bee"
+				},
+				"secretRef": {
+					"description": "If defined, we use this secret for configuring the MYSQL_ROOT_PASSWORD 
+If it is not set we generate a secret dynamically",
+					"x-kubernetes-object-ref-api-version": "v1",
+					"x-kubernetes-object-ref-kind": "Secret",
+					"$ref": "k8s.io/api/core/v1.LocalObjectReference"
+				}
+			}
+		},
+		"Dependencies": [
+			"github.com/example/pkg/apis/jingfang/v1beta1.Bee",
+			"k8s.io/api/core/v1.LocalObjectReference"
+		]
+	},
+	"github.com/example/pkg/apis/jingfang/v1beta1.MyKindStatus": {
+		"Schema": {
+			"description": "MyKindStatus defines the observed state of MyKind"
+		},
+		"Dependencies": []
+	}
+}
+`
+)
+
+func makeLoader(t *testing.T) ifc.Loader {
+	ldr := loadertest.NewFakeLoader("/testpath")
+	err := ldr.AddFile("/testpath/crd.json", []byte(crdContent))
+	if err != nil {
+		t.Fatalf("Failed to setup fake ldr.")
+	}
+	return ldr
+}
+
+func TestLoadCRDs(t *testing.T) {
+	nbrs := []builtinconfig.NameBackReferences{
+		{
+			Gvk: resid.Gvk{Kind: "Secret", Version: "v1"},
+			FieldSpecs: []types.FieldSpec{
+				{
+					CreateIfNotPresent: false,
+					Gvk:                resid.Gvk{Kind: "MyKind"},
+					Path:               "spec/secretRef/name",
+				},
+			},
+		},
+		{
+			Gvk: resid.Gvk{Kind: "Bee", Version: "v1beta1"},
+			FieldSpecs: []types.FieldSpec{
+				{
+					CreateIfNotPresent: false,
+					Gvk:                resid.Gvk{Kind: "MyKind"},
+					Path:               "spec/beeRef/name",
+				},
+			},
+		},
+	}
+
+	expectedTc := &builtinconfig.TransformerConfig{
+		NameReference: nbrs,
+	}
+
+	actualTc, err := LoadConfigFromCRDs(makeLoader(t), []string{"crd.json"})
+	if err != nil {
+		t.Fatalf("unexpected error:%v", err)
+	}
+	if !reflect.DeepEqual(actualTc, expectedTc) {
+		t.Fatalf("expected\n %v\n but got\n %v\n", expectedTc, actualTc)
+	}
+}

api/internal/accumulator/namereferencetransformer.go

@@ -0,0 +1,274 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package accumulator
+
+import (
+	"fmt"
+	"log"
+
+	"sigs.k8s.io/kustomize/v3/api/builtinconfig"
+	"sigs.k8s.io/kustomize/v3/api/resid"
+	"sigs.k8s.io/kustomize/v3/api/resmap"
+	"sigs.k8s.io/kustomize/v3/api/resource"
+	"sigs.k8s.io/kustomize/v3/api/transform"
+)
+
+type nameReferenceTransformer struct {
+	backRefs []builtinconfig.NameBackReferences
+}
+
+var _ resmap.Transformer = &nameReferenceTransformer{}
+
+// newNameReferenceTransformer constructs a nameReferenceTransformer
+// with a given slice of NameBackReferences.
+func newNameReferenceTransformer(br []builtinconfig.NameBackReferences) resmap.Transformer {
+	if br == nil {
+		log.Fatal("backrefs not expected to be nil")
+	}
+	return &nameReferenceTransformer{backRefs: br}
+}
+
+// Transform updates name references in resource A that
+// refer to resource B, given that B's name may have
+// changed.
+//
+// For example, a HorizontalPodAutoscaler (HPA)
+// necessarily refers to a Deployment, the thing that
+// the HPA scales. The Deployment name might change
+// (e.g. prefix added), and the reference in the HPA
+// has to be fixed.
+//
+// In the outer loop over the ResMap below, say we
+// encounter a specific HPA. Then, in scanning backrefs,
+// we encounter an entry like
+//
+//   - kind: Deployment
+//     fieldSpecs:
+//     - kind: HorizontalPodAutoscaler
+//       path: spec/scaleTargetRef/name
+//
+// This entry says that an HPA, via its
+// 'spec/scaleTargetRef/name' field, may refer to a
+// Deployment.  This match to HPA means we may need to
+// modify the value in its 'spec/scaleTargetRef/name'
+// field, by searching for the thing it refers to,
+// and getting its new name.
+//
+// As a filter, and search optimization, we compute a
+// subset of all resources that the HPA could refer to,
+// by excluding objects from other namespaces, and
+// excluding objects that don't have the same prefix-
+// suffix mods as the HPA.
+//
+// We look in this subset for all Deployment objects
+// with a resId that has a Name matching the field value
+// present in the HPA.  If no match do nothing; if more
+// than one match, it's an error.
+//
+// We overwrite the HPA name field with the value found
+// in the Deployment's name field (the name in the raw
+// object - the modified name - not the unmodified name
+// in the Deployment's resId).
+//
+// This process assumes that the name stored in a ResId
+// (the ResMap key) isn't modified by name transformers.
+// Name transformers should only modify the name in the
+// body of the resource object (the value in the ResMap).
+//
+func (o *nameReferenceTransformer) Transform(m resmap.ResMap) error {
+	// TODO: Too much looping, here and in transitive calls.
+	for _, referrer := range m.Resources() {
+		var candidates resmap.ResMap
+		for _, target := range o.backRefs {
+			for _, fSpec := range target.FieldSpecs {
+				if referrer.OrgId().IsSelected(&fSpec.Gvk) {
+					if candidates == nil {
+						candidates = m.SubsetThatCouldBeReferencedByResource(referrer)
+					}
+					err := transform.MutateField(
+						referrer.Map(),
+						fSpec.PathSlice(),
+						fSpec.CreateIfNotPresent,
+						o.getNewNameFunc(
+							// referrer could be an HPA instance,
+							// target could be Gvk for Deployment,
+							// candidate a list of resources "reachable"
+							// from the HPA.
+							referrer, target.Gvk, candidates))
+					if err != nil {
+						return err
+					}
+				}
+			}
+		}
+	}
+	return nil
+}
+
+// selectReferral picks the referral among a subset of candidates.
+// It returns the current name and namespace of the selected candidate.
+// Note that the content of the referricalCandidateSubset slice is most of the time
+// identical to the referralCandidates resmap. Still in some cases, such
+// as ClusterRoleBinding, the subset only contains the resources of a specific
+// namespace.
+func (o *nameReferenceTransformer) selectReferral(
+	oldName string,
+	referrer *resource.Resource,
+	target resid.Gvk,
+	referralCandidates resmap.ResMap,
+	referralCandidateSubset []*resource.Resource) (interface{}, interface{}, error) {
+
+	for _, res := range referralCandidateSubset {
+		id := res.OrgId()
+		if id.IsSelected(&target) && res.GetOriginalName() == oldName {
+			matches := referralCandidates.GetMatchingResourcesByOriginalId(id.Equals)
+			// If there's more than one match, there's no way
+			// to know which one to pick, so emit error.
+			if len(matches) > 1 {
+				return nil, nil, fmt.Errorf(
+					"multiple matches for %s:\n  %v",
+					id, getIds(matches))
+			}
+			// In the resource, note that it is referenced
+			// by the referrer.
+			res.AppendRefBy(referrer.CurId())
+			// Return transformed name of the object,
+			// complete with prefixes, hashes, etc.
+			return res.GetName(), res.GetNamespace(), nil
+		}
+	}
+
+	return oldName, nil, nil
+}
+
+// utility function to replace a simple string by the new name
+func (o *nameReferenceTransformer) getSimpleNameField(
+	oldName string,
+	referrer *resource.Resource,
+	target resid.Gvk,
+	referralCandidates resmap.ResMap,
+	referralCandidateSubset []*resource.Resource) (interface{}, error) {
+
+	newName, _, err := o.selectReferral(oldName, referrer, target,
+		referralCandidates, referralCandidateSubset)
+
+	return newName, err
+}
+
+// utility function to replace name field within a map[string]interface{}
+// and leverage the namespace field.
+func (o *nameReferenceTransformer) getNameAndNsStruct(
+	inMap map[string]interface{},
+	referrer *resource.Resource,
+	target resid.Gvk,
+	referralCandidates resmap.ResMap) (interface{}, error) {
+
+	// Example:
+	if _, ok := inMap["name"]; !ok {
+		return nil, fmt.Errorf(
+			"%#v is expected to contain a name field", inMap)
+	}
+	oldName, ok := inMap["name"].(string)
+	if !ok {
+		return nil, fmt.Errorf(
+			"%#v is expected to contain a name field of type string", oldName)
+	}
+
+	subset := referralCandidates.Resources()
+	if namespacevalue, ok := inMap["namespace"]; ok {
+		namespace := namespacevalue.(string)
+		bynamespace := referralCandidates.GroupedByOriginalNamespace()
+		if _, ok := bynamespace[namespace]; !ok {
+			return inMap, nil
+		}
+		subset = bynamespace[namespace]
+	}
+
+	newname, newnamespace, err := o.selectReferral(oldName, referrer, target,
+		referralCandidates, subset)
+	if err != nil {
+		return nil, err
+	}
+
+	if (newname == oldName) && (newnamespace == nil) {
+		// no candidate found.
+		return inMap, nil
+	}
+
+	inMap["name"] = newname
+	if newnamespace != "" {
+		// We don't want value "" to replace value "default" since
+		// the empty string is handled as a wild card here not default namespace
+		// by kubernetes.
+		inMap["namespace"] = newnamespace
+	}
+	return inMap, nil
+
+}
+
+func (o *nameReferenceTransformer) getNewNameFunc(
+	referrer *resource.Resource,
+	target resid.Gvk,
+	referralCandidates resmap.ResMap) func(in interface{}) (interface{}, error) {
+	return func(in interface{}) (interface{}, error) {
+		switch in.(type) {
+		case string:
+			oldName, _ := in.(string)
+			return o.getSimpleNameField(oldName, referrer, target,
+				referralCandidates, referralCandidates.Resources())
+		case map[string]interface{}:
+			// Kind: ValidatingWebhookConfiguration
+			// FieldSpec is webhooks/clientConfig/service
+			oldMap, _ := in.(map[string]interface{})
+			return o.getNameAndNsStruct(oldMap, referrer, target,
+				referralCandidates)
+		case []interface{}:
+			l, _ := in.([]interface{})
+			for idx, item := range l {
+				switch item.(type) {
+				case string:
+					// Kind: Role/ClusterRole
+					// FieldSpec is rules.resourceNames
+					oldName, _ := item.(string)
+					newName, err := o.getSimpleNameField(oldName, referrer, target,
+						referralCandidates, referralCandidates.Resources())
+					if err != nil {
+						return nil, err
+					}
+					l[idx] = newName
+				case map[string]interface{}:
+					// Kind: RoleBinding/ClusterRoleBinding
+					// FieldSpec is subjects
+					// Note: The corresponding fieldSpec had been changed from
+					// from path: subjects/name to just path: subjects. This is
+					// what get mutatefield to request the mapping of the whole
+					// map containing namespace and name instead of just a simple
+					// string field containing the name
+					oldMap, _ := item.(map[string]interface{})
+					newMap, err := o.getNameAndNsStruct(oldMap, referrer, target,
+						referralCandidates)
+					if err != nil {
+						return nil, err
+					}
+					l[idx] = newMap
+				default:
+					return nil, fmt.Errorf(
+						"%#v is expected to be either a []string or a []map[string]interface{}", in)
+				}
+			}
+			return in, nil
+		default:
+			return nil, fmt.Errorf(
+				"%#v is expected to be either a string or a []interface{}", in)
+		}
+	}
+}
+
+func getIds(rs []*resource.Resource) []string {
+	var result []string
+	for _, r := range rs {
+		result = append(result, r.CurId().String()+"\n")
+	}
+	return result
+}

api/internal/accumulator/refvartransformer.go

@@ -0,0 +1,110 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package accumulator
+
+import (
+	"fmt"
+
+	expansion2 "sigs.k8s.io/kustomize/v3/api/internal/accumulator/expansion"
+
+	"sigs.k8s.io/kustomize/v3/api/resmap"
+	"sigs.k8s.io/kustomize/v3/api/transform"
+	"sigs.k8s.io/kustomize/v3/api/types"
+)
+
+type refVarTransformer struct {
+	varMap            map[string]interface{}
+	replacementCounts map[string]int
+	fieldSpecs        []types.FieldSpec
+	mappingFunc       func(string) interface{}
+}
+
+// newRefVarTransformer returns a new refVarTransformer
+// that replaces $(VAR) style variables with values.
+// The fieldSpecs are the places to look for occurrences of $(VAR).
+func newRefVarTransformer(
+	varMap map[string]interface{}, fs []types.FieldSpec) *refVarTransformer {
+	return &refVarTransformer{
+		varMap:     varMap,
+		fieldSpecs: fs,
+	}
+}
+
+// replaceVars accepts as 'in' a string, or string array, which can have
+// embedded instances of $VAR style variables, e.g. a container command string.
+// The function returns the string with the variables expanded to their final
+// values.
+func (rv *refVarTransformer) replaceVars(in interface{}) (interface{}, error) {
+	switch vt := in.(type) {
+	case []interface{}:
+		var xs []interface{}
+		for _, a := range in.([]interface{}) {
+			xs = append(xs, expansion2.Expand(a.(string), rv.mappingFunc))
+		}
+		return xs, nil
+	case map[string]interface{}:
+		inMap := in.(map[string]interface{})
+		xs := make(map[string]interface{}, len(inMap))
+		for k, v := range inMap {
+			s, ok := v.(string)
+			if !ok {
+				// This field not contain a $(VAR) since it is not
+				// of string type. For instance .spec.replicas: 3 in
+				// a Deployment object
+				xs[k] = v
+			} else {
+				// This field can potentially contains a $(VAR) since it is
+				// of string type. For instance .spec.replicas: $(REPLICAS)
+				// in a Deployment object
+				xs[k] = expansion2.Expand(s, rv.mappingFunc)
+			}
+		}
+		return xs, nil
+	case interface{}:
+		s, ok := in.(string)
+		if !ok {
+			// This field not contain a $(VAR) since it is not of string type.
+			return in, nil
+		}
+		// This field can potentially contain a $(VAR) since it is
+		// of string type.
+		return expansion2.Expand(s, rv.mappingFunc), nil
+	case nil:
+		return nil, nil
+	default:
+		return "", fmt.Errorf("invalid type encountered %T", vt)
+	}
+}
+
+// UnusedVars returns slice of Var names that were unused
+// after a Transform run.
+func (rv *refVarTransformer) UnusedVars() []string {
+	var unused []string
+	for k := range rv.varMap {
+		_, ok := rv.replacementCounts[k]
+		if !ok {
+			unused = append(unused, k)
+		}
+	}
+	return unused
+}
+
+// Transform replaces $(VAR) style variables with values.
+func (rv *refVarTransformer) Transform(m resmap.ResMap) error {
+	rv.replacementCounts = make(map[string]int)
+	rv.mappingFunc = expansion2.MappingFuncFor(
+		rv.replacementCounts, rv.varMap)
+	for _, res := range m.Resources() {
+		for _, fieldSpec := range rv.fieldSpecs {
+			if res.OrgId().IsSelected(&fieldSpec.Gvk) {
+				if err := transform.MutateField(
+					res.Map(), fieldSpec.PathSlice(),
+					false, rv.replaceVars); err != nil {
+					return err
+				}
+			}
+		}
+	}
+	return nil
+}

api/internal/accumulator/refvartransformer_test.go

@@ -0,0 +1,137 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package accumulator
+
+import (
+	"reflect"
+	"testing"
+
+	"sigs.k8s.io/kustomize/v3/api/resid"
+	"sigs.k8s.io/kustomize/v3/api/resmap"
+	"sigs.k8s.io/kustomize/v3/api/resource"
+	"sigs.k8s.io/kustomize/v3/api/testutils/resmaptest"
+	"sigs.k8s.io/kustomize/v3/api/types"
+	"sigs.k8s.io/kustomize/v3/k8sdeps/kunstruct"
+)
+
+func TestRefVarTransformer(t *testing.T) {
+	type given struct {
+		varMap map[string]interface{}
+		fs     []types.FieldSpec
+		res    resmap.ResMap
+	}
+	type expected struct {
+		res    resmap.ResMap
+		unused []string
+	}
+	testCases := []struct {
+		description string
+		given       given
+		expected    expected
+	}{
+		{
+			description: "var replacement in map[string]",
+			given: given{
+				varMap: map[string]interface{}{
+					"FOO": "replacementForFoo",
+					"BAR": "replacementForBar",
+					"BAZ": int64(5),
+					"BOO": true,
+				},
+				fs: []types.FieldSpec{
+					{Gvk: resid.Gvk{Version: "v1", Kind: "ConfigMap"}, Path: "data/map"},
+					{Gvk: resid.Gvk{Version: "v1", Kind: "ConfigMap"}, Path: "data/slice"},
+					{Gvk: resid.Gvk{Version: "v1", Kind: "ConfigMap"}, Path: "data/interface"},
+					{Gvk: resid.Gvk{Version: "v1", Kind: "ConfigMap"}, Path: "data/nil"},
+					{Gvk: resid.Gvk{Version: "v1", Kind: "ConfigMap"}, Path: "data/num"},
+				},
+				res: resmaptest_test.NewRmBuilder(
+					t, resource.NewFactory(kunstruct.NewKunstructuredFactoryImpl())).
+					Add(map[string]interface{}{
+						"apiVersion": "v1",
+						"kind":       "ConfigMap",
+						"metadata": map[string]interface{}{
+							"name": "cm1",
+						},
+						"data": map[string]interface{}{
+							"map": map[string]interface{}{
+								"item1": "$(FOO)",
+								"item2": "bla",
+								"item3": "$(BAZ)",
+								"item4": "$(BAZ)+$(BAZ)",
+								"item5": "$(BOO)",
+								"item6": "if $(BOO)",
+								"item7": 2019,
+							},
+							"slice": []interface{}{
+								"$(FOO)",
+								"bla",
+								"$(BAZ)",
+								"$(BAZ)+$(BAZ)",
+								"$(BOO)",
+								"if $(BOO)",
+							},
+							"interface": "$(FOO)",
+							"nil":       nil,
+							"num":       2019,
+						}}).ResMap(),
+			},
+			expected: expected{
+				res: resmaptest_test.NewRmBuilder(
+					t, resource.NewFactory(kunstruct.NewKunstructuredFactoryImpl())).
+					Add(map[string]interface{}{
+						"apiVersion": "v1",
+						"kind":       "ConfigMap",
+						"metadata": map[string]interface{}{
+							"name": "cm1",
+						},
+						"data": map[string]interface{}{
+							"map": map[string]interface{}{
+								"item1": "replacementForFoo",
+								"item2": "bla",
+								"item3": int64(5),
+								"item4": "5+5",
+								"item5": true,
+								"item6": "if true",
+								"item7": 2019,
+							},
+							"slice": []interface{}{
+								"replacementForFoo",
+								"bla",
+								int64(5),
+								"5+5",
+								true,
+								"if true",
+							},
+							"interface": "replacementForFoo",
+							"nil":       nil,
+							"num":       2019,
+						}}).ResMap(),
+				unused: []string{"BAR"},
+			},
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.description, func(t *testing.T) {
+			// arrange
+			tr := newRefVarTransformer(tc.given.varMap, tc.given.fs)
+
+			// act
+			err := tr.Transform(tc.given.res)
+
+			// assert
+			if err != nil {
+				t.Errorf("unexpected error: %v", err)
+			}
+
+			a, e := tc.given.res, tc.expected.res
+			if !reflect.DeepEqual(a, e) {
+				err = e.ErrorIfNotEqualLists(a)
+				t.Fatalf("actual doesn't match expected: \nACTUAL:\n%v\nEXPECTED:\n%v\nERR: %v", a, e, err)
+			}
+
+		})
+	}
+}

api/internal/accumulator/resaccumulator.go

@@ -0,0 +1,166 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package accumulator
+
+import (
+	"fmt"
+	"log"
+	"strings"
+
+	"sigs.k8s.io/kustomize/v3/api/builtinconfig"
+	"sigs.k8s.io/kustomize/v3/api/resid"
+	"sigs.k8s.io/kustomize/v3/api/resmap"
+	"sigs.k8s.io/kustomize/v3/api/types"
+)
+
+// ResAccumulator accumulates resources and the rules
+// used to customize those resources.  It's a ResMap
+// plus stuff needed to modify the ResMap.
+type ResAccumulator struct {
+	resMap  resmap.ResMap
+	tConfig *builtinconfig.TransformerConfig
+	varSet  types.VarSet
+}
+
+func MakeEmptyAccumulator() *ResAccumulator {
+	ra := &ResAccumulator{}
+	ra.resMap = resmap.New()
+	ra.tConfig = &builtinconfig.TransformerConfig{}
+	ra.varSet = types.NewVarSet()
+	return ra
+}
+
+// ResMap returns a copy of the internal resMap.
+func (ra *ResAccumulator) ResMap() resmap.ResMap {
+	return ra.resMap.ShallowCopy()
+}
+
+// Vars returns a copy of underlying vars.
+func (ra *ResAccumulator) Vars() []types.Var {
+	return ra.varSet.AsSlice()
+}
+
+func (ra *ResAccumulator) AppendAll(
+	resources resmap.ResMap) error {
+	return ra.resMap.AppendAll(resources)
+}
+
+func (ra *ResAccumulator) AbsorbAll(
+	resources resmap.ResMap) error {
+	return ra.resMap.AbsorbAll(resources)
+}
+
+func (ra *ResAccumulator) MergeConfig(
+	tConfig *builtinconfig.TransformerConfig) (err error) {
+	ra.tConfig, err = ra.tConfig.Merge(tConfig)
+	return err
+}
+
+func (ra *ResAccumulator) GetTransformerConfig() *builtinconfig.TransformerConfig {
+	return ra.tConfig
+}
+
+func (ra *ResAccumulator) MergeVars(incoming []types.Var) error {
+	for _, v := range incoming {
+		targetId := resid.NewResIdWithNamespace(v.ObjRef.GVK(), v.ObjRef.Name, v.ObjRef.Namespace)
+		idMatcher := targetId.GvknEquals
+		if targetId.Namespace != "" || !targetId.IsNamespaceableKind() {
+			// Preserve backward compatibility. An empty namespace means
+			// wildcard search on the namespace hence we still use GvknEquals
+			idMatcher = targetId.Equals
+		}
+		matched := ra.resMap.GetMatchingResourcesByOriginalId(idMatcher)
+		if len(matched) > 1 {
+			return fmt.Errorf(
+				"found %d resId matches for var %s "+
+					"(unable to disambiguate)",
+				len(matched), v)
+		}
+		if len(matched) == 1 {
+			matched[0].AppendRefVarName(v)
+		}
+	}
+	return ra.varSet.MergeSlice(incoming)
+}
+
+func (ra *ResAccumulator) MergeAccumulator(other *ResAccumulator) (err error) {
+	err = ra.AppendAll(other.resMap)
+	if err != nil {
+		return err
+	}
+	err = ra.MergeConfig(other.tConfig)
+	if err != nil {
+		return err
+	}
+	return ra.varSet.MergeSet(other.varSet)
+}
+
+func (ra *ResAccumulator) findVarValueFromResources(v types.Var) (interface{}, error) {
+	for _, res := range ra.resMap.Resources() {
+		for _, varName := range res.GetRefVarNames() {
+			if varName == v.Name {
+				s, err := res.GetFieldValue(v.FieldRef.FieldPath)
+				if err != nil {
+					return "", fmt.Errorf(
+						"field specified in var '%v' "+
+							"not found in corresponding resource", v)
+				}
+
+				return s, nil
+			}
+		}
+	}
+
+	return "", fmt.Errorf(
+		"var '%v' cannot be mapped to a field "+
+			"in the set of known resources", v)
+}
+
+// makeVarReplacementMap returns a map of Var names to
+// their final values. The values are strings intended
+// for substitution wherever the $(var.Name) occurs.
+func (ra *ResAccumulator) makeVarReplacementMap() (map[string]interface{}, error) {
+	result := map[string]interface{}{}
+	for _, v := range ra.Vars() {
+		s, err := ra.findVarValueFromResources(v)
+		if err != nil {
+			return nil, err
+		}
+
+		result[v.Name] = s
+	}
+
+	return result, nil
+}
+
+func (ra *ResAccumulator) Transform(t resmap.Transformer) error {
+	return t.Transform(ra.resMap)
+}
+
+func (ra *ResAccumulator) ResolveVars() error {
+	replacementMap, err := ra.makeVarReplacementMap()
+	if err != nil {
+		return err
+	}
+	if len(replacementMap) == 0 {
+		return nil
+	}
+	t := newRefVarTransformer(
+		replacementMap, ra.tConfig.VarReference)
+	err = ra.Transform(t)
+	if len(t.UnusedVars()) > 0 {
+		log.Printf(
+			"well-defined vars that were never replaced: %s\n",
+			strings.Join(t.UnusedVars(), ","))
+	}
+	return err
+}
+
+func (ra *ResAccumulator) FixBackReferences() (err error) {
+	if ra.tConfig.NameReference == nil {
+		return nil
+	}
+	return ra.Transform(newNameReferenceTransformer(
+		ra.tConfig.NameReference))
+}

api/internal/accumulator/resaccumulator_test.go

@@ -0,0 +1,417 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package accumulator_test
+
+import (
+	"bytes"
+	"log"
+	"os"
+	"strings"
+	"testing"
+
+	"sigs.k8s.io/kustomize/v3/api/builtinconfig"
+	. "sigs.k8s.io/kustomize/v3/api/internal/accumulator"
+	"sigs.k8s.io/kustomize/v3/api/resid"
+	"sigs.k8s.io/kustomize/v3/api/resmap"
+	"sigs.k8s.io/kustomize/v3/api/resource"
+	"sigs.k8s.io/kustomize/v3/api/testutils/resmaptest"
+	"sigs.k8s.io/kustomize/v3/api/types"
+	"sigs.k8s.io/kustomize/v3/k8sdeps/kunstruct"
+)
+
+func makeResAccumulator(t *testing.T) (*ResAccumulator, *resource.Factory) {
+	ra := MakeEmptyAccumulator()
+	err := ra.MergeConfig(builtinconfig.MakeDefaultConfig())
+	if err != nil {
+		t.Fatalf("unexpected err: %v", err)
+	}
+	rf := resource.NewFactory(
+		kunstruct.NewKunstructuredFactoryImpl())
+	err = ra.AppendAll(
+		resmaptest_test.NewRmBuilder(t, rf).
+			Add(map[string]interface{}{
+				"apiVersion": "apps/v1",
+				"kind":       "Deployment",
+				"metadata": map[string]interface{}{
+					"name": "deploy1",
+				},
+				"spec": map[string]interface{}{
+					"template": map[string]interface{}{
+						"spec": map[string]interface{}{
+							"containers": []interface{}{
+								map[string]interface{}{
+									"command": []interface{}{
+										"myserver",
+										"--somebackendService $(SERVICE_ONE)",
+										"--yetAnother $(SERVICE_TWO)",
+									},
+								},
+							},
+						},
+					},
+				}}).
+			Add(map[string]interface{}{
+				"apiVersion": "v1",
+				"kind":       "Service",
+				"metadata": map[string]interface{}{
+					"name": "backendOne",
+				}}).
+			Add(map[string]interface{}{
+				"apiVersion": "v1",
+				"kind":       "Service",
+				"metadata": map[string]interface{}{
+					"name": "backendTwo",
+				}}).ResMap())
+	if err != nil {
+		t.Fatalf("unexpected err: %v", err)
+	}
+	return ra, rf
+}
+
+func TestResolveVarsHappy(t *testing.T) {
+	ra, _ := makeResAccumulator(t)
+	err := ra.MergeVars([]types.Var{
+		{
+			Name: "SERVICE_ONE",
+			ObjRef: types.Target{
+				Gvk:  resid.Gvk{Version: "v1", Kind: "Service"},
+				Name: "backendOne"},
+		},
+		{
+			Name: "SERVICE_TWO",
+			ObjRef: types.Target{
+				Gvk:  resid.Gvk{Version: "v1", Kind: "Service"},
+				Name: "backendTwo"},
+		},
+	})
+	if err != nil {
+		t.Fatalf("unexpected err: %v", err)
+	}
+	err = ra.ResolveVars()
+	if err != nil {
+		t.Fatalf("unexpected err: %v", err)
+	}
+	c := getCommand(find("deploy1", ra.ResMap()))
+	if c != "myserver --somebackendService backendOne --yetAnother backendTwo" {
+		t.Fatalf("unexpected command: %s", c)
+	}
+}
+
+func TestResolveVarsOneUnused(t *testing.T) {
+	ra, _ := makeResAccumulator(t)
+	err := ra.MergeVars([]types.Var{
+		{
+			Name: "SERVICE_ONE",
+			ObjRef: types.Target{
+				Gvk:  resid.Gvk{Version: "v1", Kind: "Service"},
+				Name: "backendOne"},
+		},
+		{
+			Name: "SERVICE_UNUSED",
+			ObjRef: types.Target{
+				Gvk:  resid.Gvk{Version: "v1", Kind: "Service"},
+				Name: "backendTwo"},
+		},
+	})
+	if err != nil {
+		t.Fatalf("unexpected err: %v", err)
+	}
+	var buf bytes.Buffer
+	log.SetOutput(&buf)
+	defer func() {
+		log.SetOutput(os.Stderr)
+	}()
+	err = ra.ResolveVars()
+	if err != nil {
+		t.Fatalf("unexpected err: %v", err)
+	}
+	expectLog(t, buf, "well-defined vars that were never replaced: SERVICE_UNUSED")
+	c := getCommand(find("deploy1", ra.ResMap()))
+	if c != "myserver --somebackendService backendOne --yetAnother $(SERVICE_TWO)" {
+		t.Fatalf("unexpected command: %s", c)
+	}
+}
+
+func expectLog(t *testing.T, log bytes.Buffer, expect string) {
+	if !strings.Contains(log.String(), expect) {
+		t.Fatalf("expected log containing '%s', got '%s'", expect, log.String())
+	}
+}
+
+func TestResolveVarsVarNeedsDisambiguation(t *testing.T) {
+	ra, rf := makeResAccumulator(t)
+
+	rm0 := resmap.New()
+	err := rm0.Append(
+		rf.FromMap(
+			map[string]interface{}{
+				"apiVersion": "v1",
+				"kind":       "Service",
+				"metadata": map[string]interface{}{
+					"name":      "backendOne",
+					"namespace": "fooNamespace",
+				},
+			}))
+	if err != nil {
+		t.Fatalf("unexpected err: %v", err)
+	}
+	err = ra.AppendAll(rm0)
+	if err != nil {
+		t.Fatalf("unexpected err: %v", err)
+	}
+
+	err = ra.MergeVars([]types.Var{
+		{
+			Name: "SERVICE_ONE",
+			ObjRef: types.Target{
+				Gvk:  resid.Gvk{Version: "v1", Kind: "Service"},
+				Name: "backendOne",
+			},
+		},
+	})
+	if err == nil {
+		t.Fatalf("expected error")
+	}
+	if !strings.Contains(
+		err.Error(), "unable to disambiguate") {
+		t.Fatalf("unexpected err: %v", err)
+	}
+}
+
+func makeNamespacedConfigMapWithDataProviderValue(
+	namespace string,
+	value string,
+) map[string]interface{} {
+	return map[string]interface{}{
+		"apiVersion": "v1",
+		"kind":       "ConfigMap",
+		"metadata": map[string]interface{}{
+			"name":      "environment",
+			"namespace": namespace,
+		},
+		"data": map[string]interface{}{
+			"provider": value,
+		},
+	}
+}
+
+func makeVarToNamepaceAndPath(
+	name string,
+	namespace string,
+	path string,
+) types.Var {
+	return types.Var{
+		Name: name,
+		ObjRef: types.Target{
+			Gvk:       resid.Gvk{Version: "v1", Kind: "ConfigMap"},
+			Name:      "environment",
+			Namespace: namespace,
+		},
+		FieldRef: types.FieldSelector{FieldPath: path},
+	}
+}
+
+func TestResolveVarConflicts(t *testing.T) {
+	rf := resource.NewFactory(kunstruct.NewKunstructuredFactoryImpl())
+
+	// create configmaps in foo and bar namespaces with `data.provider` values.
+	fooAws := makeNamespacedConfigMapWithDataProviderValue("foo", "aws")
+	barAws := makeNamespacedConfigMapWithDataProviderValue("bar", "aws")
+	barGcp := makeNamespacedConfigMapWithDataProviderValue("bar", "gcp")
+
+	// create two variables with (apparently) conflicting names that point to
+	// fieldpaths that could be generalized.
+	varFoo := makeVarToNamepaceAndPath("PROVIDER", "foo", "data.provider")
+	varBar := makeVarToNamepaceAndPath("PROVIDER", "bar", "data.provider")
+
+	// create accumulators holding apparently conflicting vars that are not
+	// actually in conflict because they point to the same concrete value.
+	rm0 := resmap.New()
+	rm0.Append(rf.FromMap(fooAws))
+	ac0 := MakeEmptyAccumulator()
+	ac0.AppendAll(rm0)
+	ac0.MergeVars([]types.Var{varFoo})
+
+	rm1 := resmap.New()
+	rm1.Append(rf.FromMap(barAws))
+	ac1 := MakeEmptyAccumulator()
+	ac1.AppendAll(rm1)
+	ac1.MergeVars([]types.Var{varBar})
+
+	// validate that two vars of the same name which reference the same concrete
+	// value do not produce a conflict.
+	err := ac0.MergeAccumulator(ac1)
+	if err == nil {
+		t.Fatalf("see bug gh-1600")
+	}
+
+	// create an accumulator will have an actually conflicting value with the
+	// two above (because it contains a variable whose name is used in the other
+	// accumulators AND whose concrete values are different).
+	rm2 := resmap.New()
+	rm2.Append(rf.FromMap(barGcp))
+	ac2 := MakeEmptyAccumulator()
+	ac2.AppendAll(rm2)
+	ac2.MergeVars([]types.Var{varBar})
+	err = ac1.MergeAccumulator(ac2)
+	if err == nil {
+		t.Fatalf("dupe vars w/ different concrete values should conflict")
+	}
+}
+
+func TestResolveVarsGoodResIdBadField(t *testing.T) {
+	ra, _ := makeResAccumulator(t)
+	err := ra.MergeVars([]types.Var{
+		{
+			Name: "SERVICE_ONE",
+			ObjRef: types.Target{
+				Gvk:  resid.Gvk{Version: "v1", Kind: "Service"},
+				Name: "backendOne"},
+			FieldRef: types.FieldSelector{FieldPath: "nope_nope_nope"},
+		},
+	})
+	if err != nil {
+		t.Fatalf("unexpected err: %v", err)
+	}
+	err = ra.ResolveVars()
+	if err == nil {
+		t.Fatalf("expected error")
+	}
+	if !strings.Contains(
+		err.Error(),
+		"not found in corresponding resource") {
+		t.Fatalf("unexpected err: %v", err)
+	}
+}
+
+func TestResolveVarsUnmappableVar(t *testing.T) {
+	ra, _ := makeResAccumulator(t)
+	err := ra.MergeVars([]types.Var{
+		{
+			Name: "SERVICE_THREE",
+			ObjRef: types.Target{
+				Gvk:  resid.Gvk{Version: "v1", Kind: "Service"},
+				Name: "doesNotExist"},
+		},
+	})
+	if err != nil {
+		t.Fatalf("unexpected err: %v", err)
+	}
+	err = ra.ResolveVars()
+	if err == nil {
+		t.Fatalf("expected error")
+	}
+	if !strings.Contains(
+		err.Error(),
+		"cannot be mapped to a field in the set of known resources") {
+		t.Fatalf("unexpected err: %v", err)
+	}
+}
+
+func TestResolveVarsWithNoambiguation(t *testing.T) {
+	ra1, rf := makeResAccumulator(t)
+	err := ra1.MergeVars([]types.Var{
+		{
+			Name: "SERVICE_ONE",
+			ObjRef: types.Target{
+				Gvk:  resid.Gvk{Version: "v1", Kind: "Service"},
+				Name: "backendOne",
+			},
+		},
+	})
+	if err != nil {
+		t.Fatalf("unexpected err: %v", err)
+	}
+
+	// Create another accumulator having a resource with different prefix
+	ra2 := MakeEmptyAccumulator()
+
+	m := resmaptest_test.NewRmBuilder(t, rf).
+		Add(map[string]interface{}{
+			"apiVersion": "apps/v1",
+			"kind":       "Deployment",
+			"metadata": map[string]interface{}{
+				"name": "deploy2",
+			},
+			"spec": map[string]interface{}{
+				"template": map[string]interface{}{
+					"spec": map[string]interface{}{
+						"containers": []interface{}{
+							map[string]interface{}{
+								"command": []interface{}{
+									"myserver",
+									"--somebackendService $(SUB_SERVICE_ONE)",
+								},
+							},
+						},
+					},
+				},
+			}}).
+		Add(map[string]interface{}{
+			"apiVersion": "v1",
+			"kind":       "Service",
+			"metadata": map[string]interface{}{
+				"name": "backendOne",
+			}}).ResMap()
+
+	// Make it seem like this resource
+	// went through a prefix transformer.
+	r := m.GetByIndex(1)
+	r.AddNamePrefix("sub-")
+	r.SetName("sub-backendOne") // original name remains "backendOne"
+
+	err = ra2.AppendAll(m)
+	if err != nil {
+		t.Fatalf("unexpected err: %v", err)
+	}
+
+	err = ra2.MergeVars([]types.Var{
+		{
+			Name: "SUB_SERVICE_ONE",
+			ObjRef: types.Target{
+				Gvk:  resid.Gvk{Version: "v1", Kind: "Service"},
+				Name: "backendOne",
+			},
+		},
+	})
+	if err != nil {
+		t.Fatalf("unexpected err: %v", err)
+	}
+	err = ra1.MergeAccumulator(ra2)
+	if err != nil {
+		t.Fatalf("unexpected err: %v", err)
+	}
+
+	err = ra1.ResolveVars()
+	if err != nil {
+		t.Fatalf("unexpected err: %v", err)
+	}
+}
+
+func find(name string, resMap resmap.ResMap) *resource.Resource {
+	for _, r := range resMap.Resources() {
+		if r.GetName() == name {
+			return r
+		}
+	}
+	return nil
+}
+
+// Assumes arg is a deployment, returns the command of first container.
+func getCommand(r *resource.Resource) string {
+	var m map[string]interface{}
+	var c []interface{}
+	m, _ = r.Map()["spec"].(map[string]interface{})
+	m, _ = m["template"].(map[string]interface{})
+	m, _ = m["spec"].(map[string]interface{})
+	c, _ = m["containers"].([]interface{})
+	m, _ = c[0].(map[string]interface{})
+
+	cmd, _ := m["command"].([]interface{})
+	n := make([]string, len(cmd))
+	for i, v := range cmd {
+		n[i] = v.(string)
+	}
+	return strings.Join(n, " ")
+}

api/internal/git/cloner.go

@@ -1,121 +0,0 @@
-// Copyright 2019 The Kubernetes Authors.
-// SPDX-License-Identifier: Apache-2.0
-
-package git
-
-import (
-	"bytes"
-	"log"
-	"os/exec"
-
-	"github.com/pkg/errors"
-	"sigs.k8s.io/kustomize/v3/api/filesys"
-)
-
-// Cloner is a function that can clone a git repo.
-type Cloner func(repoSpec *RepoSpec) error
-
-// ClonerUsingGitExec uses a local git install, as opposed
-// to say, some remote API, to obtain a local clone of
-// a remote repo.
-func ClonerUsingGitExec(repoSpec *RepoSpec) error {
-	gitProgram, err := exec.LookPath("git")
-	if err != nil {
-		return errors.Wrap(err, "no 'git' program on path")
-	}
-	repoSpec.Dir, err = filesys.NewTmpConfirmedDir()
-	if err != nil {
-		return err
-	}
-	cmd := exec.Command(
-		gitProgram,
-		"init",
-		repoSpec.Dir.String())
-	var out bytes.Buffer
-	cmd.Stdout = &out
-	cmd.Stderr = &out
-	err = cmd.Run()
-	if err != nil {
-		log.Printf("Error initializing empty git repo: %s", out.String())
-		return errors.Wrapf(
-			err,
-			"trouble initializing empty git repo in %s",
-			repoSpec.Dir.String())
-	}
-
-	cmd = exec.Command(
-		gitProgram,
-		"remote",
-		"add",
-		"origin",
-		repoSpec.CloneSpec())
-	cmd.Stdout = &out
-	cmd.Stderr = &out
-	cmd.Dir = repoSpec.Dir.String()
-	err = cmd.Run()
-	if err != nil {
-		log.Printf("Error setting git remote: %s", out.String())
-		return errors.Wrapf(
-			err,
-			"trouble adding remote %s",
-			repoSpec.CloneSpec())
-	}
-	if repoSpec.Ref == "" {
-		repoSpec.Ref = "master"
-	}
-	cmd = exec.Command(
-		gitProgram,
-		"fetch",
-		"--depth=1",
-		"origin",
-		repoSpec.Ref)
-	cmd.Stdout = &out
-	cmd.Stderr = &out
-	cmd.Dir = repoSpec.Dir.String()
-	err = cmd.Run()
-	if err != nil {
-		log.Printf("Error performing git fetch: %s", out.String())
-		return errors.Wrapf(err, "trouble fetching %s", repoSpec.Ref)
-	}
-
-	cmd = exec.Command(
-		gitProgram,
-		"reset",
-		"--hard",
-		"FETCH_HEAD")
-	cmd.Stdout = &out
-	cmd.Stderr = &out
-	cmd.Dir = repoSpec.Dir.String()
-	err = cmd.Run()
-	if err != nil {
-		log.Printf("Error performing git reset: %s", out.String())
-		return errors.Wrapf(
-			err, "trouble hard resetting empty repository to %s", repoSpec.Ref)
-	}
-
-	cmd = exec.Command(
-		gitProgram,
-		"submodule",
-		"update",
-		"--init",
-		"--recursive")
-	cmd.Stdout = &out
-	cmd.Dir = repoSpec.Dir.String()
-	err = cmd.Run()
-	if err != nil {
-		return errors.Wrapf(err, "trouble fetching submodules for %s", repoSpec.Ref)
-	}
-
-	return nil
-}
-
-// DoNothingCloner returns a cloner that only sets
-// cloneDir field in the repoSpec.  It's assumed that
-// the cloneDir is associated with some fake filesystem
-// used in a test.
-func DoNothingCloner(dir filesys.ConfirmedDir) Cloner {
-	return func(rs *RepoSpec) error {
-		rs.Dir = dir
-		return nil
-	}
-}

api/internal/git/repospec.go

@@ -1,219 +0,0 @@
-// Copyright 2019 The Kubernetes Authors.
-// SPDX-License-Identifier: Apache-2.0
-
-package git
-
-import (
-	"fmt"
-	"path/filepath"
-	"regexp"
-	"strings"
-
-	"sigs.k8s.io/kustomize/v3/api/filesys"
-)
-
-// Used as a temporary non-empty occupant of the cloneDir
-// field, as something distinguishable from the empty string
-// in various outputs (especially tests). Not using an
-// actual directory name here, as that's a temporary directory
-// with a unique name that isn't created until clone time.
-const notCloned = filesys.ConfirmedDir("/notCloned")
-
-// RepoSpec specifies a git repository and a branch and path therein.
-type RepoSpec struct {
-	// Raw, original spec, used to look for cycles.
-	// TODO(monopole): Drop raw, use processed fields instead.
-	raw string
-
-	// Host, e.g. github.com
-	Host string
-
-	// orgRepo name (organization/repoName),
-	// e.g. kubernetes-sigs/kustomize
-	OrgRepo string
-
-	// Dir where the orgRepo is cloned to.
-	Dir filesys.ConfirmedDir
-
-	// Relative path in the repository, and in the cloneDir,
-	// to a Kustomization.
-	Path string
-
-	// Branch or tag reference.
-	Ref string
-
-	// e.g. .git or empty in case of _git is present
-	GitSuffix string
-}
-
-// CloneSpec returns a string suitable for "git clone {spec}".
-func (x *RepoSpec) CloneSpec() string {
-	if isAzureHost(x.Host) || isAWSHost(x.Host) {
-		return x.Host + x.OrgRepo
-	}
-	return x.Host + x.OrgRepo + x.GitSuffix
-}
-
-func (x *RepoSpec) CloneDir() filesys.ConfirmedDir {
-	return x.Dir
-}
-
-func (x *RepoSpec) Raw() string {
-	return x.raw
-}
-
-func (x *RepoSpec) AbsPath() string {
-	return x.Dir.Join(x.Path)
-}
-
-func (x *RepoSpec) Cleaner(fSys filesys.FileSystem) func() error {
-	return func() error { return fSys.RemoveAll(x.Dir.String()) }
-}
-
-// From strings like git@github.com:someOrg/someRepo.git or
-// https://github.com/someOrg/someRepo?ref=someHash, extract
-// the parts.
-func NewRepoSpecFromUrl(n string) (*RepoSpec, error) {
-	if filepath.IsAbs(n) {
-		return nil, fmt.Errorf("uri looks like abs path: %s", n)
-	}
-	host, orgRepo, path, gitRef, gitSuffix := parseGitUrl(n)
-	if orgRepo == "" {
-		return nil, fmt.Errorf("url lacks orgRepo: %s", n)
-	}
-	if host == "" {
-		return nil, fmt.Errorf("url lacks host: %s", n)
-	}
-	return &RepoSpec{
-		raw: n, Host: host, OrgRepo: orgRepo,
-		Dir: notCloned, Path: path, Ref: gitRef, GitSuffix: gitSuffix}, nil
-}
-
-const (
-	refQuery      = "?ref="
-	refQueryRegex = "\\?(version|ref)="
-	gitSuffix     = ".git"
-	gitDelimiter  = "_git/"
-)
-
-// From strings like git@github.com:someOrg/someRepo.git or
-// https://github.com/someOrg/someRepo?ref=someHash, extract
-// the parts.
-func parseGitUrl(n string) (
-	host string, orgRepo string, path string, gitRef string, gitSuff string) {
-
-	if strings.Contains(n, gitDelimiter) {
-		index := strings.Index(n, gitDelimiter)
-		// Adding _git/ to host
-		host = normalizeGitHostSpec(n[:index+len(gitDelimiter)])
-		orgRepo = strings.Split(strings.Split(n[index+len(gitDelimiter):], "/")[0], "?")[0]
-		path, gitRef = peelQuery(n[index+len(gitDelimiter)+len(orgRepo):])
-		return
-	}
-	host, n = parseHostSpec(n)
-	gitSuff = gitSuffix
-	if strings.Contains(n, gitSuffix) {
-		index := strings.Index(n, gitSuffix)
-		orgRepo = n[0:index]
-		n = n[index+len(gitSuffix):]
-		path, gitRef = peelQuery(n)
-		return
-	}
-
-	i := strings.Index(n, "/")
-	if i < 1 {
-		return "", "", "", "", ""
-	}
-	j := strings.Index(n[i+1:], "/")
-	if j >= 0 {
-		j += i + 1
-		orgRepo = n[:j]
-		path, gitRef = peelQuery(n[j+1:])
-		return
-	}
-	path = ""
-	orgRepo, gitRef = peelQuery(n)
-	return host, orgRepo, path, gitRef, gitSuff
-}
-
-func peelQuery(arg string) (string, string) {
-
-	r, _ := regexp.Compile(refQueryRegex)
-	j := r.FindStringIndex(arg)
-
-	if len(j) > 0 {
-		return arg[:j[0]], arg[j[0]+len(r.FindString(arg)):]
-	}
-	return arg, ""
-}
-
-func parseHostSpec(n string) (string, string) {
-	var host string
-	// Start accumulating the host part.
-	for _, p := range []string{
-		// Order matters here.
-		"git::", "gh:", "ssh://", "https://", "http://",
-		"git@", "github.com:", "github.com/"} {
-		if len(p) < len(n) && strings.ToLower(n[:len(p)]) == p {
-			n = n[len(p):]
-			host += p
-		}
-	}
-	if host == "git@" {
-		i := strings.Index(n, "/")
-		if i > -1 {
-			host += n[:i+1]
-			n = n[i+1:]
-		} else {
-			i = strings.Index(n, ":")
-			if i > -1 {
-				host += n[:i+1]
-				n = n[i+1:]
-			}
-		}
-		return host, n
-	}
-
-	// If host is a http(s) or ssh URL, grab the domain part.
-	for _, p := range []string{
-		"ssh://", "https://", "http://"} {
-		if strings.HasSuffix(host, p) {
-			i := strings.Index(n, "/")
-			if i > -1 {
-				host = host + n[0:i+1]
-				n = n[i+1:]
-			}
-			break
-		}
-	}
-
-	return normalizeGitHostSpec(host), n
-}
-
-func normalizeGitHostSpec(host string) string {
-	s := strings.ToLower(host)
-	if strings.Contains(s, "github.com") {
-		if strings.Contains(s, "git@") || strings.Contains(s, "ssh:") {
-			host = "git@github.com:"
-		} else {
-			host = "https://github.com/"
-		}
-	}
-	if strings.HasPrefix(s, "git::") {
-		host = strings.TrimPrefix(s, "git::")
-	}
-	return host
-}
-
-// The format of Azure repo URL is documented
-// https://docs.microsoft.com/en-us/azure/devops/repos/git/clone?view=vsts&tabs=visual-studio#clone_url
-func isAzureHost(host string) bool {
-	return strings.Contains(host, "dev.azure.com") ||
-		strings.Contains(host, "visualstudio.com")
-}
-
-// The format of AWS repo URL is documented
-// https://docs.aws.amazon.com/codecommit/latest/userguide/regions.html
-func isAWSHost(host string) bool {
-	return strings.Contains(host, "amazonaws.com")
-}

api/internal/git/repospec_test.go

@@ -1,288 +0,0 @@
-// Copyright 2019 The Kubernetes Authors.
-// SPDX-License-Identifier: Apache-2.0
-
-package git
-
-import (
-	"fmt"
-	"path/filepath"
-	"strings"
-	"testing"
-)
-
-var orgRepos = []string{"someOrg/someRepo", "kubernetes/website"}
-
-var pathNames = []string{"README.md", "foo/krusty.txt", ""}
-
-var hrefArgs = []string{"someBranch", "master", "v0.1.0", ""}
-
-var hostNamesRawAndNormalized = [][]string{
-	{"gh:", "gh:"},
-	{"GH:", "gh:"},
-	{"gitHub.com/", "https://github.com/"},
-	{"github.com:", "https://github.com/"},
-	{"http://github.com/", "https://github.com/"},
-	{"https://github.com/", "https://github.com/"},
-	{"hTTps://github.com/", "https://github.com/"},
-	{"https://git-codecommit.us-east-2.amazonaws.com/", "https://git-codecommit.us-east-2.amazonaws.com/"},
-	{"https://fabrikops2.visualstudio.com/", "https://fabrikops2.visualstudio.com/"},
-	{"ssh://git.example.com:7999/", "ssh://git.example.com:7999/"},
-	{"git::https://gitlab.com/", "https://gitlab.com/"},
-	{"git::http://git.example.com/", "http://git.example.com/"},
-	{"git::https://git.example.com/", "https://git.example.com/"},
-	{"git@github.com:", "git@github.com:"},
-	{"git@github.com/", "git@github.com:"},
-	{"git@gitlab2.sqtools.ru:10022/", "git@gitlab2.sqtools.ru:10022/"},
-}
-
-func makeUrl(hostFmt, orgRepo, path, href string) string {
-	if len(path) > 0 {
-		orgRepo = filepath.Join(orgRepo, path)
-	}
-	url := hostFmt + orgRepo
-	if href != "" {
-		url += refQuery + href
-	}
-	return url
-}
-
-func TestNewRepoSpecFromUrl(t *testing.T) {
-	var bad [][]string
-	for _, tuple := range hostNamesRawAndNormalized {
-		hostRaw := tuple[0]
-		hostSpec := tuple[1]
-		for _, orgRepo := range orgRepos {
-			for _, pathName := range pathNames {
-				for _, hrefArg := range hrefArgs {
-					uri := makeUrl(hostRaw, orgRepo, pathName, hrefArg)
-					rs, err := NewRepoSpecFromUrl(uri)
-					if err != nil {
-						t.Errorf("problem %v", err)
-					}
-					if rs.Host != hostSpec {
-						bad = append(bad, []string{"host", uri, rs.Host, hostSpec})
-					}
-					if rs.OrgRepo != orgRepo {
-						bad = append(bad, []string{"orgRepo", uri, rs.OrgRepo, orgRepo})
-					}
-					if rs.Path != pathName {
-						bad = append(bad, []string{"path", uri, rs.Path, pathName})
-					}
-					if rs.Ref != hrefArg {
-						bad = append(bad, []string{"ref", uri, rs.Ref, hrefArg})
-					}
-				}
-			}
-		}
-	}
-	if len(bad) > 0 {
-		for _, tuple := range bad {
-			fmt.Printf("\n"+
-				"     from uri: %s\n"+
-				"  actual %4s: %s\n"+
-				"expected %4s: %s\n",
-				tuple[1], tuple[0], tuple[2], tuple[0], tuple[3])
-		}
-		t.Fail()
-	}
-}
-
-var badData = [][]string{
-	{"/tmp", "uri looks like abs path"},
-	{"iauhsdiuashduas", "url lacks orgRepo"},
-	{"htxxxtp://github.com/", "url lacks host"},
-	{"ssh://git.example.com", "url lacks orgRepo"},
-	{"git::___", "url lacks orgRepo"},
-}
-
-func TestNewRepoSpecFromUrlErrors(t *testing.T) {
-	for _, tuple := range badData {
-		_, err := NewRepoSpecFromUrl(tuple[0])
-		if err == nil {
-			t.Error("expected error")
-		}
-		if !strings.Contains(err.Error(), tuple[1]) {
-			t.Errorf("unexpected error: %s", err)
-		}
-	}
-}
-
-func TestNewRepoSpecFromUrl_CloneSpecs(t *testing.T) {
-	testcases := []struct {
-		input     string
-		cloneSpec string
-		absPath   string
-		ref       string
-	}{
-		{
-			input:     "https://git-codecommit.us-east-2.amazonaws.com/someorg/somerepo/somedir",
-			cloneSpec: "https://git-codecommit.us-east-2.amazonaws.com/someorg/somerepo",
-			absPath:   notCloned.Join("somedir"),
-			ref:       "",
-		},
-		{
-			input:     "https://git-codecommit.us-east-2.amazonaws.com/someorg/somerepo/somedir?ref=testbranch",
-			cloneSpec: "https://git-codecommit.us-east-2.amazonaws.com/someorg/somerepo",
-			absPath:   notCloned.Join("somedir"),
-			ref:       "testbranch",
-		},
-		{
-			input:     "https://fabrikops2.visualstudio.com/someorg/somerepo?ref=master",
-			cloneSpec: "https://fabrikops2.visualstudio.com/someorg/somerepo",
-			absPath:   notCloned.String(),
-			ref:       "master",
-		},
-		{
-			input:     "http://github.com/someorg/somerepo/somedir",
-			cloneSpec: "https://github.com/someorg/somerepo.git",
-			absPath:   notCloned.Join("somedir"),
-			ref:       "",
-		},
-		{
-			input:     "git@github.com:someorg/somerepo/somedir",
-			cloneSpec: "git@github.com:someorg/somerepo.git",
-			absPath:   notCloned.Join("somedir"),
-			ref:       "",
-		},
-		{
-			input:     "git@gitlab2.sqtools.ru:10022/infra/kubernetes/thanos-base.git?ref=v0.1.0",
-			cloneSpec: "git@gitlab2.sqtools.ru:10022/infra/kubernetes/thanos-base.git",
-			absPath:   notCloned.String(),
-			ref:       "v0.1.0",
-		},
-		{
-			input:     "git@bitbucket.org:company/project.git//path?ref=branch",
-			cloneSpec: "git@bitbucket.org:company/project.git",
-			absPath:   notCloned.Join("path"),
-			ref:       "branch",
-		},
-		{
-			input:     "https://itfs.mycompany.com/collection/project/_git/somerepos",
-			cloneSpec: "https://itfs.mycompany.com/collection/project/_git/somerepos",
-			absPath:   notCloned.String(),
-			ref:       "",
-		},
-		{
-			input:     "https://itfs.mycompany.com/collection/project/_git/somerepos?version=v1.0.0",
-			cloneSpec: "https://itfs.mycompany.com/collection/project/_git/somerepos",
-			absPath:   notCloned.String(),
-			ref:       "v1.0.0",
-		},
-		{
-			input:     "https://itfs.mycompany.com/collection/project/_git/somerepos/somedir?version=v1.0.0",
-			cloneSpec: "https://itfs.mycompany.com/collection/project/_git/somerepos",
-			absPath:   notCloned.Join("somedir"),
-			ref:       "v1.0.0",
-		},
-		{
-			input:     "git::https://itfs.mycompany.com/collection/project/_git/somerepos",
-			cloneSpec: "https://itfs.mycompany.com/collection/project/_git/somerepos",
-			absPath:   notCloned.String(),
-			ref:       "",
-		},
-	}
-	for _, testcase := range testcases {
-		rs, err := NewRepoSpecFromUrl(testcase.input)
-		if err != nil {
-			t.Errorf("Unexpected error: %v", err)
-		}
-		if rs.CloneSpec() != testcase.cloneSpec {
-			t.Errorf("CloneSpec expected to be %v, but got %v on %s",
-				testcase.cloneSpec, rs.CloneSpec(), testcase.input)
-		}
-		if rs.AbsPath() != testcase.absPath {
-			t.Errorf("AbsPath expected to be %v, but got %v on %s",
-				testcase.absPath, rs.AbsPath(), testcase.input)
-		}
-		if rs.Ref != testcase.ref {
-			t.Errorf("ref expected to be %v, but got %v on %s",
-				testcase.ref, rs.Ref, testcase.input)
-		}
-	}
-}
-
-func TestIsAzureHost(t *testing.T) {
-	testcases := []struct {
-		input  string
-		expect bool
-	}{
-		{
-			input:  "https://git-codecommit.us-east-2.amazonaws.com",
-			expect: false,
-		},
-		{
-			input:  "ssh://git-codecommit.us-east-2.amazonaws.com",
-			expect: false,
-		},
-		{
-			input:  "https://fabrikops2.visualstudio.com/",
-			expect: true,
-		},
-		{
-			input:  "https://dev.azure.com/myorg/myproject/",
-			expect: true,
-		},
-	}
-	for _, testcase := range testcases {
-		actual := isAzureHost(testcase.input)
-		if actual != testcase.expect {
-			t.Errorf("IsAzureHost: expected %v, but got %v on %s", testcase.expect, actual, testcase.input)
-		}
-	}
-}
-
-func TestPeelQuery(t *testing.T) {
-	testcases := []struct {
-		input  string
-		expect [2]string
-	}{
-		{
-			input:  "somerepos?ref=v1.0.0",
-			expect: [2]string{"somerepos", "v1.0.0"},
-		},
-		{
-			input:  "somerepos?version=master",
-			expect: [2]string{"somerepos", "master"},
-		},
-		{
-			input:  "somerepos",
-			expect: [2]string{"somerepos", ""},
-		},
-	}
-	for _, testcase := range testcases {
-		path, ref := peelQuery(testcase.input)
-		if path != testcase.expect[0] || ref != testcase.expect[1] {
-			t.Errorf("peelQuery: expected (%s, %s) got (%s, %s) on %s", testcase.expect[0], testcase.expect[1], path, ref, testcase.input)
-		}
-	}
-}
-
-func TestIsAWSHost(t *testing.T) {
-	testcases := []struct {
-		input  string
-		expect bool
-	}{
-		{
-			input:  "https://git-codecommit.us-east-2.amazonaws.com",
-			expect: true,
-		},
-		{
-			input:  "ssh://git-codecommit.us-east-2.amazonaws.com",
-			expect: true,
-		},
-		{
-			input:  "git@github.com:",
-			expect: false,
-		},
-		{
-			input:  "http://github.com/",
-			expect: false,
-		},
-	}
-	for _, testcase := range testcases {
-		actual := isAWSHost(testcase.input)
-		if actual != testcase.expect {
-			t.Errorf("IsAWSHost: expected %v, but got %v on %s", testcase.expect, actual, testcase.input)
-		}
-	}
-}

api/internal/kusterr/yamlformaterror.go

@@ -0,0 +1,35 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+// Package error has contextual error types.
+package kusterr
+
+import (
+	"fmt"
+	"strings"
+)
+
+// YamlFormatError represents error with yaml file name where json/yaml format error happens.
+type YamlFormatError struct {
+	Path     string
+	ErrorMsg string
+}
+
+func (e YamlFormatError) Error() string {
+	return fmt.Sprintf("YAML file [%s] encounters a format error.\n%s\n", e.Path, e.ErrorMsg)
+}
+
+// Handler handles YamlFormatError
+func Handler(e error, path string) error {
+	if isYAMLSyntaxError(e) {
+		return YamlFormatError{
+			Path:     path,
+			ErrorMsg: e.Error(),
+		}
+	}
+	return e
+}
+
+func isYAMLSyntaxError(e error) bool {
+	return strings.Contains(e.Error(), "error converting YAML to JSON") || strings.Contains(e.Error(), "error unmarshaling JSON")
+}

api/internal/kusterr/yamlformaterror_test.go

@@ -0,0 +1,41 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package kusterr
+
+import (
+	"fmt"
+	"testing"
+)
+
+const (
+	filepath = "/path/to/whatever"
+	expected = "YAML file [/path/to/whatever] encounters a format error.\n" +
+		"error converting YAML to JSON: yaml: line 2: found character that cannot start any token\n"
+)
+
+func TestYamlFormatError_Error(t *testing.T) {
+	testErr := YamlFormatError{
+		Path:     filepath,
+		ErrorMsg: "error converting YAML to JSON: yaml: line 2: found character that cannot start any token",
+	}
+	if testErr.Error() != expected {
+		t.Errorf("Expected : %s\n, but found : %s\n", expected, testErr.Error())
+	}
+}
+
+func TestErrorHandler(t *testing.T) {
+	err := fmt.Errorf("error converting YAML to JSON: yaml: line 2: found character that cannot start any token")
+	testErr := Handler(err, filepath)
+	expectedErr := fmt.Errorf("format error message")
+	fmtErr := Handler(expectedErr, filepath)
+	if fmtErr.Error() != expectedErr.Error() {
+		t.Errorf("Expected returning fmt.Error, but found %T", fmtErr)
+	}
+	if _, ok := testErr.(YamlFormatError); !ok {
+		t.Errorf("Expected returning YamlFormatError, but found %T", testErr)
+	}
+	if testErr == nil || testErr.Error() != expected {
+		t.Errorf("Expected : %s\n, but found : %s\n", expected, testErr.Error())
+	}
+}

api/internal/loadertest/fakeloader.go

@@ -0,0 +1,76 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+// Package loadertest holds a fake for the Loader interface.
+package loadertest
+
+import (
+	"log"
+
+	"sigs.k8s.io/kustomize/v3/api/filesys"
+	"sigs.k8s.io/kustomize/v3/api/ifc"
+	"sigs.k8s.io/kustomize/v3/api/loader"
+)
+
+// FakeLoader encapsulates the delegate Loader and the fake file system.
+type FakeLoader struct {
+	fs       filesys.FileSystem
+	delegate ifc.Loader
+}
+
+// NewFakeLoader returns a Loader that uses a fake filesystem.
+// The loader will be restricted to root only.
+// The initialDir argument should be an absolute file path.
+func NewFakeLoader(initialDir string) FakeLoader {
+	return NewFakeLoaderWithRestrictor(
+		loader.RestrictionRootOnly, initialDir)
+}
+
+// NewFakeLoaderWithRestrictor returns a Loader that
+// uses a fake filesystem.
+// The initialDir argument should be an absolute file path.
+func NewFakeLoaderWithRestrictor(
+	lr loader.LoadRestrictorFunc, initialDir string) FakeLoader {
+	// Create fake filesystem and inject it into initial Loader.
+	fSys := filesys.MakeFsInMemory()
+	fSys.Mkdir(initialDir)
+	ldr, err := loader.NewLoader(lr, initialDir, fSys)
+	if err != nil {
+		log.Fatalf("Unable to make loader: %v", err)
+	}
+	return FakeLoader{fs: fSys, delegate: ldr}
+}
+
+// AddFile adds a fake file to the file system.
+func (f FakeLoader) AddFile(fullFilePath string, content []byte) error {
+	return f.fs.WriteFile(fullFilePath, content)
+}
+
+// AddDirectory adds a fake directory to the file system.
+func (f FakeLoader) AddDirectory(fullDirPath string) error {
+	return f.fs.Mkdir(fullDirPath)
+}
+
+// Root delegates.
+func (f FakeLoader) Root() string {
+	return f.delegate.Root()
+}
+
+// New creates a new loader from a new root.
+func (f FakeLoader) New(newRoot string) (ifc.Loader, error) {
+	l, err := f.delegate.New(newRoot)
+	if err != nil {
+		return nil, err
+	}
+	return FakeLoader{fs: f.fs, delegate: l}, nil
+}
+
+// Load delegates.
+func (f FakeLoader) Load(location string) ([]byte, error) {
+	return f.delegate.Load(location)
+}
+
+// Cleanup delegates.
+func (f FakeLoader) Cleanup() error {
+	return f.delegate.Cleanup()
+}

api/internal/tools/tools.go

@@ -0,0 +1,9 @@
+// +build tools
+
+package tools
+
+import (
+	// These imports are all tools used in the building and testing process
+	_ "github.com/golangci/golangci-lint/cmd/golangci-lint"
+	_ "github.com/monopole/mdrip"
+)