Merge expansion package into refvar package.

api/filters/refvar/expand.go

@@ -0,0 +1,121 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+// Package expansion provides functions find and replace $(FOO) style variables in strings.
+package refvar
+
+import (
+	"bytes"
+	"fmt"
+)
+
+const (
+	operator        = '$'
+	referenceOpener = '('
+	referenceCloser = ')'
+)
+
+// syntaxWrap returns the input string wrapped by the expansion syntax.
+func syntaxWrap(input string) string {
+	return string(operator) + string(referenceOpener) + input + string(referenceCloser)
+}
+
+// MappingFuncFor returns a mapping function for use with Expand that
+// implements the expansion semantics defined in the expansion spec; it
+// returns the input string wrapped in the expansion syntax if no mapping
+// for the input is found.
+func MappingFuncFor(
+	counts map[string]int,
+	context ...map[string]interface{}) func(string) interface{} {
+	return func(input string) interface{} {
+		for _, vars := range context {
+			val, ok := vars[input]
+			if ok {
+				counts[input]++
+				switch typedV := val.(type) {
+				case string, int64, float64, bool:
+					return typedV
+				default:
+					return syntaxWrap(input)
+				}
+			}
+		}
+		return syntaxWrap(input)
+	}
+}
+
+// Expand replaces variable references in the input string according to
+// the expansion spec using the given mapping function to resolve the
+// values of variables.
+func Expand(input string, mapping func(string) interface{}) interface{} {
+	var buf bytes.Buffer
+	checkpoint := 0
+	for cursor := 0; cursor < len(input); cursor++ {
+		if input[cursor] == operator && cursor+1 < len(input) {
+			// Copy the portion of the input string since the last
+			// checkpoint into the buffer
+			buf.WriteString(input[checkpoint:cursor])
+
+			// Attempt to read the variable name as defined by the
+			// syntax from the input string
+			read, isVar, advance := tryReadVariableName(input[cursor+1:])
+
+			if isVar {
+				// We were able to read a variable name correctly;
+				// apply the mapping to the variable name and copy the
+				// bytes into the buffer
+				mapped := mapping(read)
+				if input == syntaxWrap(read) {
+					// Preserve the type of variable
+					return mapped
+				}
+
+				// Variable is used in a middle of a string
+				buf.WriteString(fmt.Sprintf("%v", mapped))
+			} else {
+				// Not a variable name; copy the read bytes into the buffer
+				buf.WriteString(read)
+			}
+
+			// Advance the cursor in the input string to account for
+			// bytes consumed to read the variable name expression
+			cursor += advance
+
+			// Advance the checkpoint in the input string
+			checkpoint = cursor + 1
+		}
+	}
+
+	// Return the buffer and any remaining unwritten bytes in the
+	// input string.
+	return buf.String() + input[checkpoint:]
+}
+
+// tryReadVariableName attempts to read a variable name from the input
+// string and returns the content read from the input, whether that content
+// represents a variable name to perform mapping on, and the number of bytes
+// consumed in the input string.
+//
+// The input string is assumed not to contain the initial operator.
+func tryReadVariableName(input string) (string, bool, int) {
+	switch input[0] {
+	case operator:
+		// Escaped operator; return it.
+		return input[0:1], false, 1
+	case referenceOpener:
+		// Scan to expression closer
+		for i := 1; i < len(input); i++ {
+			if input[i] == referenceCloser {
+				return input[1:i], true, i + 1
+			}
+		}
+
+		// Incomplete reference; return it.
+		return string(operator) + string(referenceOpener), false, 1
+	default:
+		// Not the beginning of an expression, ie, an operator
+		// that doesn't begin an expression.  Return the operator
+		// and the first rune in the string.
+		return string(operator) + string(input[0]), false, 1
+	}
+}

api/filters/refvar/expand_test.go

@@ -0,0 +1,365 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package refvar_test
+
+import (
+	"fmt"
+	"testing"
+
+	. "sigs.k8s.io/kustomize/api/filters/refvar"
+)
+
+type expected struct {
+	count  int
+	edited string
+}
+
+func TestMapReference(t *testing.T) {
+	type env struct {
+		Name  string
+		Value interface{}
+	}
+	envs := []env{
+		{
+			Name:  "FOO",
+			Value: "bar",
+		},
+		{
+			Name:  "ZOO",
+			Value: "$(FOO)-1",
+		},
+		{
+			Name:  "BLU",
+			Value: "$(ZOO)-2",
+		},
+		{
+			Name:  "INT",
+			Value: 2,
+		},
+		{
+			Name:  "ZINT",
+			Value: "$(INT)",
+		},
+		{
+			Name:  "BOOL",
+			Value: true,
+		},
+		{
+			Name:  "ZBOOL",
+			Value: "$(BOOL)",
+		},
+	}
+
+	declaredEnv := map[string]interface{}{
+		"FOO":   "bar",
+		"ZOO":   "$(FOO)-1",
+		"BLU":   "$(ZOO)-2",
+		"INT":   "2",
+		"ZINT":  "$(INT)",
+		"BOOL":  "true",
+		"ZBOOL": "$(BOOL)",
+	}
+
+	counts := make(map[string]int)
+	mapping := MappingFuncFor(counts, declaredEnv)
+
+	for _, env := range envs {
+		declaredEnv[env.Name] = Expand(fmt.Sprintf("%v", env.Value), mapping)
+	}
+
+	expectedEnv := map[string]expected{
+		"FOO":   {count: 1, edited: "bar"},
+		"ZOO":   {count: 1, edited: "bar-1"},
+		"BLU":   {count: 0, edited: "bar-1-2"},
+		"INT":   {count: 1, edited: "2"},
+		"ZINT":  {count: 0, edited: "2"},
+		"BOOL":  {count: 1, edited: "true"},
+		"ZBOOL": {count: 0, edited: "true"},
+	}
+
+	for k, v := range expectedEnv {
+		if e, a := v, declaredEnv[k]; e.edited != a || e.count != counts[k] {
+			t.Errorf("Expected %v count=%d, got %v count=%d",
+				e.edited, e.count, a, counts[k])
+		} else {
+			delete(declaredEnv, k)
+		}
+	}
+
+	if len(declaredEnv) != 0 {
+		t.Errorf("Unexpected keys in declared env: %v", declaredEnv)
+	}
+}
+
+func TestMapping(t *testing.T) {
+	context := map[string]interface{}{
+		"VAR_A":     "A",
+		"VAR_B":     "B",
+		"VAR_C":     "C",
+		"VAR_REF":   "$(VAR_A)",
+		"VAR_EMPTY": "",
+	}
+	doExpansionTest(t, context)
+}
+
+func TestMappingDual(t *testing.T) {
+	context := map[string]interface{}{
+		"VAR_A":     "A",
+		"VAR_EMPTY": "",
+	}
+	context2 := map[string]interface{}{
+		"VAR_B":   "B",
+		"VAR_C":   "C",
+		"VAR_REF": "$(VAR_A)",
+	}
+
+	doExpansionTest(t, context, context2)
+}
+
+func doExpansionTest(t *testing.T, context ...map[string]interface{}) {
+	cases := []struct {
+		name     string
+		input    string
+		expected string
+		counts   map[string]int
+	}{
+		{
+			name:     "whole string",
+			input:    "$(VAR_A)",
+			expected: "A",
+			counts:   map[string]int{"VAR_A": 1},
+		},
+		{
+			name:     "repeat",
+			input:    "$(VAR_A)-$(VAR_A)",
+			expected: "A-A",
+			counts:   map[string]int{"VAR_A": 2},
+		},
+		{
+			name:     "multiple repeats",
+			input:    "$(VAR_A)-$(VAR_B)-$(VAR_B)-$(VAR_B)-$(VAR_A)",
+			expected: "A-B-B-B-A",
+			counts:   map[string]int{"VAR_A": 2, "VAR_B": 3},
+		},
+		{
+			name:     "beginning",
+			input:    "$(VAR_A)-1",
+			expected: "A-1",
+			counts:   map[string]int{"VAR_A": 1},
+		},
+		{
+			name:     "middle",
+			input:    "___$(VAR_B)___",
+			expected: "___B___",
+			counts:   map[string]int{"VAR_B": 1},
+		},
+		{
+			name:     "end",
+			input:    "___$(VAR_C)",
+			expected: "___C",
+			counts:   map[string]int{"VAR_C": 1},
+		},
+		{
+			name:     "compound",
+			input:    "$(VAR_A)_$(VAR_B)_$(VAR_C)",
+			expected: "A_B_C",
+			counts:   map[string]int{"VAR_A": 1, "VAR_B": 1, "VAR_C": 1},
+		},
+		{
+			name:     "escape & expand",
+			input:    "$$(VAR_B)_$(VAR_A)",
+			expected: "$(VAR_B)_A",
+			counts:   map[string]int{"VAR_A": 1},
+		},
+		{
+			name:     "compound escape",
+			input:    "$$(VAR_A)_$$(VAR_B)",
+			expected: "$(VAR_A)_$(VAR_B)",
+		},
+		{
+			name:     "mixed in escapes",
+			input:    "f000-$$VAR_A",
+			expected: "f000-$VAR_A",
+		},
+		{
+			name:     "backslash escape ignored",
+			input:    "foo\\$(VAR_C)bar",
+			expected: "foo\\Cbar",
+			counts:   map[string]int{"VAR_C": 1},
+		},
+		{
+			name:     "backslash escape ignored",
+			input:    "foo\\\\$(VAR_C)bar",
+			expected: "foo\\\\Cbar",
+			counts:   map[string]int{"VAR_C": 1},
+		},
+		{
+			name:     "lots of backslashes",
+			input:    "foo\\\\\\\\$(VAR_A)bar",
+			expected: "foo\\\\\\\\Abar",
+			counts:   map[string]int{"VAR_A": 1},
+		},
+		{
+			name:     "nested var references",
+			input:    "$(VAR_A$(VAR_B))",
+			expected: "$(VAR_A$(VAR_B))",
+		},
+		{
+			name:     "nested var references second type",
+			input:    "$(VAR_A$(VAR_B)",
+			expected: "$(VAR_A$(VAR_B)",
+		},
+		{
+			name:     "value is a reference",
+			input:    "$(VAR_REF)",
+			expected: "$(VAR_A)",
+			counts:   map[string]int{"VAR_REF": 1},
+		},
+		{
+			name:     "value is a reference x 2",
+			input:    "%%$(VAR_REF)--$(VAR_REF)%%",
+			expected: "%%$(VAR_A)--$(VAR_A)%%",
+			counts:   map[string]int{"VAR_REF": 2},
+		},
+		{
+			name:     "empty var",
+			input:    "foo$(VAR_EMPTY)bar",
+			expected: "foobar",
+			counts:   map[string]int{"VAR_EMPTY": 1},
+		},
+		{
+			name:     "unterminated expression",
+			input:    "foo$(VAR_Awhoops!",
+			expected: "foo$(VAR_Awhoops!",
+		},
+		{
+			name:     "expression without operator",
+			input:    "f00__(VAR_A)__",
+			expected: "f00__(VAR_A)__",
+		},
+		{
+			name:     "shell special vars pass through",
+			input:    "$?_boo_$!",
+			expected: "$?_boo_$!",
+		},
+		{
+			name:     "bare operators are ignored",
+			input:    "$VAR_A",
+			expected: "$VAR_A",
+		},
+		{
+			name:     "undefined vars are passed through",
+			input:    "$(VAR_DNE)",
+			expected: "$(VAR_DNE)",
+		},
+		{
+			name:     "multiple (even) operators, var undefined",
+			input:    "$$$$$$(BIG_MONEY)",
+			expected: "$$$(BIG_MONEY)",
+		},
+		{
+			name:     "multiple (even) operators, var defined",
+			input:    "$$$$$$(VAR_A)",
+			expected: "$$$(VAR_A)",
+		},
+		{
+			name:     "multiple (odd) operators, var undefined",
+			input:    "$$$$$$$(GOOD_ODDS)",
+			expected: "$$$$(GOOD_ODDS)",
+		},
+		{
+			name:     "multiple (odd) operators, var defined",
+			input:    "$$$$$$$(VAR_A)",
+			expected: "$$$A",
+			counts:   map[string]int{"VAR_A": 1},
+		},
+		{
+			name:     "missing open expression",
+			input:    "$VAR_A)",
+			expected: "$VAR_A)",
+		},
+		{
+			name:     "shell syntax ignored",
+			input:    "${VAR_A}",
+			expected: "${VAR_A}",
+		},
+		{
+			name:     "trailing incomplete expression not consumed",
+			input:    "$(VAR_B)_______$(A",
+			expected: "B_______$(A",
+			counts:   map[string]int{"VAR_B": 1},
+		},
+		{
+			name:     "trailing incomplete expression, no content, is not consumed",
+			input:    "$(VAR_C)_______$(",
+			expected: "C_______$(",
+			counts:   map[string]int{"VAR_C": 1},
+		},
+		{
+			name:     "operator at end of input string is preserved",
+			input:    "$(VAR_A)foobarzab$",
+			expected: "Afoobarzab$",
+			counts:   map[string]int{"VAR_A": 1},
+		},
+		{
+			name:     "shell escaped incomplete expr",
+			input:    "foo-\\$(VAR_A",
+			expected: "foo-\\$(VAR_A",
+		},
+		{
+			name:     "lots of $( in middle",
+			input:    "--$($($($($--",
+			expected: "--$($($($($--",
+		},
+		{
+			name:     "lots of $( in beginning",
+			input:    "$($($($($--foo$(",
+			expected: "$($($($($--foo$(",
+		},
+		{
+			name:     "lots of $( at end",
+			input:    "foo0--$($($($(",
+			expected: "foo0--$($($($(",
+		},
+		{
+			name:     "escaped operators in variable names are not escaped",
+			input:    "$(foo$$var)",
+			expected: "$(foo$$var)",
+		},
+		{
+			name:     "newline not expanded",
+			input:    "\n",
+			expected: "\n",
+		},
+	}
+
+	for _, tc := range cases {
+		counts := make(map[string]int)
+		mapping := MappingFuncFor(counts, context...)
+		expanded := Expand(fmt.Sprintf("%v", tc.input), mapping)
+		if e, a := tc.expected, expanded; e != a {
+			t.Errorf("%v: expected %q, got %q", tc.name, e, a)
+		}
+		if len(counts) != len(tc.counts) {
+			t.Errorf("%v: len(counts)=%d != len(tc.counts)=%d",
+				tc.name, len(counts), len(tc.counts))
+		}
+		if len(tc.counts) > 0 {
+			for k, expectedCount := range tc.counts {
+				c, ok := counts[k]
+				if ok {
+					if c != expectedCount {
+						t.Errorf(
+							"%v: k=%s, expected count %d, got %d",
+							tc.name, k, expectedCount, c)
+					}
+				} else {
+					t.Errorf(
+						"%v: k=%s, expected count %d, got zero",
+						tc.name, k, expectedCount)
+				}
+			}
+		}
+	}
+}

api/filters/refvar/refvar.go

@@ -8,8 +8,6 @@ import (
 	"sigs.k8s.io/kustomize/api/types"
 	"sigs.k8s.io/kustomize/kyaml/kio"
 	"sigs.k8s.io/kustomize/kyaml/yaml"
-
-	expansion2 "sigs.k8s.io/kustomize/api/internal/accumulator/expansion"
 )
 
 // Filter updates $(VAR) style variables with values.
@@ -69,7 +67,7 @@ func (f Filter) setScalar(node *yaml.RNode) error {
 	if !yaml.IsYNodeString(node.YNode()) {
 		return nil
 	}
-	v := expansion2.Expand(node.YNode().Value, f.MappingFunc)
+	v := Expand(node.YNode().Value, f.MappingFunc)
 	updateNodeValue(node.YNode(), v)
 	return nil
 }
@@ -85,7 +83,7 @@ func (f Filter) setMap(node *yaml.RNode) error {
 		if !yaml.IsYNodeString(contents[i+1]) {
 			continue
 		}
-		newValue := expansion2.Expand(contents[i+1].Value, f.MappingFunc)
+		newValue := Expand(contents[i+1].Value, f.MappingFunc)
 		updateNodeValue(contents[i+1], newValue)
 	}
 	return nil
@@ -96,7 +94,7 @@ func (f Filter) setSeq(node *yaml.RNode) error {
 		if !yaml.IsYNodeString(item) {
 			return fmt.Errorf("invalid value type expect a string")
 		}
-		newValue := expansion2.Expand(item.Value, f.MappingFunc)
+		newValue := Expand(item.Value, f.MappingFunc)
 		updateNodeValue(item, newValue)
 	}
 	return nil

api/filters/refvar/refvar_test.go

@@ -1,11 +1,11 @@
-package refvar
+package refvar_test
 
 import (
 	"strings"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
-	expansion2 "sigs.k8s.io/kustomize/api/internal/accumulator/expansion"
+	. "sigs.k8s.io/kustomize/api/filters/refvar"
 	filtertest_test "sigs.k8s.io/kustomize/api/testutils/filtertest"
 	"sigs.k8s.io/kustomize/api/types"
 	"sigs.k8s.io/kustomize/kyaml/yaml"
@@ -35,7 +35,7 @@ metadata:
 spec:
   replicas: 5`,
 			filter: Filter{
-				MappingFunc: expansion2.MappingFuncFor(replacementCounts, map[string]interface{}{
+				MappingFunc: MappingFuncFor(replacementCounts, map[string]interface{}{
 					"VAR": int64(5),
 				}),
 				FieldSpec: types.FieldSpec{Path: "spec/replicas"},
@@ -57,7 +57,7 @@ metadata:
 spec:
   replicas: 1`,
 			filter: Filter{
-				MappingFunc: expansion2.MappingFuncFor(replacementCounts, map[string]interface{}{
+				MappingFunc: MappingFuncFor(replacementCounts, map[string]interface{}{
 					"VAR": int64(5),
 				}),
 				FieldSpec: types.FieldSpec{Path: "spec/replicas"},
@@ -79,7 +79,7 @@ metadata:
 spec:
   replicas: 1`,
 			filter: Filter{
-				MappingFunc: expansion2.MappingFuncFor(replacementCounts, map[string]interface{}{
+				MappingFunc: MappingFuncFor(replacementCounts, map[string]interface{}{
 					"VAR": int64(5),
 				}),
 				FieldSpec: types.FieldSpec{Path: "a/b/c"},
@@ -111,7 +111,7 @@ data:
 - false
 - 1.23`,
 			filter: Filter{
-				MappingFunc: expansion2.MappingFuncFor(replacementCounts, map[string]interface{}{
+				MappingFunc: MappingFuncFor(replacementCounts, map[string]interface{}{
 					"FOO":   "foo",
 					"BAR":   "bar",
 					"BOOL":  false,
@@ -142,7 +142,7 @@ data:
   BAZ: $(BAZ)
   PLUS: foo+bar`,
 			filter: Filter{
-				MappingFunc: expansion2.MappingFuncFor(replacementCounts, map[string]interface{}{
+				MappingFunc: MappingFuncFor(replacementCounts, map[string]interface{}{
 					"FOO": "foo",
 					"BAR": "bar",
 				}),
@@ -181,7 +181,7 @@ data:
     SLICE:
     - $(FOO)`,
 			filter: Filter{
-				MappingFunc: expansion2.MappingFuncFor(replacementCounts, map[string]interface{}{
+				MappingFunc: MappingFuncFor(replacementCounts, map[string]interface{}{
 					"FOO": "foo",
 					"BAR": "bar",
 				}),
@@ -204,7 +204,7 @@ metadata:
 data:
   FOO: null`,
 			filter: Filter{
-				MappingFunc: expansion2.MappingFuncFor(replacementCounts, map[string]interface{}{}),
+				MappingFunc: MappingFuncFor(replacementCounts, map[string]interface{}{}),
 				FieldSpec:   types.FieldSpec{Path: "data/FOO"},
 			},
 		},
@@ -250,7 +250,7 @@ data:
   - false
 ' at path 'data/slice': invalid value type expect a string`,
 			filter: Filter{
-				MappingFunc: expansion2.MappingFuncFor(replacementCounts, map[string]interface{}{
+				MappingFunc: MappingFuncFor(replacementCounts, map[string]interface{}{
 					"VAR": int64(5),
 				}),
 				FieldSpec: types.FieldSpec{Path: "data/slice"},
@@ -274,7 +274,7 @@ data:
   1: str
 ' at path 'data': invalid map key: value='1', tag='` + yaml.NodeTagInt + `'`,
 			filter: Filter{
-				MappingFunc: expansion2.MappingFuncFor(replacementCounts, map[string]interface{}{
+				MappingFunc: MappingFuncFor(replacementCounts, map[string]interface{}{
 					"VAR": int64(5),
 				}),
 				FieldSpec: types.FieldSpec{Path: "data"},