Merge pull request #764 from sethpollack/kv

refactor kv pairs

k8sdeps/configmapandsecret/configmapfactory.go

@@ -19,7 +19,6 @@ package configmapandsecret
 
 import (
 	"fmt"
-	"path"
 	"strings"
 	"unicode/utf8"
 
@@ -27,6 +26,7 @@ import (
 	"k8s.io/api/core/v1"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/util/validation"
+	"sigs.k8s.io/kustomize/k8sdeps/kv"
 	"sigs.k8s.io/kustomize/pkg/ifc"
 	"sigs.k8s.io/kustomize/pkg/types"
 )
@@ -55,7 +55,7 @@ func (f *ConfigMapFactory) makeFreshConfigMap(
 // MakeConfigMap returns a new ConfigMap, or nil and an error.
 func (f *ConfigMapFactory) MakeConfigMap(
 	args *types.ConfigMapArgs, options *types.GeneratorOptions) (*corev1.ConfigMap, error) {
-	var all []kvPair
+	var all []kv.KVPair
 	var err error
 	cm := f.makeFreshConfigMap(args)
 
@@ -82,7 +82,7 @@ func (f *ConfigMapFactory) MakeConfigMap(
 	all = append(all, pairs...)
 
 	for _, kv := range all {
-		err = addKvToConfigMap(cm, kv.key, kv.value)
+		err = addKvToConfigMap(cm, kv.Key, kv.Value)
 		if err != nil {
 			return nil, err
 		}
@@ -94,45 +94,6 @@ func (f *ConfigMapFactory) MakeConfigMap(
 	return cm, nil
 }
 
-func keyValuesFromLiteralSources(sources []string) ([]kvPair, error) {
-	var kvs []kvPair
-	for _, s := range sources {
-		k, v, err := parseLiteralSource(s)
-		if err != nil {
-			return nil, err
-		}
-		kvs = append(kvs, kvPair{key: k, value: v})
-	}
-	return kvs, nil
-}
-
-func keyValuesFromFileSources(ldr ifc.Loader, sources []string) ([]kvPair, error) {
-	var kvs []kvPair
-	for _, s := range sources {
-		k, fPath, err := parseFileSource(s)
-		if err != nil {
-			return nil, err
-		}
-		content, err := ldr.Load(fPath)
-		if err != nil {
-			return nil, err
-		}
-		kvs = append(kvs, kvPair{key: k, value: string(content)})
-	}
-	return kvs, nil
-}
-
-func keyValuesFromEnvFile(l ifc.Loader, path string) ([]kvPair, error) {
-	if path == "" {
-		return nil, nil
-	}
-	content, err := l.Load(path)
-	if err != nil {
-		return nil, err
-	}
-	return keyValuesFromLines(content)
-}
-
 // addKvToConfigMap adds the given key and data to the given config map.
 // Error if key invalid, or already exists.
 func addKvToConfigMap(configMap *v1.ConfigMap, keyName, data string) error {
@@ -163,44 +124,3 @@ func addKvToConfigMap(configMap *v1.ConfigMap, keyName, data string) error {
 	configMap.BinaryData[keyName] = []byte(data)
 	return nil
 }
-
-// parseFileSource parses the source given.
-//
-//  Acceptable formats include:
-//   1.  source-path: the basename will become the key name
-//   2.  source-name=source-path: the source-name will become the key name and
-//       source-path is the path to the key file.
-//
-// Key names cannot include '='.
-func parseFileSource(source string) (keyName, filePath string, err error) {
-	numSeparators := strings.Count(source, "=")
-	switch {
-	case numSeparators == 0:
-		return path.Base(source), source, nil
-	case numSeparators == 1 && strings.HasPrefix(source, "="):
-		return "", "", fmt.Errorf("key name for file path %v missing", strings.TrimPrefix(source, "="))
-	case numSeparators == 1 && strings.HasSuffix(source, "="):
-		return "", "", fmt.Errorf("file path for key name %v missing", strings.TrimSuffix(source, "="))
-	case numSeparators > 1:
-		return "", "", errors.New("key names or file paths cannot contain '='")
-	default:
-		components := strings.Split(source, "=")
-		return components[0], components[1], nil
-	}
-}
-
-// parseLiteralSource parses the source key=val pair into its component pieces.
-// This functionality is distinguished from strings.SplitN(source, "=", 2) since
-// it returns an error in the case of empty keys, values, or a missing equals sign.
-func parseLiteralSource(source string) (keyName, value string, err error) {
-	// leading equal is invalid
-	if strings.Index(source, "=") == 0 {
-		return "", "", fmt.Errorf("invalid literal source %v, expected key=value", source)
-	}
-	// split after the first equal (so values can have the = character)
-	items := strings.SplitN(source, "=", 2)
-	if len(items) != 2 {
-		return "", "", fmt.Errorf("invalid literal source %v, expected key=value", source)
-	}
-	return items[0], strings.Trim(items[1], "\"'"), nil
-}

k8sdeps/configmapandsecret/kv.go

@@ -1,5 +1,5 @@
 /*
-Copyright 2018 The Kubernetes Authors.
+Copyright 2019 The Kubernetes Authors.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -17,86 +17,91 @@ limitations under the License.
 package configmapandsecret
 
 import (
-	"bufio"
-	"bytes"
 	"fmt"
-	"os"
+	"path"
 	"strings"
-	"unicode"
-	"unicode/utf8"
 
-	"k8s.io/apimachinery/pkg/util/validation"
+	"github.com/pkg/errors"
+	"sigs.k8s.io/kustomize/k8sdeps/kv"
+	"sigs.k8s.io/kustomize/pkg/ifc"
 )
 
-// kvPair represents a key value pair.
+func keyValuesFromLiteralSources(sources []string) ([]kv.KVPair, error) {
-type kvPair struct {
+	var kvs []kv.KVPair
-	key   string
+	for _, s := range sources {
-	value string
+		k, v, err := parseLiteralSource(s)
-}
-
-var utf8bom = []byte{0xEF, 0xBB, 0xBF}
-
-// keyValuesFromLines parses given content in to a list of key-value pairs.
-func keyValuesFromLines(content []byte) ([]kvPair, error) {
-	var kvs []kvPair
-
-	scanner := bufio.NewScanner(bytes.NewReader(content))
-	currentLine := 0
-	for scanner.Scan() {
-		// Process the current line, retrieving a key/value pair if
-		// possible.
-		scannedBytes := scanner.Bytes()
-		kv, err := kvFromLine(scannedBytes, currentLine)
 		if err != nil {
 			return nil, err
 		}
-		currentLine++
+		kvs = append(kvs, kv.KVPair{Key: k, Value: v})
-
-		if len(kv.key) == 0 {
-			// no key means line was empty or a comment
-			continue
-		}
-
-		kvs = append(kvs, kv)
 	}
 	return kvs, nil
 }
 
-// kvFromLine returns a kv with blank key if the line is empty or a comment.
+func keyValuesFromFileSources(ldr ifc.Loader, sources []string) ([]kv.KVPair, error) {
-// The value will be retrieved from the environment if necessary.
+	var kvs []kv.KVPair
-func kvFromLine(line []byte, currentLine int) (kvPair, error) {
+	for _, s := range sources {
-	kv := kvPair{}
+		k, fPath, err := parseFileSource(s)
-
+		if err != nil {
-	if !utf8.Valid(line) {
+			return nil, err
-		return kv, fmt.Errorf("line %d has invalid utf8 bytes : %v", line, string(line))
+		}
+		content, err := ldr.Load(fPath)
+		if err != nil {
+			return nil, err
+		}
+		kvs = append(kvs, kv.KVPair{Key: k, Value: string(content)})
 	}
-
+	return kvs, nil
-	// We trim UTF8 BOM from the first line of the file but no others
+}
-	if currentLine == 0 {
+
-		line = bytes.TrimPrefix(line, utf8bom)
+func keyValuesFromEnvFile(l ifc.Loader, path string) ([]kv.KVPair, error) {
-	}
+	if path == "" {
-
+		return nil, nil
-	// trim the line from all leading whitespace first
+	}
-	line = bytes.TrimLeftFunc(line, unicode.IsSpace)
+	content, err := l.Load(path)
-
+	if err != nil {
-	// If the line is empty or a comment, we return a blank key/value pair.
+		return nil, err
-	if len(line) == 0 || line[0] == '#' {
+	}
-		return kv, nil
+	return kv.KeyValuesFromLines(content)
-	}
+}
-
+
-	data := strings.SplitN(string(line), "=", 2)
+// parseFileSource parses the source given.
-	key := data[0]
+//
-	if errs := validation.IsEnvVarName(key); len(errs) != 0 {
+//  Acceptable formats include:
-		return kv, fmt.Errorf("%q is not a valid key name: %s", key, strings.Join(errs, ";"))
+//   1.  source-path: the basename will become the key name
-	}
+//   2.  source-name=source-path: the source-name will become the key name and
-
+//       source-path is the path to the key file.
-	if len(data) == 2 {
+//
-		kv.value = data[1]
+// Key names cannot include '='.
-	} else {
+func parseFileSource(source string) (keyName, filePath string, err error) {
-		// No value (no `=` in the line) is a signal to obtain the value
+	numSeparators := strings.Count(source, "=")
-		// from the environment.
+	switch {
-		kv.value = os.Getenv(key)
+	case numSeparators == 0:
-	}
+		return path.Base(source), source, nil
-	kv.key = key
+	case numSeparators == 1 && strings.HasPrefix(source, "="):
-	return kv, nil
+		return "", "", fmt.Errorf("key name for file path %v missing", strings.TrimPrefix(source, "="))
+	case numSeparators == 1 && strings.HasSuffix(source, "="):
+		return "", "", fmt.Errorf("file path for key name %v missing", strings.TrimSuffix(source, "="))
+	case numSeparators > 1:
+		return "", "", errors.New("key names or file paths cannot contain '='")
+	default:
+		components := strings.Split(source, "=")
+		return components[0], components[1], nil
+	}
+}
+
+// parseLiteralSource parses the source key=val pair into its component pieces.
+// This functionality is distinguished from strings.SplitN(source, "=", 2) since
+// it returns an error in the case of empty keys, values, or a missing equals sign.
+func parseLiteralSource(source string) (keyName, value string, err error) {
+	// leading equal is invalid
+	if strings.Index(source, "=") == 0 {
+		return "", "", fmt.Errorf("invalid literal source %v, expected key=value", source)
+	}
+	// split after the first equal (so values can have the = character)
+	items := strings.SplitN(source, "=", 2)
+	if len(items) != 2 {
+		return "", "", fmt.Errorf("invalid literal source %v, expected key=value", source)
+	}
+	return items[0], strings.Trim(items[1], "\"'"), nil
 }

k8sdeps/configmapandsecret/kv_test.go

@@ -1,5 +1,5 @@
 /*
-Copyright 2018 The Kubernetes Authors.
+Copyright 2019 The Kubernetes Authors.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -19,50 +19,39 @@ package configmapandsecret
 import (
 	"reflect"
 	"testing"
+
+	"sigs.k8s.io/kustomize/k8sdeps/kv"
+	"sigs.k8s.io/kustomize/pkg/fs"
+	"sigs.k8s.io/kustomize/pkg/loader"
 )
 
-func TestKeyValuesFromLines(t *testing.T) {
+func TestKeyValuesFromFileSources(t *testing.T) {
 	tests := []struct {
-		desc          string
+		description string
-		content       string
+		sources     []string
-		expectedPairs []kvPair
+		expected    []kv.KVPair
-		expectedErr   bool
 	}{
 		{
-			desc: "valid kv content parse",
+			description: "create kvs from file sources",
-			content: `
+			sources:     []string{"files/app-init.ini"},
-		k1=v1
+			expected: []kv.KVPair{
-		k2=v2
+				{
-		`,
+					Key:   "app-init.ini",
-			expectedPairs: []kvPair{
+					Value: "FOO=bar",
-				{key: "k1", value: "v1"},
+				},
-				{key: "k2", value: "v2"},
 			},
-			expectedErr: false,
 		},
-		{
-			desc: "content with comments",
-			content: `
-		k1=v1
-		#k2=v2
-		`,
-			expectedPairs: []kvPair{
-				{key: "k1", value: "v1"},
-			},
-			expectedErr: false,
-		},
-		// TODO: add negative testcases
 	}
 
-	for _, test := range tests {
+	fSys := fs.MakeFakeFS()
-		pairs, err := keyValuesFromLines([]byte(test.content))
+	fSys.WriteFile("/files/app-init.ini", []byte("FOO=bar"))
-		if test.expectedErr && err == nil {
+	for _, tc := range tests {
-			t.Fatalf("%s should not return error", test.desc)
+		kvs, err := keyValuesFromFileSources(loader.NewFileLoaderAtRoot(fSys), tc.sources)
+		if err != nil {
+			t.Fatalf("unexpected error: %v", err)
 		}
-
+		if !reflect.DeepEqual(kvs, tc.expected) {
-		if !reflect.DeepEqual(pairs, test.expectedPairs) {
+			t.Fatalf("in testcase: %q updated:\n%#v\ndoesn't match expected:\n%#v\n", tc.description, kvs, tc.expected)
-			t.Errorf("%s should succeed, got:%v exptected:%v", test.desc, pairs, test.expectedPairs)
 		}
-
 	}
 }

k8sdeps/configmapandsecret/secretfactory.go

@@ -23,6 +23,7 @@ import (
 	"github.com/pkg/errors"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/util/validation"
+	"sigs.k8s.io/kustomize/k8sdeps/kv"
 	"sigs.k8s.io/kustomize/pkg/ifc"
 	"sigs.k8s.io/kustomize/pkg/types"
 )
@@ -53,7 +54,7 @@ func (f *SecretFactory) makeFreshSecret(args *types.SecretArgs) *corev1.Secret {
 
 // MakeSecret returns a new secret.
 func (f *SecretFactory) MakeSecret(args *types.SecretArgs, options *types.GeneratorOptions) (*corev1.Secret, error) {
-	var all []kvPair
+	var all []kv.KVPair
 	var err error
 	s := f.makeFreshSecret(args)
 
@@ -80,7 +81,7 @@ func (f *SecretFactory) MakeSecret(args *types.SecretArgs, options *types.Genera
 	all = append(all, pairs...)
 
 	for _, kv := range all {
-		err = addKvToSecret(s, kv.key, kv.value)
+		err = addKvToSecret(s, kv.Key, kv.Value)
 		if err != nil {
 			return nil, err
 		}

k8sdeps/kv/kv.go

@@ -0,0 +1,101 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package kv
+
+import (
+	"bufio"
+	"bytes"
+	"fmt"
+	"os"
+	"strings"
+	"unicode"
+	"unicode/utf8"
+
+	"k8s.io/apimachinery/pkg/util/validation"
+)
+
+type KVPair struct {
+	Key   string
+	Value string
+}
+
+var utf8bom = []byte{0xEF, 0xBB, 0xBF}
+
+// KeyValuesFromLines parses given content in to a list of key-value pairs.
+func KeyValuesFromLines(content []byte) ([]KVPair, error) {
+	var kvs []KVPair
+
+	scanner := bufio.NewScanner(bytes.NewReader(content))
+	currentLine := 0
+	for scanner.Scan() {
+		// Process the current line, retrieving a key/value pair if
+		// possible.
+		scannedBytes := scanner.Bytes()
+		kv, err := KeyValuesFromLine(scannedBytes, currentLine)
+		if err != nil {
+			return nil, err
+		}
+		currentLine++
+
+		if len(kv.Key) == 0 {
+			// no key means line was empty or a comment
+			continue
+		}
+
+		kvs = append(kvs, kv)
+	}
+	return kvs, nil
+}
+
+// KeyValuesFromLine returns a kv with blank key if the line is empty or a comment.
+// The value will be retrieved from the environment if necessary.
+func KeyValuesFromLine(line []byte, currentLine int) (KVPair, error) {
+	kv := KVPair{}
+
+	if !utf8.Valid(line) {
+		return kv, fmt.Errorf("line %d has invalid utf8 bytes : %v", line, string(line))
+	}
+
+	// We trim UTF8 BOM from the first line of the file but no others
+	if currentLine == 0 {
+		line = bytes.TrimPrefix(line, utf8bom)
+	}
+
+	// trim the line from all leading whitespace first
+	line = bytes.TrimLeftFunc(line, unicode.IsSpace)
+
+	// If the line is empty or a comment, we return a blank key/value pair.
+	if len(line) == 0 || line[0] == '#' {
+		return kv, nil
+	}
+
+	data := strings.SplitN(string(line), "=", 2)
+	key := data[0]
+	if errs := validation.IsEnvVarName(key); len(errs) != 0 {
+		return kv, fmt.Errorf("%q is not a valid key name: %s", key, strings.Join(errs, ";"))
+	}
+
+	if len(data) == 2 {
+		kv.Value = data[1]
+	} else {
+		// No value (no `=` in the line) is a signal to obtain the value
+		// from the environment.
+		kv.Value = os.Getenv(key)
+	}
+	kv.Key = key
+	return kv, nil
+}

k8sdeps/kv/kv_test.go

@@ -0,0 +1,68 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package kv
+
+import (
+	"reflect"
+	"testing"
+)
+
+func TestKeyValuesFromLines(t *testing.T) {
+	tests := []struct {
+		desc          string
+		content       string
+		expectedPairs []KVPair
+		expectedErr   bool
+	}{
+		{
+			desc: "valid kv content parse",
+			content: `
+		k1=v1
+		k2=v2
+		`,
+			expectedPairs: []KVPair{
+				{Key: "k1", Value: "v1"},
+				{Key: "k2", Value: "v2"},
+			},
+			expectedErr: false,
+		},
+		{
+			desc: "content with comments",
+			content: `
+		k1=v1
+		#k2=v2
+		`,
+			expectedPairs: []KVPair{
+				{Key: "k1", Value: "v1"},
+			},
+			expectedErr: false,
+		},
+		// TODO: add negative testcases
+	}
+
+	for _, test := range tests {
+		pairs, err := KeyValuesFromLines([]byte(test.content))
+		if test.expectedErr && err == nil {
+			t.Fatalf("%s should not return error", test.desc)
+		}
+
+		if !reflect.DeepEqual(pairs, test.expectedPairs) {
+			t.Errorf("%s should succeed, got:%v exptected:%v", test.desc, pairs, test.expectedPairs)
+		}
+
+	}
+}