diff --git a/k8sdeps/configmapandsecret/configmapfactory.go b/k8sdeps/configmapandsecret/configmapfactory.go index f0d8f4962..7dabad4b5 100644 --- a/k8sdeps/configmapandsecret/configmapfactory.go +++ b/k8sdeps/configmapandsecret/configmapfactory.go @@ -19,7 +19,6 @@ package configmapandsecret import ( "fmt" - "path" "strings" "unicode/utf8" @@ -27,6 +26,7 @@ import ( "k8s.io/api/core/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/validation" + "sigs.k8s.io/kustomize/k8sdeps/kv" "sigs.k8s.io/kustomize/pkg/ifc" "sigs.k8s.io/kustomize/pkg/types" ) @@ -55,7 +55,7 @@ func (f *ConfigMapFactory) makeFreshConfigMap( // MakeConfigMap returns a new ConfigMap, or nil and an error. func (f *ConfigMapFactory) MakeConfigMap( args *types.ConfigMapArgs, options *types.GeneratorOptions) (*corev1.ConfigMap, error) { - var all []kvPair + var all []kv.KVPair var err error cm := f.makeFreshConfigMap(args) @@ -82,7 +82,7 @@ func (f *ConfigMapFactory) MakeConfigMap( all = append(all, pairs...) for _, kv := range all { - err = addKvToConfigMap(cm, kv.key, kv.value) + err = addKvToConfigMap(cm, kv.Key, kv.Value) if err != nil { return nil, err } @@ -94,45 +94,6 @@ func (f *ConfigMapFactory) MakeConfigMap( return cm, nil } -func keyValuesFromLiteralSources(sources []string) ([]kvPair, error) { - var kvs []kvPair - for _, s := range sources { - k, v, err := parseLiteralSource(s) - if err != nil { - return nil, err - } - kvs = append(kvs, kvPair{key: k, value: v}) - } - return kvs, nil -} - -func keyValuesFromFileSources(ldr ifc.Loader, sources []string) ([]kvPair, error) { - var kvs []kvPair - for _, s := range sources { - k, fPath, err := parseFileSource(s) - if err != nil { - return nil, err - } - content, err := ldr.Load(fPath) - if err != nil { - return nil, err - } - kvs = append(kvs, kvPair{key: k, value: string(content)}) - } - return kvs, nil -} - -func keyValuesFromEnvFile(l ifc.Loader, path string) ([]kvPair, error) { - if path == "" { - return nil, nil - } - content, err := l.Load(path) - if err != nil { - return nil, err - } - return keyValuesFromLines(content) -} - // addKvToConfigMap adds the given key and data to the given config map. // Error if key invalid, or already exists. func addKvToConfigMap(configMap *v1.ConfigMap, keyName, data string) error { @@ -163,44 +124,3 @@ func addKvToConfigMap(configMap *v1.ConfigMap, keyName, data string) error { configMap.BinaryData[keyName] = []byte(data) return nil } - -// parseFileSource parses the source given. -// -// Acceptable formats include: -// 1. source-path: the basename will become the key name -// 2. source-name=source-path: the source-name will become the key name and -// source-path is the path to the key file. -// -// Key names cannot include '='. -func parseFileSource(source string) (keyName, filePath string, err error) { - numSeparators := strings.Count(source, "=") - switch { - case numSeparators == 0: - return path.Base(source), source, nil - case numSeparators == 1 && strings.HasPrefix(source, "="): - return "", "", fmt.Errorf("key name for file path %v missing", strings.TrimPrefix(source, "=")) - case numSeparators == 1 && strings.HasSuffix(source, "="): - return "", "", fmt.Errorf("file path for key name %v missing", strings.TrimSuffix(source, "=")) - case numSeparators > 1: - return "", "", errors.New("key names or file paths cannot contain '='") - default: - components := strings.Split(source, "=") - return components[0], components[1], nil - } -} - -// parseLiteralSource parses the source key=val pair into its component pieces. -// This functionality is distinguished from strings.SplitN(source, "=", 2) since -// it returns an error in the case of empty keys, values, or a missing equals sign. -func parseLiteralSource(source string) (keyName, value string, err error) { - // leading equal is invalid - if strings.Index(source, "=") == 0 { - return "", "", fmt.Errorf("invalid literal source %v, expected key=value", source) - } - // split after the first equal (so values can have the = character) - items := strings.SplitN(source, "=", 2) - if len(items) != 2 { - return "", "", fmt.Errorf("invalid literal source %v, expected key=value", source) - } - return items[0], strings.Trim(items[1], "\"'"), nil -} diff --git a/k8sdeps/configmapandsecret/kv.go b/k8sdeps/configmapandsecret/kv.go index 6d6c99249..1ad4264e7 100644 --- a/k8sdeps/configmapandsecret/kv.go +++ b/k8sdeps/configmapandsecret/kv.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Kubernetes Authors. +Copyright 2019 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -17,86 +17,91 @@ limitations under the License. package configmapandsecret import ( - "bufio" - "bytes" "fmt" - "os" + "path" "strings" - "unicode" - "unicode/utf8" - "k8s.io/apimachinery/pkg/util/validation" + "github.com/pkg/errors" + "sigs.k8s.io/kustomize/k8sdeps/kv" + "sigs.k8s.io/kustomize/pkg/ifc" ) -// kvPair represents a key value pair. -type kvPair struct { - key string - value string -} - -var utf8bom = []byte{0xEF, 0xBB, 0xBF} - -// keyValuesFromLines parses given content in to a list of key-value pairs. -func keyValuesFromLines(content []byte) ([]kvPair, error) { - var kvs []kvPair - - scanner := bufio.NewScanner(bytes.NewReader(content)) - currentLine := 0 - for scanner.Scan() { - // Process the current line, retrieving a key/value pair if - // possible. - scannedBytes := scanner.Bytes() - kv, err := kvFromLine(scannedBytes, currentLine) +func keyValuesFromLiteralSources(sources []string) ([]kv.KVPair, error) { + var kvs []kv.KVPair + for _, s := range sources { + k, v, err := parseLiteralSource(s) if err != nil { return nil, err } - currentLine++ - - if len(kv.key) == 0 { - // no key means line was empty or a comment - continue - } - - kvs = append(kvs, kv) + kvs = append(kvs, kv.KVPair{Key: k, Value: v}) } return kvs, nil } -// kvFromLine returns a kv with blank key if the line is empty or a comment. -// The value will be retrieved from the environment if necessary. -func kvFromLine(line []byte, currentLine int) (kvPair, error) { - kv := kvPair{} - - if !utf8.Valid(line) { - return kv, fmt.Errorf("line %d has invalid utf8 bytes : %v", line, string(line)) +func keyValuesFromFileSources(ldr ifc.Loader, sources []string) ([]kv.KVPair, error) { + var kvs []kv.KVPair + for _, s := range sources { + k, fPath, err := parseFileSource(s) + if err != nil { + return nil, err + } + content, err := ldr.Load(fPath) + if err != nil { + return nil, err + } + kvs = append(kvs, kv.KVPair{Key: k, Value: string(content)}) } - - // We trim UTF8 BOM from the first line of the file but no others - if currentLine == 0 { - line = bytes.TrimPrefix(line, utf8bom) - } - - // trim the line from all leading whitespace first - line = bytes.TrimLeftFunc(line, unicode.IsSpace) - - // If the line is empty or a comment, we return a blank key/value pair. - if len(line) == 0 || line[0] == '#' { - return kv, nil - } - - data := strings.SplitN(string(line), "=", 2) - key := data[0] - if errs := validation.IsEnvVarName(key); len(errs) != 0 { - return kv, fmt.Errorf("%q is not a valid key name: %s", key, strings.Join(errs, ";")) - } - - if len(data) == 2 { - kv.value = data[1] - } else { - // No value (no `=` in the line) is a signal to obtain the value - // from the environment. - kv.value = os.Getenv(key) - } - kv.key = key - return kv, nil + return kvs, nil +} + +func keyValuesFromEnvFile(l ifc.Loader, path string) ([]kv.KVPair, error) { + if path == "" { + return nil, nil + } + content, err := l.Load(path) + if err != nil { + return nil, err + } + return kv.KeyValuesFromLines(content) +} + +// parseFileSource parses the source given. +// +// Acceptable formats include: +// 1. source-path: the basename will become the key name +// 2. source-name=source-path: the source-name will become the key name and +// source-path is the path to the key file. +// +// Key names cannot include '='. +func parseFileSource(source string) (keyName, filePath string, err error) { + numSeparators := strings.Count(source, "=") + switch { + case numSeparators == 0: + return path.Base(source), source, nil + case numSeparators == 1 && strings.HasPrefix(source, "="): + return "", "", fmt.Errorf("key name for file path %v missing", strings.TrimPrefix(source, "=")) + case numSeparators == 1 && strings.HasSuffix(source, "="): + return "", "", fmt.Errorf("file path for key name %v missing", strings.TrimSuffix(source, "=")) + case numSeparators > 1: + return "", "", errors.New("key names or file paths cannot contain '='") + default: + components := strings.Split(source, "=") + return components[0], components[1], nil + } +} + +// parseLiteralSource parses the source key=val pair into its component pieces. +// This functionality is distinguished from strings.SplitN(source, "=", 2) since +// it returns an error in the case of empty keys, values, or a missing equals sign. +func parseLiteralSource(source string) (keyName, value string, err error) { + // leading equal is invalid + if strings.Index(source, "=") == 0 { + return "", "", fmt.Errorf("invalid literal source %v, expected key=value", source) + } + // split after the first equal (so values can have the = character) + items := strings.SplitN(source, "=", 2) + if len(items) != 2 { + return "", "", fmt.Errorf("invalid literal source %v, expected key=value", source) + } + return items[0], strings.Trim(items[1], "\"'"), nil } diff --git a/k8sdeps/configmapandsecret/kv_test.go b/k8sdeps/configmapandsecret/kv_test.go index 939e231f5..0b7fe5c37 100644 --- a/k8sdeps/configmapandsecret/kv_test.go +++ b/k8sdeps/configmapandsecret/kv_test.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Kubernetes Authors. +Copyright 2019 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -19,50 +19,39 @@ package configmapandsecret import ( "reflect" "testing" + + "sigs.k8s.io/kustomize/k8sdeps/kv" + "sigs.k8s.io/kustomize/pkg/fs" + "sigs.k8s.io/kustomize/pkg/loader" ) -func TestKeyValuesFromLines(t *testing.T) { +func TestKeyValuesFromFileSources(t *testing.T) { tests := []struct { - desc string - content string - expectedPairs []kvPair - expectedErr bool + description string + sources []string + expected []kv.KVPair }{ { - desc: "valid kv content parse", - content: ` - k1=v1 - k2=v2 - `, - expectedPairs: []kvPair{ - {key: "k1", value: "v1"}, - {key: "k2", value: "v2"}, + description: "create kvs from file sources", + sources: []string{"files/app-init.ini"}, + expected: []kv.KVPair{ + { + Key: "app-init.ini", + Value: "FOO=bar", + }, }, - expectedErr: false, }, - { - desc: "content with comments", - content: ` - k1=v1 - #k2=v2 - `, - expectedPairs: []kvPair{ - {key: "k1", value: "v1"}, - }, - expectedErr: false, - }, - // TODO: add negative testcases } - for _, test := range tests { - pairs, err := keyValuesFromLines([]byte(test.content)) - if test.expectedErr && err == nil { - t.Fatalf("%s should not return error", test.desc) + fSys := fs.MakeFakeFS() + fSys.WriteFile("/files/app-init.ini", []byte("FOO=bar")) + for _, tc := range tests { + kvs, err := keyValuesFromFileSources(loader.NewFileLoaderAtRoot(fSys), tc.sources) + if err != nil { + t.Fatalf("unexpected error: %v", err) } - - if !reflect.DeepEqual(pairs, test.expectedPairs) { - t.Errorf("%s should succeed, got:%v exptected:%v", test.desc, pairs, test.expectedPairs) + if !reflect.DeepEqual(kvs, tc.expected) { + t.Fatalf("in testcase: %q updated:\n%#v\ndoesn't match expected:\n%#v\n", tc.description, kvs, tc.expected) } - } } diff --git a/k8sdeps/configmapandsecret/secretfactory.go b/k8sdeps/configmapandsecret/secretfactory.go index 1303c812c..cc5d68b42 100644 --- a/k8sdeps/configmapandsecret/secretfactory.go +++ b/k8sdeps/configmapandsecret/secretfactory.go @@ -23,6 +23,7 @@ import ( "github.com/pkg/errors" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/validation" + "sigs.k8s.io/kustomize/k8sdeps/kv" "sigs.k8s.io/kustomize/pkg/ifc" "sigs.k8s.io/kustomize/pkg/types" ) @@ -53,7 +54,7 @@ func (f *SecretFactory) makeFreshSecret(args *types.SecretArgs) *corev1.Secret { // MakeSecret returns a new secret. func (f *SecretFactory) MakeSecret(args *types.SecretArgs, options *types.GeneratorOptions) (*corev1.Secret, error) { - var all []kvPair + var all []kv.KVPair var err error s := f.makeFreshSecret(args) @@ -80,7 +81,7 @@ func (f *SecretFactory) MakeSecret(args *types.SecretArgs, options *types.Genera all = append(all, pairs...) for _, kv := range all { - err = addKvToSecret(s, kv.key, kv.value) + err = addKvToSecret(s, kv.Key, kv.Value) if err != nil { return nil, err } diff --git a/k8sdeps/kv/kv.go b/k8sdeps/kv/kv.go new file mode 100644 index 000000000..36451fa68 --- /dev/null +++ b/k8sdeps/kv/kv.go @@ -0,0 +1,101 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kv + +import ( + "bufio" + "bytes" + "fmt" + "os" + "strings" + "unicode" + "unicode/utf8" + + "k8s.io/apimachinery/pkg/util/validation" +) + +type KVPair struct { + Key string + Value string +} + +var utf8bom = []byte{0xEF, 0xBB, 0xBF} + +// KeyValuesFromLines parses given content in to a list of key-value pairs. +func KeyValuesFromLines(content []byte) ([]KVPair, error) { + var kvs []KVPair + + scanner := bufio.NewScanner(bytes.NewReader(content)) + currentLine := 0 + for scanner.Scan() { + // Process the current line, retrieving a key/value pair if + // possible. + scannedBytes := scanner.Bytes() + kv, err := KeyValuesFromLine(scannedBytes, currentLine) + if err != nil { + return nil, err + } + currentLine++ + + if len(kv.Key) == 0 { + // no key means line was empty or a comment + continue + } + + kvs = append(kvs, kv) + } + return kvs, nil +} + +// KeyValuesFromLine returns a kv with blank key if the line is empty or a comment. +// The value will be retrieved from the environment if necessary. +func KeyValuesFromLine(line []byte, currentLine int) (KVPair, error) { + kv := KVPair{} + + if !utf8.Valid(line) { + return kv, fmt.Errorf("line %d has invalid utf8 bytes : %v", line, string(line)) + } + + // We trim UTF8 BOM from the first line of the file but no others + if currentLine == 0 { + line = bytes.TrimPrefix(line, utf8bom) + } + + // trim the line from all leading whitespace first + line = bytes.TrimLeftFunc(line, unicode.IsSpace) + + // If the line is empty or a comment, we return a blank key/value pair. + if len(line) == 0 || line[0] == '#' { + return kv, nil + } + + data := strings.SplitN(string(line), "=", 2) + key := data[0] + if errs := validation.IsEnvVarName(key); len(errs) != 0 { + return kv, fmt.Errorf("%q is not a valid key name: %s", key, strings.Join(errs, ";")) + } + + if len(data) == 2 { + kv.Value = data[1] + } else { + // No value (no `=` in the line) is a signal to obtain the value + // from the environment. + kv.Value = os.Getenv(key) + } + kv.Key = key + return kv, nil +} diff --git a/k8sdeps/kv/kv_test.go b/k8sdeps/kv/kv_test.go new file mode 100644 index 000000000..7a398ac8a --- /dev/null +++ b/k8sdeps/kv/kv_test.go @@ -0,0 +1,68 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kv + +import ( + "reflect" + "testing" +) + +func TestKeyValuesFromLines(t *testing.T) { + tests := []struct { + desc string + content string + expectedPairs []KVPair + expectedErr bool + }{ + { + desc: "valid kv content parse", + content: ` + k1=v1 + k2=v2 + `, + expectedPairs: []KVPair{ + {Key: "k1", Value: "v1"}, + {Key: "k2", Value: "v2"}, + }, + expectedErr: false, + }, + { + desc: "content with comments", + content: ` + k1=v1 + #k2=v2 + `, + expectedPairs: []KVPair{ + {Key: "k1", Value: "v1"}, + }, + expectedErr: false, + }, + // TODO: add negative testcases + } + + for _, test := range tests { + pairs, err := KeyValuesFromLines([]byte(test.content)) + if test.expectedErr && err == nil { + t.Fatalf("%s should not return error", test.desc) + } + + if !reflect.DeepEqual(pairs, test.expectedPairs) { + t.Errorf("%s should succeed, got:%v exptected:%v", test.desc, pairs, test.expectedPairs) + } + + } +}