Localize configMapGenerator, secretGenerator fields (#4894)

* Localize configMapGenerator, secretGenerator fields

* Improve readability

* Expose kv parseFileSource

* Add localizeGenerator to Localizer

* Improve and test ParseFileSource error messages

api/internal/generators/utils.go

@@ -5,6 +5,8 @@ package generators
 
 import (
 	"fmt"
+	"path"
+	"strings"
 
 	"github.com/go-errors/errors"
 	"sigs.k8s.io/kustomize/api/ifc"
@@ -95,3 +97,28 @@ func setImmutable(
 
 	return nil
 }
+
+// ParseFileSource parses the source given.
+//
+// Acceptable formats include:
+//  1. source-path: the basename will become the key name
+//  2. source-name=source-path: the source-name will become the key name and
+//     source-path is the path to the key file.
+//
+// Key names cannot include '='.
+func ParseFileSource(source string) (keyName, filePath string, err error) {
+	numSeparators := strings.Count(source, "=")
+	switch {
+	case numSeparators == 0:
+		return path.Base(source), source, nil
+	case numSeparators == 1 && strings.HasPrefix(source, "="):
+		return "", "", errors.Errorf("missing key name for file path %q in source %q", strings.TrimPrefix(source, "="), source)
+	case numSeparators == 1 && strings.HasSuffix(source, "="):
+		return "", "", errors.Errorf("missing file path for key name %q in source %q", strings.TrimSuffix(source, "="), source)
+	case numSeparators > 1:
+		return "", "", errors.Errorf("source %q key name or file path contains '='", source)
+	default:
+		components := strings.Split(source, "=")
+		return components[0], components[1], nil
+	}
+}

api/internal/generators/utils_test.go

@@ -0,0 +1,51 @@
+// Copyright 2020 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package generators_test
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+	. "sigs.k8s.io/kustomize/api/internal/generators"
+)
+
+func TestParseFileSource(t *testing.T) {
+	tests := map[string]*struct {
+		Input    string
+		Error    string
+		Key      string
+		Filename string
+	}{
+		"filename only": {
+			Input:    "./path/myfile",
+			Key:      "myfile",
+			Filename: "./path/myfile",
+		},
+		"key and filename": {
+			Input:    "newName.ini=oldName",
+			Key:      "newName.ini",
+			Filename: "oldName",
+		},
+		"multiple =": {
+			Input: "newName.ini==oldName",
+			Error: `source "newName.ini==oldName" key name or file path contains '='`,
+		},
+		"missing key": {
+			Input: "=myfile",
+			Error: `missing key name for file path "myfile" in source "=myfile"`,
+		},
+	}
+	for name, test := range tests {
+		t.Run(name, func(t *testing.T) {
+			key, file, err := ParseFileSource(test.Input)
+			if test.Error != "" {
+				require.EqualError(t, err, test.Error)
+			} else {
+				require.NoError(t, err)
+				require.Equal(t, test.Key, key)
+				require.Equal(t, test.Filename, file)
+			}
+		})
+	}
+}

api/internal/localizer/localizer.go

@@ -8,6 +8,7 @@ import (
 	"path/filepath"
 
 	"sigs.k8s.io/kustomize/api/ifc"
+	"sigs.k8s.io/kustomize/api/internal/generators"
 	pLdr "sigs.k8s.io/kustomize/api/internal/plugins/loader"
 	"sigs.k8s.io/kustomize/api/internal/target"
 	"sigs.k8s.io/kustomize/api/konfig"
@@ -128,8 +129,50 @@ func (lc *localizer) localizeNativeFields(kust *types.Kustomization) error {
 			kust.Patches[i].Path = newPath
 		}
 	}
+
+	for i := range kust.ConfigMapGenerator {
+		if err := lc.localizeGenerator(&kust.ConfigMapGenerator[i].GeneratorArgs); err != nil {
+			return errors.WrapPrefixf(err, "unable to localize configMapGenerator")
+		}
+	}
+	for i := range kust.SecretGenerator {
+		if err := lc.localizeGenerator(&kust.SecretGenerator[i].GeneratorArgs); err != nil {
+			return errors.WrapPrefixf(err, "unable to localize secretGenerator")
+		}
+	}
+
 	// TODO(annasong): localize all other kustomization fields: resources, bases, crds, configurations,
-	// openapi, patchesJson6902, patchesStrategicMerge, replacements, configMapGenerators, secretGenerators
+	// openapi, patchesJson6902, patchesStrategicMerge, replacements
+	return nil
+}
+
+// localizeGenerator localizes the file paths on generator.
+func (lc *localizer) localizeGenerator(generator *types.GeneratorArgs) error {
+	locEnvs := make([]string, len(generator.EnvSources))
+	for i, env := range generator.EnvSources {
+		newPath, err := lc.localizeFile(env)
+		if err != nil {
+			return errors.WrapPrefixf(err, "unable to localize generator envs file")
+		}
+		locEnvs[i] = newPath
+	}
+	locFiles := make([]string, len(generator.FileSources))
+	for i, file := range generator.FileSources {
+		k, f, err := generators.ParseFileSource(file)
+		if err != nil {
+			return errors.WrapPrefixf(err, "unable to parse generator files entry %q", file)
+		}
+		newFile, err := lc.localizeFile(f)
+		if err != nil {
+			return errors.WrapPrefixf(err, "unable to localize generator files path in entry %q", file)
+		}
+		if f != file {
+			newFile = k + "=" + newFile
+		}
+		locFiles[i] = newFile
+	}
+	generator.EnvSources = locEnvs
+	generator.FileSources = locFiles
 	return nil
 }
 

api/internal/localizer/localizer_test.go

@@ -220,6 +220,34 @@ patches:
 	checkFSys(t, fSysExpected, fSys)
 }
 
+func TestLocalizeUnreferencedIgnored(t *testing.T) {
+	fSys := makeMemoryFs(t)
+	targetAndUnreferenced := map[string]string{
+		"kustomization.yaml": `apiVersion: kustomize.config.k8s.io/v1beta1
+configMapGenerator:
+- envs:
+  - env
+  name: referenced-file
+kind: Kustomization
+`,
+		"env":            "APPLE=orange",
+		"env.properties": "USERNAME=password",
+		"resource.yaml":  podConfiguration,
+	}
+	addFiles(t, fSys, "/alpha/beta", targetAndUnreferenced)
+
+	err := Run("/alpha/beta", "/alpha", "/beta", fSys)
+	require.NoError(t, err)
+
+	fSysExpected := makeMemoryFs(t)
+	addFiles(t, fSysExpected, "/alpha/beta", targetAndUnreferenced)
+	addFiles(t, fSysExpected, "/beta/beta", map[string]string{
+		"kustomization.yaml": targetAndUnreferenced["kustomization.yaml"],
+		"env":                targetAndUnreferenced["env"],
+	})
+	checkFSys(t, fSysExpected, fSys)
+}
+
 func TestLocalizePatches(t *testing.T) {
 	fSys := makeMemoryFs(t)
 	kustAndPatch := map[string]string{
@@ -262,6 +290,76 @@ spec:
 	checkFSys(t, fSysExpected, fSys)
 }
 
+func TestLocalizeConfigMapGenerator(t *testing.T) {
+	fSys := makeMemoryFs(t)
+	kustAndData := map[string]string{
+		"kustomization.yaml": `apiVersion: kustomize.config.k8s.io/v1beta1
+configMapGenerator:
+- envs:
+  - standard.env
+  namespace: my
+  options:
+    immutable: true
+- behavior: merge
+  files:
+  - key.properties
+  literals:
+  - PEAR=pineapple
+kind: Kustomization
+metadata:
+  name: test
+`,
+		"standard.env": `SIZE=0.1
+IS_GLOBAL=true`,
+		"key.properties": "value",
+	}
+	addFiles(t, fSys, "/a/b", kustAndData)
+
+	err := Run("/a/b", "", "", fSys)
+	require.NoError(t, err)
+
+	fSysExpected := makeMemoryFs(t)
+	addFiles(t, fSysExpected, "/a/b", kustAndData)
+	addFiles(t, fSysExpected, "/localized-b", kustAndData)
+	checkFSys(t, fSysExpected, fSys)
+}
+
+func TestLocalizeSecretGenerator(t *testing.T) {
+	fSys := makeMemoryFs(t)
+	kustAndData := map[string]string{
+		"kustomization.yaml": `apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+secretGenerator:
+- behavior: create
+  files:
+  - key=b/value.properties
+  - b/value
+  name: secret
+- envs:
+  - crt
+  - key
+  type: kubernetes.io/tls
+- literals:
+  - APPLE=b3Jhbmdl
+  - PLUM=cGx1b3Q=
+  name: no-files
+`,
+		"crt":                "tls.crt=LS0tLS1CRUd...0tLQo=",
+		"key":                "tls.key=LS0tLS1CRUd...0tLQo=",
+		"b/value.properties": "dmFsdWU=",
+		"b/value":            "dmFsdWU=",
+	}
+	addFiles(t, fSys, "/a", kustAndData)
+
+	err := Run("/a", "/", "/localized-a", fSys)
+	require.NoError(t, err)
+
+	fSysExpected := makeMemoryFs(t)
+	addFiles(t, fSysExpected, "/a", kustAndData)
+	addFiles(t, fSysExpected, "/localized-a/a", kustAndData)
+	checkFSys(t, fSysExpected, fSys)
+}
+
 func TestLocalizeFileNoFile(t *testing.T) {
 	fSys := makeMemoryFs(t)
 	kustAndPatch := map[string]string{

api/kv/kv.go

@@ -8,13 +8,13 @@ import (
 	"bytes"
 	"fmt"
 	"os"
-	"path"
 	"strings"
 	"unicode"
 	"unicode/utf8"
 
 	"github.com/pkg/errors"
 	"sigs.k8s.io/kustomize/api/ifc"
+	"sigs.k8s.io/kustomize/api/internal/generators"
 	"sigs.k8s.io/kustomize/api/types"
 )
 
@@ -77,7 +77,7 @@ func keyValuesFromLiteralSources(sources []string) ([]types.Pair, error) {
 func (kvl *loader) keyValuesFromFileSources(sources []string) ([]types.Pair, error) {
 	var kvs []types.Pair
 	for _, s := range sources {
-		k, fPath, err := parseFileSource(s)
+		k, fPath, err := generators.ParseFileSource(s)
 		if err != nil {
 			return nil, err
 		}
@@ -175,31 +175,6 @@ func (kvl *loader) keyValuesFromLine(line []byte, currentLine int) (types.Pair,
 	return kv, nil
 }
 
-// ParseFileSource parses the source given.
-//
-//  Acceptable formats include:
-//   1.  source-path: the basename will become the key name
-//   2.  source-name=source-path: the source-name will become the key name and
-//       source-path is the path to the key file.
-//
-// Key names cannot include '='.
-func parseFileSource(source string) (keyName, filePath string, err error) {
-	numSeparators := strings.Count(source, "=")
-	switch {
-	case numSeparators == 0:
-		return path.Base(source), source, nil
-	case numSeparators == 1 && strings.HasPrefix(source, "="):
-		return "", "", fmt.Errorf("key name for file path %v missing", strings.TrimPrefix(source, "="))
-	case numSeparators == 1 && strings.HasSuffix(source, "="):
-		return "", "", fmt.Errorf("file path for key name %v missing", strings.TrimSuffix(source, "="))
-	case numSeparators > 1:
-		return "", "", errors.New("key names or file paths cannot contain '='")
-	default:
-		components := strings.Split(source, "=")
-		return components[0], components[1], nil
-	}
-}
-
 // ParseLiteralSource parses the source key=val pair into its component pieces.
 // This functionality is distinguished from strings.SplitN(source, "=", 2) since
 // it returns an error in the case of empty keys, values, or a missing equals sign.