Load built-in plugins for kustomize localize (#4869)

* Load and filter built-in plugins

* Improve readability

* Process plugins as resources instead of bytes

* Throw error for validators

* Differentiate generators, transformers processing

* Enable validators

* add wrapper error
* improve documentation

api/internal/localizer/builtinplugins.go

@@ -0,0 +1,38 @@
+// Copyright 2022 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package localizer
+
+import (
+	"sigs.k8s.io/kustomize/kyaml/kio"
+	"sigs.k8s.io/kustomize/kyaml/yaml"
+)
+
+// localizeBuiltinGenerators localizes built-in generators with file paths.
+// Note that this excludes helm, which needs a repo.
+type localizeBuiltinGenerators struct {
+}
+
+var _ kio.Filter = &localizeBuiltinGenerators{}
+
+// Filter localizes the built-in generators with file paths. Filter returns an error if
+// generators contains a resource that is not a built-in generator, cannot contain a file path,
+// needs more than a file path like helm, or is not localizable.
+// TODO(annasong): implement
+func (lbg *localizeBuiltinGenerators) Filter(generators []*yaml.RNode) ([]*yaml.RNode, error) {
+	return generators, nil
+}
+
+// localizeBuiltinTransformers localizes built-in transformers with file paths.
+type localizeBuiltinTransformers struct {
+}
+
+var _ kio.Filter = &localizeBuiltinTransformers{}
+
+// Filter localizes the built-in transformers with file paths. Filter returns an error if
+// transformers contains a resource that is not a built-in transformer, cannot contain a file path,
+// or is not localizable.
+// TODO(annasong): implement
+func (lbt *localizeBuiltinTransformers) Filter(transformers []*yaml.RNode) ([]*yaml.RNode, error) {
+	return transformers, nil
+}

api/internal/localizer/errors.go

@@ -0,0 +1,18 @@
+// Copyright 2022 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package localizer
+
+import "fmt"
+
+type ResourceLoadError struct {
+	InlineError error
+	FileError   error
+}
+
+var _ error = ResourceLoadError{}
+
+func (rle ResourceLoadError) Error() string {
+	return fmt.Sprintf(`when parsing as inline received error: %s
+when parsing as filepath received error: %s`, rle.InlineError, rle.FileError)
+}

api/internal/localizer/localizer.go

@@ -16,6 +16,7 @@ import (
 	"sigs.k8s.io/kustomize/api/types"
 	"sigs.k8s.io/kustomize/kyaml/errors"
 	"sigs.k8s.io/kustomize/kyaml/filesys"
+	"sigs.k8s.io/kustomize/kyaml/kio"
 	"sigs.k8s.io/yaml"
 )
 
@@ -62,11 +63,18 @@ func (lc *Localizer) Localize() error {
 	if err != nil {
 		return errors.Wrap(err)
 	}
-	kust, err := lc.processKust(kt)
+
+	kustomization := kt.Kustomization()
+	err = lc.localizeNativeFields(&kustomization)
 	if err != nil {
 		return err
 	}
-	content, err := yaml.Marshal(kust)
+	err = lc.localizeBuiltinPlugins(&kustomization)
+	if err != nil {
+		return err
+	}
+
+	content, err := yaml.Marshal(&kustomization)
 	if err != nil {
 		return errors.WrapPrefixf(err, "unable to serialize localized kustomization file")
 	}
@@ -76,22 +84,22 @@ func (lc *Localizer) Localize() error {
 	return nil
 }
 
-// processKust returns a copy of the kustomization at kt with paths localized.
-func (lc *Localizer) processKust(kt *target.KustTarget) (*types.Kustomization, error) {
-	kust := kt.Kustomization()
+// localizeNativeFields localizes paths on kustomize-native fields, like configMapGenerator, that kustomize has a
+// built-in understanding of. This excludes helm-related fields, such as `helmGlobals` and `helmCharts`.
+func (lc *Localizer) localizeNativeFields(kust *types.Kustomization) error {
 	for i := range kust.Patches {
 		if kust.Patches[i].Path != "" {
 			newPath, err := lc.localizeFile(kust.Patches[i].Path)
 			if err != nil {
-				return nil, errors.WrapPrefixf(err, "unable to localize patches path %q", kust.Patches[i].Path)
+				return errors.WrapPrefixf(err, "unable to localize patches path %q", kust.Patches[i].Path)
 			}
 			kust.Patches[i].Path = newPath
 		}
 	}
-	// TODO(annasong): localize all other kustomization fields: resources, components, crds, configurations,
-	// openapi, patchesStrategicMerge, replacements, configMapGenerators, secretGenerators
+	// TODO(annasong): localize all other kustomization fields: resources, bases, components, crds, configurations,
+	// openapi, patchesJson6902, patchesStrategicMerge, replacements, configMapGenerators, secretGenerators
 	// TODO(annasong): localize built-in plugins under generators, transformers, and validators fields
-	return &kust, nil
+	return nil
 }
 
 // localizeFile localizes file path and returns the localized path
@@ -127,3 +135,69 @@ func (lc *Localizer) localizeFile(path string) (string, error) {
 	}
 	return locPath, nil
 }
+
+// localizeBuiltinPlugins localizes built-in plugins on kust that can contain file paths. The built-in plugins
+// can be inline or in a file. This excludes the HelmChartInflationGenerator.
+//
+// Note that the localization in this function has not been implemented yet.
+func (lc *Localizer) localizeBuiltinPlugins(kust *types.Kustomization) error {
+	for fieldName, plugins := range map[string]struct {
+		entries   []string
+		localizer kio.Filter
+	}{
+		"generators": {
+			kust.Generators,
+			&localizeBuiltinGenerators{},
+		},
+		"transformers": {
+			kust.Transformers,
+			&localizeBuiltinTransformers{},
+		},
+		"validators": {
+			kust.Validators,
+			&localizeBuiltinTransformers{},
+		},
+	} {
+		for i, entry := range plugins.entries {
+			rm, isPath, err := lc.loadResource(entry)
+			if err != nil {
+				return errors.WrapPrefixf(err, "unable to load %s entry", fieldName)
+			}
+			err = rm.ApplyFilter(plugins.localizer)
+			if err != nil {
+				return errors.Wrap(err)
+			}
+			localizedPlugin, err := rm.AsYaml()
+			if err != nil {
+				return errors.WrapPrefixf(err, "unable to serialize localized %s entry %q", fieldName, entry)
+			}
+			var newEntry string
+			if isPath {
+				// TODO(annasong): write localizedPlugin to dst
+				newEntry = entry
+			} else {
+				newEntry = string(localizedPlugin)
+			}
+			plugins.entries[i] = newEntry
+		}
+	}
+	return nil
+}
+
+// loadResource tries to load resourceEntry as a file path or inline.
+// On success, loadResource returns the loaded resource map and whether resourceEntry is a file path.
+func (lc *Localizer) loadResource(resourceEntry string) (resmap.ResMap, bool, error) {
+	var fileErr error
+	rm, inlineErr := lc.rFactory.NewResMapFromBytes([]byte(resourceEntry))
+	if inlineErr != nil {
+		rm, fileErr = lc.rFactory.FromFile(lc.ldr, resourceEntry)
+		if fileErr != nil {
+			err := ResourceLoadError{
+				InlineError: inlineErr,
+				FileError:   fileErr,
+			}
+			return nil, false, errors.WrapPrefixf(err, "unable to load resource entry %q", resourceEntry)
+		}
+	}
+	return rm, fileErr == nil, nil
+}

api/internal/localizer/localizer_test.go

@@ -299,3 +299,208 @@ patches:
 	lclzr := createLocalizer(t, fSys, "/a/b", "", "/dst")
 	require.Error(t, lclzr.Localize())
 }
+
+func TestLocalizeGenerators(t *testing.T) {
+	fSys := makeMemoryFs(t)
+	kustAndPlugins := map[string]string{
+		"kustomization.yaml": `apiVersion: kustomize.config.k8s.io/v1beta1
+generators:
+- plugin.yaml
+- |
+  apiVersion: builtin
+  behavior: create
+  kind: ConfigMapGenerator
+  literals:
+  - APPLE=orange
+  metadata:
+    name: another-map
+  ---
+  apiVersion: builtin
+  kind: SecretGenerator
+  literals:
+  - APPLE=b3Jhbmdl
+  metadata:
+    name: secret
+  options:
+    disableNameSuffixHash: true
+kind: Kustomization
+`,
+		"plugin.yaml": `apiVersion: builtin
+kind: ConfigMapGenerator
+metadata:
+  name: map
+`,
+	}
+	addFiles(t, fSys, "/a", kustAndPlugins)
+
+	lclzr := createLocalizer(t, fSys, "/a", "", "/alpha/dst")
+	require.NoError(t, lclzr.Localize())
+
+	fSysExpected := makeMemoryFs(t)
+	addFiles(t, fSysExpected, "/a", kustAndPlugins)
+	addFiles(t, fSysExpected, "/alpha/dst", map[string]string{
+		"kustomization.yaml": kustAndPlugins["kustomization.yaml"],
+	})
+	checkFSys(t, fSysExpected, fSys)
+}
+
+func TestLocalizeTransformers(t *testing.T) {
+	fSys := makeMemoryFs(t)
+	kustAndPlugins := map[string]string{
+		"kustomization.yaml": `apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+transformers:
+- |
+  apiVersion: builtin
+  jsonOp: '[{"op": "add", "path": "/spec/template/spec/dnsPolicy", "value": "ClusterFirst"}]'
+  kind: PatchJson6902Transformer
+  metadata:
+    name: patch6902
+  target:
+    name: deployment
+  ---
+  apiVersion: builtin
+  kind: ReplacementTransformer
+  metadata:
+    name: replacement
+  replacements:
+  - source:
+      fieldPath: spec.template.spec.containers.0.image
+      kind: Deployment
+    targets:
+    - fieldPaths:
+      - spec.template.spec.containers.1.image
+      select:
+        kind: Deployment
+- plugin.yaml
+`,
+		"plugin.yaml": `apiVersion: builtin
+kind: PatchStrategicMergeTransformer
+metadata:
+  name: patchSM
+paths:
+- pod.yaml
+`,
+	}
+	addFiles(t, fSys, "/a", kustAndPlugins)
+
+	lclzr := createLocalizer(t, fSys, "/a", "", "/dst")
+	require.NoError(t, lclzr.Localize())
+
+	fSysExpected := makeMemoryFs(t)
+	addFiles(t, fSysExpected, "/a", kustAndPlugins)
+	addFiles(t, fSysExpected, "/dst", map[string]string{
+		"kustomization.yaml": kustAndPlugins["kustomization.yaml"],
+	})
+	checkFSys(t, fSysExpected, fSys)
+}
+
+func TestLocalizeValidators(t *testing.T) {
+	fSys := makeMemoryFs(t)
+	kustAndPlugin := map[string]string{
+		"kustomization.yaml": `apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+validators:
+- |-
+  apiVersion: builtin
+  kind: ReplacementTransformer
+  metadata:
+    name: replacement
+  replacements:
+  - source:
+      kind: ConfigMap
+      fieldPath: metadata.name
+    targets:
+    - select:
+        kind: ConfigMap
+      fieldPaths:
+      - metadata.name
+- replacement.yaml
+`,
+		"replacement.yaml": `apiVersion: builtin
+kind: ReplacementTransformer
+metadata:
+  name: replacement-2
+replacements:
+- source:
+    kind: Secret
+    fieldPath: data.USER_NAME
+  targets:
+  - select:
+      kind: Secret
+    fieldPaths:
+    - data.USER_NAME
+`,
+	}
+	addFiles(t, fSys, "/", kustAndPlugin)
+	lclzr := createLocalizer(t, fSys, "/", "", "/dst")
+	require.NoError(t, lclzr.Localize())
+
+	fSysExpected := makeMemoryFs(t)
+	addFiles(t, fSysExpected, "/", kustAndPlugin)
+	addFiles(t, fSysExpected, "/dst", map[string]string{
+		"kustomization.yaml": kustAndPlugin["kustomization.yaml"],
+	})
+	checkFSys(t, fSysExpected, fSys)
+}
+
+func TestLocalizeBuiltinPluginsNotResource(t *testing.T) {
+	type testCase struct {
+		name         string
+		files        map[string]string
+		errPrefix    string
+		inlineErrMsg string
+		fileErrMsg   string
+	}
+	for _, test := range []testCase{
+		{
+			name: "bad_inline_resource",
+			files: map[string]string{
+				"kustomization.yaml": `apiVersion: kustomize.config.k8s.io/v1beta1
+generators:
+- |
+  apiVersion: builtin
+  kind: ConfigMapGenerator
+kind: Kustomization
+`,
+			},
+			errPrefix:    `unable to load generators entry: unable to load resource entry "apiVersion: builtin\nkind: ConfigMapGenerator\n"`,
+			inlineErrMsg: `missing metadata.name in object {{builtin ConfigMapGenerator} {{ } map[] map[]}}`,
+			fileErrMsg: `invalid file reference: '/apiVersion: builtin
+kind: ConfigMapGenerator
+' doesn't exist`,
+		},
+		{
+			name: "bad_file_resource",
+			files: map[string]string{
+				"kustomization.yaml": `apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+transformers:
+- plugin.yaml
+`,
+				"plugin.yaml": `apiVersion: builtin
+metadata:
+  name: PatchTransformer
+`,
+			},
+			errPrefix:    `unable to load transformers entry: unable to load resource entry "plugin.yaml"`,
+			inlineErrMsg: `missing Resource metadata`,
+			fileErrMsg:   `missing kind in object {{builtin } {{PatchTransformer } map[] map[]}}`,
+		},
+	} {
+		t.Run(test.name, func(t *testing.T) {
+			fSys := makeMemoryFs(t)
+			addFiles(t, fSys, "/", test.files)
+			lclzr := createLocalizer(t, fSys, "/", "", "/dst")
+			err := lclzr.Localize()
+
+			var actualErr ResourceLoadError
+			require.ErrorAs(t, err, &actualErr)
+			require.EqualError(t, actualErr.InlineError, test.inlineErrMsg)
+			require.EqualError(t, actualErr.FileError, test.fileErrMsg)
+
+			require.EqualError(t, err, fmt.Sprintf(`%s: when parsing as inline received error: %s
+when parsing as filepath received error: %s`, test.errPrefix, test.inlineErrMsg, test.fileErrMsg))
+		})
+	}
+}