Improvements to kyaml fn framework

This commit creates a new version of the alpha configuration functions framework. Goals include: - Make it easy to build multi-version APIs with the framework (not previously facilitated at all). - Simplify the framework's APIs where redundant configuration options exist (leaving the most powerful, replacing others with helpers to maintain usability they provided). - Make the Framework's APIs more consistent (e.g. between the various template types, usage of kio.Filter, field names) - Decouple responsibilities (e.g. command creation, resource list processing, generation of templating functions). - Make the framework even more powerfully pluggable (e.g. any kio.Filter can be a selector, and the selector the framework provides is itself a filter built from reusable abstractions). - Improve documentation. - Make container patches merge fields (notably list fields like `env`) correctly.
2026-05-17 18:25:26 +00:00 · 2021-01-21 17:52:45 -08:00
parent 1d524b6fbe
commit 5c4b5b1bf0
61 changed files with 4059 additions and 2624 deletions
--- a/kyaml/fn/framework/processors.go
+++ b/kyaml/fn/framework/processors.go
@@ -0,0 +1,410 @@
+// Copyright 2021 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package framework
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+	"text/template"
+
+	"github.com/markbates/pkger"
+
+	"sigs.k8s.io/kustomize/kyaml/errors"
+	"sigs.k8s.io/kustomize/kyaml/kio"
+	"sigs.k8s.io/kustomize/kyaml/kio/filters"
+	"sigs.k8s.io/kustomize/kyaml/yaml"
+)
+
+// SimpleProcessor processes a ResourceList by loading the FunctionConfig into
+// the given Config type and then running the provided Filter on the ResourceList.
+// The provided Config MAY implement Defaulter and Validator to have Default and Validate
+// respectively called between unmarshalling and filter execution.
+//
+// Typical uses include functions that do not actually require config, and simple functions built
+// with a filter that closes over the Config instance to access ResourceList.functionConfig values.
+type SimpleProcessor struct {
+	// Filter is the kio.Filter that will be used to process the ResourceList's items.
+	// Note that kio.FilterFunc is available to transform a compatible func into a kio.Filter.
+	Filter kio.Filter
+	// Config must be a struct capable of receiving the data from ResourceList.functionConfig.
+	// Filter functions may close over this struct to access its data.
+	Config interface{}
+}
+
+// Process makes SimpleProcessor implement the ResourceListProcessor interface.
+// It loads the ResourceList.functionConfig into the provided Config type, applying
+// defaulting and validation if supported by Config. It then executes the processor's filter.
+func (p SimpleProcessor) Process(rl *ResourceList) error {
+	if err := LoadFunctionConfig(rl.FunctionConfig, p.Config); err != nil {
+		return errors.Wrap(err)
+	}
+	return errors.Wrap(rl.Filter(p.Filter))
+}
+
+// GVKFilterMap is a FilterProvider that resolves Filters through a simple lookup in a map.
+// It is intended for use in VersionedAPIProcessor.
+type GVKFilterMap map[string]map[string]kio.Filter
+
+// ProviderFor makes GVKFilterMap implement the FilterProvider interface.
+// It uses the given apiVersion and kind to do a simple lookup in the map and
+// returns an error if no exact match is found.
+func (m GVKFilterMap) ProviderFor(apiVersion, kind string) (kio.Filter, error) {
+	if kind == "" {
+		return nil, errors.Errorf("kind is required")
+	}
+	if apiVersion == "" {
+		return nil, errors.Errorf("apiVersion is required")
+	}
+
+	var ok bool
+	var versionMap map[string]kio.Filter
+	if versionMap, ok = m[kind]; !ok {
+		return nil, errors.Errorf("kind %q is not supported", kind)
+	}
+
+	var p kio.Filter
+	if p, ok = versionMap[apiVersion]; !ok {
+		return nil, errors.Errorf("apiVersion %q is not supported for kind %q", apiVersion, kind)
+	}
+	return p, nil
+}
+
+// FilterProvider is implemented by types that provide a way to look up which Filter
+// should be used to process a ResourceList based on the ApiVersion and Kind of the
+// ResourceList.functionConfig in the input. FilterProviders are intended to be used
+// as part of VersionedAPIProcessor.
+type FilterProvider interface {
+	// ProviderFor returns the appropriate filter for the given APIVersion and Kind.
+	ProviderFor(apiVersion, kind string) (kio.Filter, error)
+}
+
+// FilterProviderFunc converts a compatible function to a FilterProvider.
+type FilterProviderFunc func(apiVersion, kind string) (kio.Filter, error)
+
+// ProviderFor makes FilterProviderFunc implement FilterProvider.
+func (f FilterProviderFunc) ProviderFor(apiVersion, kind string) (kio.Filter, error) {
+	return f(apiVersion, kind)
+}
+
+// VersionedAPIProcessor selects the appropriate kio.Filter based on the ApiVersion
+// and Kind of the ResourceList.functionConfig in the input.
+// It can be used to implement configuration function APIs that evolve over time,
+// or create processors that support multiple configuration APIs with a single entrypoint.
+// All provided Filters MUST be structs capable of receiving ResourceList.functionConfig data.
+// Provided Filters MAY implement Defaulter and Validator to have Default and Validate
+// respectively called between unmarshalling and filter execution.
+type VersionedAPIProcessor struct {
+	// FilterProvider resolves a kio.Filter for each supported API, based on its APIVersion and Kind.
+	// GVKFilterMap is a simple FilterProvider implementation for use here.
+	FilterProvider FilterProvider
+}
+
+// Process makes VersionedAPIProcessor implement the ResourceListProcessor interface.
+// It looks up the configuration object to use based on the ApiVersion and Kind of the
+// input ResourceList.functionConfig, loads ResourceList.functionConfig into that object,
+// invokes Validate and Default if supported, and finally invokes Filter.
+func (p *VersionedAPIProcessor) Process(rl *ResourceList) error {
+	api, err := p.FilterProvider.ProviderFor(extractGVK(rl.FunctionConfig))
+	if err != nil {
+		return errors.WrapPrefixf(err, "unable to identify provider for resource")
+	}
+	if err := LoadFunctionConfig(rl.FunctionConfig, api); err != nil {
+		return errors.Wrap(err)
+	}
+	return errors.Wrap(rl.Filter(api))
+}
+
+// extractGVK returns the apiVersion and kind fields from the given RNodes if it contains
+// valid TypeMeta. It returns an empty string if a value is not found.
+func extractGVK(src *yaml.RNode) (apiVersion, kind string) {
+	if src == nil {
+		return "", ""
+	}
+	if versionNode := src.Field("apiVersion"); versionNode != nil {
+		if a, err := versionNode.Value.String(); err == nil {
+			apiVersion = strings.TrimSpace(a)
+		}
+	}
+	if kindNode := src.Field("kind"); kindNode != nil {
+		if k, err := kindNode.Value.String(); err == nil {
+			kind = strings.TrimSpace(k)
+		}
+	}
+	return apiVersion, kind
+}
+
+// LoadFunctionConfig reads a configuration resource from YAML into the provided data structure
+// and then prepares it for use by running defaulting and validation on it, if supported.
+// ResourceListProcessors should use this function to load ResourceList.functionConfig.
+func LoadFunctionConfig(src *yaml.RNode, api interface{}) error {
+	if api == nil {
+		return nil
+	}
+	if err := yaml.Unmarshal([]byte(src.MustString()), api); err != nil {
+		return errors.Wrap(err)
+	}
+
+	if d, ok := api.(Defaulter); ok {
+		if err := d.Default(); err != nil {
+			return err
+		}
+	}
+
+	if v, ok := api.(Validator); ok {
+		return v.Validate()
+	}
+	return nil
+}
+
+// TemplateProcessor is a ResourceListProcessor based on rendering templates with the data in
+// ResourceList.functionConfig. It works as follows:
+// - loads ResourceList.functionConfig into TemplateData
+// - runs PreProcessFilters
+// - renders ResourceTemplates and adds them to ResourceList.items
+// - renders PatchTemplates and applies them to ResourceList.items
+// - executes a merge on ResourceList.items if configured to
+// - runs PostProcessFilters
+// The TemplateData struct MAY implement Defaulter and Validator to have Default and Validate
+// respectively called between unmarshalling and filter execution.
+//
+// TemplateProcessor also implements kio.Filter directly and can be used in the construction of
+// higher-level processors. For example, you might use TemplateProcessors as the filters for each
+// API supported by a VersionedAPIProcessor (see VersionedAPIProcessor examples).
+type TemplateProcessor struct {
+	// TemplateData will will be exposed to all the templates in the processor (unless explicitly
+	// overridden for a template).
+	// If TemplateProcessor is used directly as a ResourceListProcessor, TemplateData will contain the
+	// value of ResourceList.functionConfig.
+	TemplateData interface{}
+
+	// ResourceTemplates returns a list of templates to render into resources.
+	// If MergeResources is set, any matching resources in ResourceList.items will be used as patches
+	// modifying the rendered templates. Otherwise, the rendered resources will be appended to
+	// the input resources as-is.
+	ResourceTemplates []ResourceTemplate
+
+	// PatchTemplates is a list of templates to render into patches that apply to ResourceList.items.
+	// ResourcePatchTemplate can be used here to patch entire resources.
+	// ContainerPatchTemplate can be used here to patch specific containers within resources.
+	PatchTemplates []PatchTemplate
+
+	// MergeResources, if set to true, will cause the resources in ResourceList.items to be
+	// will be applied as patches on any matching resources generated by ResourceTemplates.
+	MergeResources bool
+
+	// PreProcessFilters provides a hook to manipulate the ResourceList's items or config after
+	// TemplateData has been populated but before template-based filters are applied.
+	PreProcessFilters []kio.Filter
+
+	// PostProcessFilters provides a hook to manipulate the ResourceList's items after template
+	// filters are applied.
+	PostProcessFilters []kio.Filter
+}
+
+// Filter implements the kio.Filter interface, enabling you to use TemplateProcessor
+// as part of a higher-level ResourceListProcessor like VersionedAPIProcessor.
+// It sets up all the features of TemplateProcessors as a pipeline of filters and executes them.
+func (tp TemplateProcessor) Filter(items []*yaml.RNode) ([]*yaml.RNode, error) {
+	buf := &kio.PackageBuffer{Nodes: items}
+	pipeline := kio.Pipeline{
+		Inputs: []kio.Reader{buf},
+		Filters: []kio.Filter{
+			kio.FilterFunc(tp.doPreProcess),
+			kio.FilterFunc(tp.doResourceTemplates),
+			kio.FilterFunc(tp.doPatchTemplates),
+			kio.FilterFunc(tp.doMerge),
+			kio.FilterFunc(tp.doPostProcess),
+		},
+		Outputs:               []kio.Writer{buf},
+		ContinueOnEmptyResult: true,
+	}
+	if err := pipeline.Execute(); err != nil {
+		return nil, err
+	}
+
+	return buf.Nodes, nil
+}
+
+// Process implements the ResourceListProcessor interface, enabling you to use TemplateProcessor
+// directly as a processor. As a Processor, it loads the ResourceList.functionConfig into the
+// TemplateData field, exposing it to all templates by default.
+func (tp TemplateProcessor) Process(rl *ResourceList) error {
+	if err := LoadFunctionConfig(rl.FunctionConfig, tp.TemplateData); err != nil {
+		return errors.Wrap(err)
+	}
+	return errors.Wrap(rl.Filter(tp))
+}
+
+// TemplatesFunc is a function that provides a list of templates.
+// TemplateProcessor uses this to defer loading of templates to the point where they are used.
+type TemplatesFunc func() ([]*template.Template, error)
+
+// PatchTemplate is implemented by kio.Filters that work by rendering patches and applying them to
+// the given resource nodes.
+type PatchTemplate interface {
+	// Filter is a kio.Filter-compliant function that applies PatchTemplate's templates as patches
+	// on the given resource nodes.
+	Filter(items []*yaml.RNode) ([]*yaml.RNode, error)
+	// DefaultTemplateData accepts default data to be used in template rendering when no template
+	// data was explicitly provided to the PatchTemplate.
+	DefaultTemplateData(interface{})
+}
+
+func (tp *TemplateProcessor) doPreProcess(items []*yaml.RNode) ([]*yaml.RNode, error) {
+	if tp.PreProcessFilters == nil {
+		return items, nil
+	}
+	for i := range tp.PreProcessFilters {
+		filter := tp.PreProcessFilters[i]
+		var err error
+		items, err = filter.Filter(items)
+		if err != nil {
+			return nil, err
+		}
+	}
+	return items, nil
+}
+
+func (tp *TemplateProcessor) doMerge(items []*yaml.RNode) ([]*yaml.RNode, error) {
+	var err error
+	if tp.MergeResources {
+		items, err = filters.MergeFilter{}.Filter(items)
+	}
+	return items, err
+}
+
+func (tp *TemplateProcessor) doPostProcess(items []*yaml.RNode) ([]*yaml.RNode, error) {
+	if tp.PostProcessFilters == nil {
+		return items, nil
+	}
+	for i := range tp.PostProcessFilters {
+		filter := tp.PostProcessFilters[i]
+		var err error
+		items, err = filter.Filter(items)
+		if err != nil {
+			return nil, err
+		}
+	}
+	return items, nil
+}
+
+func (tp *TemplateProcessor) doResourceTemplates(items []*yaml.RNode) ([]*yaml.RNode, error) {
+	if tp.ResourceTemplates == nil {
+		return items, nil
+	}
+
+	for i := range tp.ResourceTemplates {
+		tp.ResourceTemplates[i].DefaultTemplateData(tp.TemplateData)
+		newItems, err := tp.ResourceTemplates[i].Render()
+		if err != nil {
+			return nil, err
+		}
+		if tp.MergeResources {
+			// apply inputs as patches -- add the new items to the front of the list
+			items = append(newItems, items...)
+		} else {
+			// assume these are new unique resources--append to the list
+			items = append(items, newItems...)
+		}
+	}
+	return items, nil
+}
+
+func (tp *TemplateProcessor) doPatchTemplates(items []*yaml.RNode) ([]*yaml.RNode, error) {
+	if tp.PatchTemplates == nil {
+		return items, nil
+	}
+
+	for i := range tp.PatchTemplates {
+		// Default the template data for the patch to the processor's data
+		tp.PatchTemplates[i].DefaultTemplateData(tp.TemplateData)
+		var err error
+		if items, err = tp.PatchTemplates[i].Filter(items); err != nil {
+			return nil, err
+		}
+	}
+	return items, nil
+}
+
+// StringTemplates returns a TemplatesFunc that will generate templates from the provided strings.
+// This is a helper to facilitate providing ResourceTemplates, PatchTemplates and
+// ContainerPatchTemplates to a TemplateProcessor.
+func StringTemplates(data ...string) TemplatesFunc {
+	return func() ([]*template.Template, error) {
+		var templates []*template.Template
+		for i := range data {
+			t, err := template.New(fmt.Sprintf("inline%d", i)).Parse(data[i])
+			if err != nil {
+				return nil, err
+			}
+			templates = append(templates, t)
+		}
+		return templates, nil
+	}
+}
+
+// TemplatesFromFile returns a TemplatesFunc that will generate templates from the provided files.
+// This is a helper to facilitate providing ResourceTemplates, PatchTemplates and
+// ContainerPatchTemplates to a TemplateProcessor.
+func TemplatesFromFile(files ...string) TemplatesFunc {
+	return func() ([]*template.Template, error) {
+		var templates []*template.Template
+		for i := range files {
+			n := filepath.Base(files[i])
+			t, err := template.New(n).ParseFiles(files[i])
+			if err != nil {
+				return nil, err
+			}
+			templates = append(templates, t)
+		}
+		return templates, nil
+	}
+}
+
+// TemplatesFromDir returns a TemplatesFunc that will generate templates from the provided
+// directories. Only files suffixed with .template.yaml will be included.
+// This is a helper to facilitate providing ResourceTemplates, PatchTemplates and
+// ContainerPatchTemplates to a TemplateProcessor.
+func TemplatesFromDir(dirs ...pkger.Dir) TemplatesFunc {
+	return func() ([]*template.Template, error) {
+		var pt []*template.Template
+		for i := range dirs {
+			dir := string(dirs[i])
+			err := pkger.Walk(dir, func(p string, info os.FileInfo, err error) error {
+				if err != nil {
+					return err
+				}
+				if !strings.HasSuffix(info.Name(), ".template.yaml") {
+					return nil
+				}
+				name := path.Join(dir, info.Name())
+				f, err := pkger.Open(name)
+				if err != nil {
+					return err
+				}
+				defer f.Close()
+
+				b, err := ioutil.ReadAll(f)
+				if err != nil {
+					return err
+				}
+				t, err := template.New(info.Name()).Parse(string(b))
+				if err != nil {
+					return err
+				}
+
+				pt = append(pt, t)
+				return nil
+			})
+			if err != nil {
+				return nil, err
+			}
+		}
+		return pt, nil
+	}
+}