kustomize/kyaml/fn/framework/processors.go

// Copyright 2021 The Kubernetes Authors.
// SPDX-License-Identifier: Apache-2.0

package framework

import (
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"strings"
	"text/template"

	"github.com/markbates/pkger"

	"sigs.k8s.io/kustomize/kyaml/errors"
	"sigs.k8s.io/kustomize/kyaml/kio"
	"sigs.k8s.io/kustomize/kyaml/kio/filters"
	"sigs.k8s.io/kustomize/kyaml/yaml"
)

// SimpleProcessor processes a ResourceList by loading the FunctionConfig into
// the given Config type and then running the provided Filter on the ResourceList.
// The provided Config MAY implement Defaulter and Validator to have Default and Validate
// respectively called between unmarshalling and filter execution.
//
// Typical uses include functions that do not actually require config, and simple functions built
// with a filter that closes over the Config instance to access ResourceList.functionConfig values.
type SimpleProcessor struct {
	// Filter is the kio.Filter that will be used to process the ResourceList's items.
	// Note that kio.FilterFunc is available to transform a compatible func into a kio.Filter.
	Filter kio.Filter
	// Config must be a struct capable of receiving the data from ResourceList.functionConfig.
	// Filter functions may close over this struct to access its data.
	Config interface{}
}

// Process makes SimpleProcessor implement the ResourceListProcessor interface.
// It loads the ResourceList.functionConfig into the provided Config type, applying
// defaulting and validation if supported by Config. It then executes the processor's filter.
func (p SimpleProcessor) Process(rl *ResourceList) error {
	if err := LoadFunctionConfig(rl.FunctionConfig, p.Config); err != nil {
		return errors.Wrap(err)
	}
	return errors.Wrap(rl.Filter(p.Filter))
}

// GVKFilterMap is a FilterProvider that resolves Filters through a simple lookup in a map.
// It is intended for use in VersionedAPIProcessor.
type GVKFilterMap map[string]map[string]kio.Filter

// ProviderFor makes GVKFilterMap implement the FilterProvider interface.
// It uses the given apiVersion and kind to do a simple lookup in the map and
// returns an error if no exact match is found.
func (m GVKFilterMap) ProviderFor(apiVersion, kind string) (kio.Filter, error) {
	if kind == "" {
		return nil, errors.Errorf("kind is required")
	}
	if apiVersion == "" {
		return nil, errors.Errorf("apiVersion is required")
	}

	var ok bool
	var versionMap map[string]kio.Filter
	if versionMap, ok = m[kind]; !ok {
		return nil, errors.Errorf("kind %q is not supported", kind)
	}

	var p kio.Filter
	if p, ok = versionMap[apiVersion]; !ok {
		return nil, errors.Errorf("apiVersion %q is not supported for kind %q", apiVersion, kind)
	}
	return p, nil
}

// FilterProvider is implemented by types that provide a way to look up which Filter
// should be used to process a ResourceList based on the ApiVersion and Kind of the
// ResourceList.functionConfig in the input. FilterProviders are intended to be used
// as part of VersionedAPIProcessor.
type FilterProvider interface {
	// ProviderFor returns the appropriate filter for the given APIVersion and Kind.
	ProviderFor(apiVersion, kind string) (kio.Filter, error)
}

// FilterProviderFunc converts a compatible function to a FilterProvider.
type FilterProviderFunc func(apiVersion, kind string) (kio.Filter, error)

// ProviderFor makes FilterProviderFunc implement FilterProvider.
func (f FilterProviderFunc) ProviderFor(apiVersion, kind string) (kio.Filter, error) {
	return f(apiVersion, kind)
}

// VersionedAPIProcessor selects the appropriate kio.Filter based on the ApiVersion
// and Kind of the ResourceList.functionConfig in the input.
// It can be used to implement configuration function APIs that evolve over time,
// or create processors that support multiple configuration APIs with a single entrypoint.
// All provided Filters MUST be structs capable of receiving ResourceList.functionConfig data.
// Provided Filters MAY implement Defaulter and Validator to have Default and Validate
// respectively called between unmarshalling and filter execution.
type VersionedAPIProcessor struct {
	// FilterProvider resolves a kio.Filter for each supported API, based on its APIVersion and Kind.
	// GVKFilterMap is a simple FilterProvider implementation for use here.
	FilterProvider FilterProvider
}

// Process makes VersionedAPIProcessor implement the ResourceListProcessor interface.
// It looks up the configuration object to use based on the ApiVersion and Kind of the
// input ResourceList.functionConfig, loads ResourceList.functionConfig into that object,
// invokes Validate and Default if supported, and finally invokes Filter.
func (p *VersionedAPIProcessor) Process(rl *ResourceList) error {
	api, err := p.FilterProvider.ProviderFor(extractGVK(rl.FunctionConfig))
	if err != nil {
		return errors.WrapPrefixf(err, "unable to identify provider for resource")
	}
	if err := LoadFunctionConfig(rl.FunctionConfig, api); err != nil {
		return errors.Wrap(err)
	}
	return errors.Wrap(rl.Filter(api))
}

// extractGVK returns the apiVersion and kind fields from the given RNodes if it contains
// valid TypeMeta. It returns an empty string if a value is not found.
func extractGVK(src *yaml.RNode) (apiVersion, kind string) {
	if src == nil {
		return "", ""
	}
	if versionNode := src.Field("apiVersion"); versionNode != nil {
		if a, err := versionNode.Value.String(); err == nil {
			apiVersion = strings.TrimSpace(a)
		}
	}
	if kindNode := src.Field("kind"); kindNode != nil {
		if k, err := kindNode.Value.String(); err == nil {
			kind = strings.TrimSpace(k)
		}
	}
	return apiVersion, kind
}

// LoadFunctionConfig reads a configuration resource from YAML into the provided data structure
// and then prepares it for use by running defaulting and validation on it, if supported.
// ResourceListProcessors should use this function to load ResourceList.functionConfig.
func LoadFunctionConfig(src *yaml.RNode, api interface{}) error {
	if api == nil {
		return nil
	}
	if err := yaml.Unmarshal([]byte(src.MustString()), api); err != nil {
		return errors.Wrap(err)
	}

	if d, ok := api.(Defaulter); ok {
		if err := d.Default(); err != nil {
			return err
		}
	}

	if v, ok := api.(Validator); ok {
		return v.Validate()
	}
	return nil
}

// TemplateProcessor is a ResourceListProcessor based on rendering templates with the data in
// ResourceList.functionConfig. It works as follows:
// - loads ResourceList.functionConfig into TemplateData
// - runs PreProcessFilters
// - renders ResourceTemplates and adds them to ResourceList.items
// - renders PatchTemplates and applies them to ResourceList.items
// - executes a merge on ResourceList.items if configured to
// - runs PostProcessFilters
// The TemplateData struct MAY implement Defaulter and Validator to have Default and Validate
// respectively called between unmarshalling and filter execution.
//
// TemplateProcessor also implements kio.Filter directly and can be used in the construction of
// higher-level processors. For example, you might use TemplateProcessors as the filters for each
// API supported by a VersionedAPIProcessor (see VersionedAPIProcessor examples).
type TemplateProcessor struct {
	// TemplateData will will be exposed to all the templates in the processor (unless explicitly
	// overridden for a template).
	// If TemplateProcessor is used directly as a ResourceListProcessor, TemplateData will contain the
	// value of ResourceList.functionConfig.
	TemplateData interface{}

	// ResourceTemplates returns a list of templates to render into resources.
	// If MergeResources is set, any matching resources in ResourceList.items will be used as patches
	// modifying the rendered templates. Otherwise, the rendered resources will be appended to
	// the input resources as-is.
	ResourceTemplates []ResourceTemplate

	// PatchTemplates is a list of templates to render into patches that apply to ResourceList.items.
	// ResourcePatchTemplate can be used here to patch entire resources.
	// ContainerPatchTemplate can be used here to patch specific containers within resources.
	PatchTemplates []PatchTemplate

	// MergeResources, if set to true, will cause the resources in ResourceList.items to be
	// will be applied as patches on any matching resources generated by ResourceTemplates.
	MergeResources bool

	// PreProcessFilters provides a hook to manipulate the ResourceList's items or config after
	// TemplateData has been populated but before template-based filters are applied.
	PreProcessFilters []kio.Filter

	// PostProcessFilters provides a hook to manipulate the ResourceList's items after template
	// filters are applied.
	PostProcessFilters []kio.Filter
}

// Filter implements the kio.Filter interface, enabling you to use TemplateProcessor
// as part of a higher-level ResourceListProcessor like VersionedAPIProcessor.
// It sets up all the features of TemplateProcessors as a pipeline of filters and executes them.
func (tp TemplateProcessor) Filter(items []*yaml.RNode) ([]*yaml.RNode, error) {
	buf := &kio.PackageBuffer{Nodes: items}
	pipeline := kio.Pipeline{
		Inputs: []kio.Reader{buf},
		Filters: []kio.Filter{
			kio.FilterFunc(tp.doPreProcess),
			kio.FilterFunc(tp.doResourceTemplates),
			kio.FilterFunc(tp.doPatchTemplates),
			kio.FilterFunc(tp.doMerge),
			kio.FilterFunc(tp.doPostProcess),
		},
		Outputs:               []kio.Writer{buf},
		ContinueOnEmptyResult: true,
	}
	if err := pipeline.Execute(); err != nil {
		return nil, err
	}

	return buf.Nodes, nil
}

// Process implements the ResourceListProcessor interface, enabling you to use TemplateProcessor
// directly as a processor. As a Processor, it loads the ResourceList.functionConfig into the
// TemplateData field, exposing it to all templates by default.
func (tp TemplateProcessor) Process(rl *ResourceList) error {
	if err := LoadFunctionConfig(rl.FunctionConfig, tp.TemplateData); err != nil {
		return errors.Wrap(err)
	}
	return errors.Wrap(rl.Filter(tp))
}

// TemplatesFunc is a function that provides a list of templates.
// TemplateProcessor uses this to defer loading of templates to the point where they are used.
type TemplatesFunc func() ([]*template.Template, error)

// PatchTemplate is implemented by kio.Filters that work by rendering patches and applying them to
// the given resource nodes.
type PatchTemplate interface {
	// Filter is a kio.Filter-compliant function that applies PatchTemplate's templates as patches
	// on the given resource nodes.
	Filter(items []*yaml.RNode) ([]*yaml.RNode, error)
	// DefaultTemplateData accepts default data to be used in template rendering when no template
	// data was explicitly provided to the PatchTemplate.
	DefaultTemplateData(interface{})
}

func (tp *TemplateProcessor) doPreProcess(items []*yaml.RNode) ([]*yaml.RNode, error) {
	if tp.PreProcessFilters == nil {
		return items, nil
	}
	for i := range tp.PreProcessFilters {
		filter := tp.PreProcessFilters[i]
		var err error
		items, err = filter.Filter(items)
		if err != nil {
			return nil, err
		}
	}
	return items, nil
}

func (tp *TemplateProcessor) doMerge(items []*yaml.RNode) ([]*yaml.RNode, error) {
	var err error
	if tp.MergeResources {
		items, err = filters.MergeFilter{}.Filter(items)
	}
	return items, err
}

func (tp *TemplateProcessor) doPostProcess(items []*yaml.RNode) ([]*yaml.RNode, error) {
	if tp.PostProcessFilters == nil {
		return items, nil
	}
	for i := range tp.PostProcessFilters {
		filter := tp.PostProcessFilters[i]
		var err error
		items, err = filter.Filter(items)
		if err != nil {
			return nil, err
		}
	}
	return items, nil
}

func (tp *TemplateProcessor) doResourceTemplates(items []*yaml.RNode) ([]*yaml.RNode, error) {
	if tp.ResourceTemplates == nil {
		return items, nil
	}

	for i := range tp.ResourceTemplates {
		tp.ResourceTemplates[i].DefaultTemplateData(tp.TemplateData)
		newItems, err := tp.ResourceTemplates[i].Render()
		if err != nil {
			return nil, err
		}
		if tp.MergeResources {
			// apply inputs as patches -- add the new items to the front of the list
			items = append(newItems, items...)
		} else {
			// assume these are new unique resources--append to the list
			items = append(items, newItems...)
		}
	}
	return items, nil
}

func (tp *TemplateProcessor) doPatchTemplates(items []*yaml.RNode) ([]*yaml.RNode, error) {
	if tp.PatchTemplates == nil {
		return items, nil
	}

	for i := range tp.PatchTemplates {
		// Default the template data for the patch to the processor's data
		tp.PatchTemplates[i].DefaultTemplateData(tp.TemplateData)
		var err error
		if items, err = tp.PatchTemplates[i].Filter(items); err != nil {
			return nil, err
		}
	}
	return items, nil
}

// StringTemplates returns a TemplatesFunc that will generate templates from the provided strings.
// This is a helper to facilitate providing ResourceTemplates, PatchTemplates and
// ContainerPatchTemplates to a TemplateProcessor.
func StringTemplates(data ...string) TemplatesFunc {
	return func() ([]*template.Template, error) {
		var templates []*template.Template
		for i := range data {
			t, err := template.New(fmt.Sprintf("inline%d", i)).Parse(data[i])
			if err != nil {
				return nil, err
			}
			templates = append(templates, t)
		}
		return templates, nil
	}
}

// TemplatesFromFile returns a TemplatesFunc that will generate templates from the provided files.
// This is a helper to facilitate providing ResourceTemplates, PatchTemplates and
// ContainerPatchTemplates to a TemplateProcessor.
func TemplatesFromFile(files ...string) TemplatesFunc {
	return func() ([]*template.Template, error) {
		var templates []*template.Template
		for i := range files {
			n := filepath.Base(files[i])
			t, err := template.New(n).ParseFiles(files[i])
			if err != nil {
				return nil, err
			}
			templates = append(templates, t)
		}
		return templates, nil
	}
}

// TemplatesFromDir returns a TemplatesFunc that will generate templates from the provided
// directories. Only files suffixed with .template.yaml will be included.
// This is a helper to facilitate providing ResourceTemplates, PatchTemplates and
// ContainerPatchTemplates to a TemplateProcessor.
func TemplatesFromDir(dirs ...pkger.Dir) TemplatesFunc {
	return func() ([]*template.Template, error) {
		var pt []*template.Template
		for i := range dirs {
			dir := string(dirs[i])
			err := pkger.Walk(dir, func(p string, info os.FileInfo, err error) error {
				if err != nil {
					return err
				}
				if !strings.HasSuffix(info.Name(), ".template.yaml") {
					return nil
				}
				name := path.Join(dir, info.Name())
				f, err := pkger.Open(name)
				if err != nil {
					return err
				}
				defer f.Close()

				b, err := ioutil.ReadAll(f)
				if err != nil {
					return err
				}
				t, err := template.New(info.Name()).Parse(string(b))
				if err != nil {
					return err
				}

				pt = append(pt, t)
				return nil
			})
			if err != nil {
				return nil, err
			}
		}
		return pt, nil
	}
}