kustomize/kyaml/kio/kio.go

// Copyright 2019 The Kubernetes Authors.
// SPDX-License-Identifier: Apache-2.0

// Package kio contains low-level libraries for reading, modifying and writing
// Resource Configuration and packages.
package kio

import (
	"fmt"

	"sigs.k8s.io/kustomize/kyaml/errors"
	"sigs.k8s.io/kustomize/kyaml/kio/kioutil"
	"sigs.k8s.io/kustomize/kyaml/yaml"
)

// Reader reads ResourceNodes. Analogous to io.Reader.
type Reader interface {
	Read() ([]*yaml.RNode, error)
}

// ResourceNodeSlice is a collection of ResourceNodes.
// While ResourceNodeSlice has no inherent constraints on ordering or uniqueness, specific
// Readers, Filters or Writers may have constraints.
type ResourceNodeSlice []*yaml.RNode

var _ Reader = ResourceNodeSlice{}

func (o ResourceNodeSlice) Read() ([]*yaml.RNode, error) {
	return o, nil
}

// Writer writes ResourceNodes. Analogous to io.Writer.
type Writer interface {
	Write([]*yaml.RNode) error
}

// WriterFunc implements a Writer as a function.
type WriterFunc func([]*yaml.RNode) error

func (fn WriterFunc) Write(o []*yaml.RNode) error {
	return fn(o)
}

// ReaderWriter implements both Reader and Writer interfaces
type ReaderWriter interface {
	Reader
	Writer
}

// Filter modifies a collection of Resource Configuration by returning the modified slice.
// When possible, Filters should be serializable to yaml so that they can be described
// as either data or code.
//
// Analogous to http://www.linfo.org/filters.html
type Filter interface {
	Filter([]*yaml.RNode) ([]*yaml.RNode, error)
}

// FilterFunc implements a Filter as a function.
type FilterFunc func([]*yaml.RNode) ([]*yaml.RNode, error)

func (fn FilterFunc) Filter(o []*yaml.RNode) ([]*yaml.RNode, error) {
	return fn(o)
}

// Pipeline reads Resource Configuration from a set of Inputs, applies some
// transformation filters, and writes the results to a set of Outputs.
//
// Analogous to http://www.linfo.org/pipes.html
type Pipeline struct {
	// Inputs provide sources for Resource Configuration to be read.
	Inputs []Reader `yaml:"inputs,omitempty"`

	// Filters are transformations applied to the Resource Configuration.
	// They are applied in the order they are specified.
	// Analogous to http://www.linfo.org/filters.html
	Filters []Filter `yaml:"filters,omitempty"`

	// Outputs are where the transformed Resource Configuration is written.
	Outputs []Writer `yaml:"outputs,omitempty"`

	// ContinueOnEmptyResult configures what happens when a filter in the pipeline
	// returns an empty result.
	// If it is false (default), subsequent filters will be skipped and the result
	// will be returned immediately. This is useful as an optimization when you
	// know that subsequent filters will not alter the empty result.
	// If it is true, the empty result will be provided as input to the next
	// filter in the list. This is useful when subsequent functions in the
	// pipeline may generate new resources.
	ContinueOnEmptyResult bool `yaml:"continueOnEmptyResult,omitempty"`
}

// Execute executes each step in the sequence, returning immediately after encountering
// any error as part of the Pipeline.
func (p Pipeline) Execute() error {
	return p.ExecuteWithCallback(nil)
}

// PipelineExecuteCallbackFunc defines a callback function that will be called each time a step in the pipeline succeeds.
type PipelineExecuteCallbackFunc = func(op Filter)

// ExecuteWithCallback executes each step in the sequence, returning immediately after encountering
// any error as part of the Pipeline. The callback will be called each time a step succeeds.
func (p Pipeline) ExecuteWithCallback(callback PipelineExecuteCallbackFunc) error {
	var result []*yaml.RNode

	// read from the inputs
	for _, i := range p.Inputs {
		nodes, err := i.Read()
		if err != nil {
			return errors.Wrap(err)
		}
		result = append(result, nodes...)
	}

	// apply operations
	var err error
	for i := range p.Filters {
		// Not all RNodes passed through kio.Pipeline have metadata nor should
		// they all be required to.
		var nodeAnnos map[string]map[string]string
		nodeAnnos, err = storeInternalAnnotations(result)
		if err != nil && err != yaml.ErrMissingMetadata {
			return err
		}

		op := p.Filters[i]
		if callback != nil {
			callback(op)
		}
		result, err = op.Filter(result)
		// TODO (issue 2872): This len(result) == 0 should be removed and empty result list should be
		// handled by outputs. However currently some writer like LocalPackageReadWriter
		// will clear the output directory and which will cause unpredictable results
		if len(result) == 0 && !p.ContinueOnEmptyResult || err != nil {
			return errors.Wrap(err)
		}

		// If either the internal annotations for path, index, and id OR the legacy
		// annotations for path, index, and id are changed, we have to update the other.
		err = reconcileInternalAnnotations(result, nodeAnnos)
		if err != nil && err != yaml.ErrMissingMetadata {
			return err
		}
	}

	// write to the outputs
	for _, o := range p.Outputs {
		if err := o.Write(result); err != nil {
			return errors.Wrap(err)
		}
	}
	return nil
}

// FilterAll runs the yaml.Filter against all inputs
func FilterAll(filter yaml.Filter) Filter {
	return FilterFunc(func(nodes []*yaml.RNode) ([]*yaml.RNode, error) {
		for i := range nodes {
			_, err := filter.Filter(nodes[i])
			if err != nil {
				return nil, errors.Wrap(err)
			}
		}
		return nodes, nil
	})
}

// Store the original path, index, and id annotations so that we can reconcile
// it later. This is necessary because currently both internal-prefixed annotations
// and legacy annotations are currently supported, and a change to one must be
// reflected in the other.
func storeInternalAnnotations(result []*yaml.RNode) (map[string]map[string]string, error) {
	nodeAnnosMap := make(map[string]map[string]string)

	for i := range result {
		if err := kioutil.CopyLegacyAnnotations(result[i]); err != nil {
			return nil, err
		}
		meta, err := result[i].GetMeta()
		if err != nil {
			return nil, err
		}
		if err := checkMismatchedAnnos(meta); err != nil {
			return nil, err
		}

		path := meta.Annotations[kioutil.PathAnnotation]
		index := meta.Annotations[kioutil.IndexAnnotation]
		id := meta.Annotations[kioutil.IdAnnotation]

		if _, ok := nodeAnnosMap[path]; !ok {
			nodeAnnosMap[path] = make(map[string]string)
		}
		nodeAnnosMap[path][index] = id
	}
	return nodeAnnosMap, nil
}

func checkMismatchedAnnos(meta yaml.ResourceMeta) error {
	path := meta.Annotations[kioutil.PathAnnotation]
	index := meta.Annotations[kioutil.IndexAnnotation]
	id := meta.Annotations[kioutil.IdAnnotation]

	legacyPath := meta.Annotations[kioutil.LegacyPathAnnotation]
	legacyIndex := meta.Annotations[kioutil.LegacyIndexAnnotation]
	legacyId := meta.Annotations[kioutil.LegacyIdAnnotation]

	// if prior to running the functions, the legacy and internal annotations differ,
	// throw an error as we cannot infer the user's intent.
	if path != legacyPath {
		return fmt.Errorf("resource input to function has mismatched legacy and internal path annotations")
	}
	if index != legacyIndex {
		return fmt.Errorf("resource input to function has mismatched legacy and internal index annotations")
	}
	if id != legacyId {
		return fmt.Errorf("resource input to function has mismatched legacy and internal id annotations")
	}
	return nil
}

type nodeAnnotations struct {
	path  string
	index string
	id    string
}

func reconcileInternalAnnotations(result []*yaml.RNode, nodeAnnosMap map[string]map[string]string) error {
	for _, node := range result {
		meta, err := node.GetMeta()
		if err != nil {
			return err
		}
		// if only one annotation is set, set the other.
		err = missingInternalOrLegacyAnnotations(node, meta)
		if err != nil {
			return err
		}
		// we must check to see if the function changed either the new internal annotations
		// or the old legacy annotations. If one is changed, the change must be reflected
		// in the other.
		err = checkAnnotationsAltered(node, meta, nodeAnnosMap)
		if err != nil {
			return err
		}
		// if the annotations are still somehow out of sync, throw an error
		meta, err = node.GetMeta()
		if err != nil {
			return err
		}
		err = checkMismatchedAnnos(meta)
		if err != nil {
			return err
		}
	}
	return nil
}

func missingInternalOrLegacyAnnotations(rn *yaml.RNode, meta yaml.ResourceMeta) error {
	if err := missingInternalOrLegacyAnnotation(rn, meta, kioutil.PathAnnotation, kioutil.LegacyPathAnnotation); err != nil {
		return err
	}
	if err := missingInternalOrLegacyAnnotation(rn, meta, kioutil.IndexAnnotation, kioutil.LegacyIndexAnnotation); err != nil {
		return err
	}
	if err := missingInternalOrLegacyAnnotation(rn, meta, kioutil.IdAnnotation, kioutil.LegacyIdAnnotation); err != nil {
		return err
	}
	return nil
}

func missingInternalOrLegacyAnnotation(rn *yaml.RNode, meta yaml.ResourceMeta, newKey string, legacyKey string) error {
	value := meta.Annotations[newKey]
	legacyValue := meta.Annotations[legacyKey]

	if value == "" && legacyValue == "" {
		// do nothing
		return nil
	}

	if value == "" {
		// new key is not set, copy from legacy key
		if err := rn.PipeE(yaml.SetAnnotation(newKey, legacyValue)); err != nil {
			return err
		}
	} else if legacyValue == "" {
		// legacy key is not set, copy from new key
		if err := rn.PipeE(yaml.SetAnnotation(legacyKey, value)); err != nil {
			return err
		}
	}
	return nil
}

func checkAnnotationsAltered(rn *yaml.RNode, meta yaml.ResourceMeta, nodeAnnosMap map[string]map[string]string) error {
	// get the resource's current path, index, and ids from the new annotations
	internal := nodeAnnotations{
		path:  meta.Annotations[kioutil.PathAnnotation],
		index: meta.Annotations[kioutil.IndexAnnotation],
		id:    meta.Annotations[kioutil.IdAnnotation],
	}

	// get the resource's current path, index, and ids from the legacy annotations
	legacy := nodeAnnotations{
		path:  meta.Annotations[kioutil.LegacyPathAnnotation],
		index: meta.Annotations[kioutil.LegacyIndexAnnotation],
		id:    meta.Annotations[kioutil.LegacyIdAnnotation],
	}

	if internal.path == legacy.path &&
		internal.index == legacy.index &&
		internal.id == legacy.id {
		// none of the annotations differ, so no reconciliation is needed
		return nil
	}

	// nodeAnnosMap is a map of structure path -> index -> id that stores
	// all of the resources' path/index/id annotations prior to the functions
	// being run. We use that to check whether the legacy or new internal
	// annotations have been changed, and make sure the change is reflected
	// in the other.

	// first, check if the internal annotations are found in nodeAnnosMap
	if indexIdMap, ok := nodeAnnosMap[internal.path]; ok {
		if id, ok := indexIdMap[internal.index]; ok {
			if id == internal.id {
				// the internal annotations of the resource match the ones stored in
				// nodeAnnosMap, so we should copy the legacy annotations to the
				// internal ones
				if err := updateAnnotations(rn, meta,
					[]string{
						kioutil.PathAnnotation,
						kioutil.IndexAnnotation,
						kioutil.IdAnnotation,
					},
					[]string{
						legacy.path,
						legacy.index,
						legacy.id,
					}); err != nil {
					return err
				}
			}
		}
	}

	// check the opposite, to see if the legacy annotations are in nodeAnnosMap
	if indexIdMap, ok := nodeAnnosMap[legacy.path]; ok {
		if id, ok := indexIdMap[legacy.index]; ok {
			if id == legacy.id {
				// the legacy annotations of the resource match the ones stored in
				// nodeAnnosMap, so we should copy the internal annotations to the
				// legacy ones
				if err := updateAnnotations(rn, meta,
					[]string{
						kioutil.LegacyPathAnnotation,
						kioutil.LegacyIndexAnnotation,
						kioutil.LegacyIdAnnotation,
					},
					[]string{
						internal.path,
						internal.index,
						internal.id,
					}); err != nil {
					return err
				}
			}
		}
	}
	return nil
}

func updateAnnotations(rn *yaml.RNode, meta yaml.ResourceMeta, keys []string, values []string) error {
	if len(keys) != len(values) {
		return fmt.Errorf("keys is not same length as values")
	}
	for i := range keys {
		_, ok := meta.Annotations[keys[i]]
		if values[i] == "" && !ok {
			// don't set "" if annotation is not already there
			continue
		}
		if err := rn.PipeE(yaml.SetAnnotation(keys[i], values[i])); err != nil {
			return err
		}

	}
	return nil
}