kyaml: initial support for yaml and resource manipulation

kyaml/yaml/walk/associative_sequence.go

@@ -0,0 +1,128 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package walk
+
+import (
+	"strings"
+
+	"github.com/go-errors/errors"
+	"sigs.k8s.io/kustomize/kyaml/sets"
+	"sigs.k8s.io/kustomize/kyaml/yaml"
+)
+
+func (l *Walker) walkAssociativeSequence() (*yaml.RNode, error) {
+
+	// may require initializing the dest node
+	dest, err := l.Sources.setDestNode(l.VisitList(l.Sources, AssociativeList))
+	if dest == nil || err != nil {
+		return nil, err
+	}
+
+	// find the list of elements we need to recursively walk
+	key, err := l.elementKey()
+	if err != nil {
+		return nil, err
+	}
+	values, err := l.elementValues(key)
+	if err != nil {
+		return nil, err
+	}
+
+	// recursively set the elements in the list
+	for _, value := range values {
+		val, err := Walker{Visitor: l,
+			Sources: l.elementValue(key, value)}.Walk()
+		if err != nil {
+			return nil, err
+		}
+		if yaml.IsEmpty(val) {
+			_, err = dest.Pipe(yaml.ElementSetter{Key: key, Value: value})
+			if err != nil {
+				return nil, err
+			}
+			continue
+		}
+
+		if val.Field(key) == nil {
+			// make sure the key is set on the field
+			_, err = val.Pipe(yaml.SetField(key, yaml.NewScalarRNode(value)))
+			if err != nil {
+				return nil, err
+			}
+		}
+
+		// this handles empty and non-empty values
+		_, err = dest.Pipe(yaml.ElementSetter{Element: val.YNode(), Key: key, Value: value})
+		if err != nil {
+			return nil, err
+		}
+	}
+	// field is empty
+	if yaml.IsEmpty(dest) {
+		return nil, nil
+	}
+	return dest, nil
+}
+
+// elementKey returns the merge key to use for the associative list
+func (l Walker) elementKey() (string, error) {
+	var key string
+	for i := range l.Sources {
+		if l.Sources[i] != nil && len(l.Sources[i].Content()) > 0 {
+			newKey := l.Sources[i].GetAssociativeKey()
+			if key != "" && key != newKey {
+				return "", errors.Errorf(
+					"conflicting merge keys [%s,%s] for field %s",
+					key, newKey, strings.Join(l.Path, "."))
+			}
+			key = newKey
+		}
+	}
+	if key == "" {
+		return "", errors.Errorf("no merge key found for field %s",
+			strings.Join(l.Path, "."))
+	}
+	return key, nil
+}
+
+// elementValues returns a slice containing all values for the field across all elements
+// from all sources.
+// Return value slice is ordered using the original ordering from the elements, where
+// elements missing from earlier sources appear later.
+func (l Walker) elementValues(key string) ([]string, error) {
+	// use slice to to keep elements in the original order
+	// dest node must be first
+	var returnValues []string
+	seen := sets.String{}
+	for i := range l.Sources {
+		if l.Sources[i] == nil {
+			continue
+		}
+
+		// add the value of the field for each element
+		// don't check error, we know this is a list node
+		values, _ := l.Sources[i].ElementValues(key)
+		for _, s := range values {
+			if seen.Has(s) {
+				continue
+			}
+			returnValues = append(returnValues, s)
+			seen.Insert(s)
+		}
+	}
+	return returnValues, nil
+}
+
+// fieldValue returns a slice containing each source's value for fieldName
+func (l Walker) elementValue(key, value string) []*yaml.RNode {
+	var fields []*yaml.RNode
+	for i := range l.Sources {
+		if l.Sources[i] == nil {
+			fields = append(fields, nil)
+			continue
+		}
+		fields = append(fields, l.Sources[i].Element(key, value))
+	}
+	return fields
+}

kyaml/yaml/walk/map.go

@@ -0,0 +1,81 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package walk
+
+import (
+	"sort"
+
+	"sigs.k8s.io/kustomize/kyaml/sets"
+	"sigs.k8s.io/kustomize/kyaml/yaml"
+)
+
+// walkMap returns the value of VisitMap
+//
+// - call VisitMap
+// - set the return value on l.Dest
+// - walk each source field
+// - set each source field value on l.Dest
+func (l Walker) walkMap() (*yaml.RNode, error) {
+	// get the new map value
+	dest, err := l.Sources.setDestNode(l.VisitMap(l.Sources))
+	if dest == nil || err != nil {
+		return nil, err
+	}
+
+	// recursively set the field values on the map
+	for _, key := range l.fieldNames() {
+		val, err := Walker{Visitor: l,
+			Sources: l.fieldValue(key), Path: append(l.Path, key)}.Walk()
+		if err != nil {
+			return nil, err
+		}
+
+		// this handles empty and non-empty values
+		_, err = dest.Pipe(yaml.FieldSetter{Name: key, Value: val})
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return dest, nil
+}
+
+// valueIfPresent returns node.Value if node is non-nil, otherwise returns nil
+func (l Walker) valueIfPresent(node *yaml.MapNode) *yaml.RNode {
+	if node == nil {
+		return nil
+	}
+	return node.Value
+}
+
+// fieldNames returns a sorted slice containing the names of all fields that appear in any of
+// the sources
+func (l Walker) fieldNames() []string {
+	fields := sets.String{}
+	for _, s := range l.Sources {
+		if s == nil {
+			continue
+		}
+		// don't check error, we know this is a mapping node
+		sFields, _ := s.Fields()
+		fields.Insert(sFields...)
+	}
+	result := fields.List()
+	sort.Strings(result)
+	return result
+}
+
+// fieldValue returns a slice containing each source's value for fieldName
+func (l Walker) fieldValue(fieldName string) []*yaml.RNode {
+	var fields []*yaml.RNode
+	for i := range l.Sources {
+		if l.Sources[i] == nil {
+			fields = append(fields, nil)
+			continue
+		}
+		field := l.Sources[i].Field(fieldName)
+		fields = append(fields, l.valueIfPresent(field))
+	}
+	return fields
+}

kyaml/yaml/walk/nonassociative_sequence.go

@@ -0,0 +1,13 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package walk
+
+import (
+	"sigs.k8s.io/kustomize/kyaml/yaml"
+)
+
+// walkNonAssociativeSequence returns the value of VisitList
+func (l Walker) walkNonAssociativeSequence() (*yaml.RNode, error) {
+	return l.VisitList(l.Sources, NonAssociateList)
+}

kyaml/yaml/walk/scalar.go

@@ -0,0 +1,11 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package walk
+
+import "sigs.k8s.io/kustomize/kyaml/yaml"
+
+// walkScalar returns the value of VisitScalar
+func (l Walker) walkScalar() (*yaml.RNode, error) {
+	return l.VisitScalar(l.Sources)
+}

kyaml/yaml/walk/visitor.go

@@ -0,0 +1,38 @@
+// Copyright 2019 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package walk
+
+import (
+	"sigs.k8s.io/kustomize/kyaml/yaml"
+)
+
+type ListKind int32
+
+const (
+	AssociativeList ListKind = 1 + iota
+	NonAssociateList
+)
+
+// Visitor is invoked by walk with source and destination node pairs
+type Visitor interface {
+	VisitMap(nodes Sources) (*yaml.RNode, error)
+
+	VisitScalar(nodes Sources) (*yaml.RNode, error)
+
+	VisitList(nodes Sources, kind ListKind) (*yaml.RNode, error)
+}
+
+// NoOp is returned if GrepFilter should do nothing after calling Set
+var ClearNode *yaml.RNode = nil

kyaml/yaml/walk/walk.go

@@ -0,0 +1,144 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package walk
+
+import (
+	"fmt"
+	"os"
+	"strings"
+
+	"sigs.k8s.io/kustomize/kyaml/yaml"
+)
+
+// Filter walks the Source RNode and modifies the RNode provided to GrepFilter.
+type Walker struct {
+	// Visitor is invoked by GrepFilter
+	Visitor
+
+	// Source is the RNode to walk.  All Source fields and associative list elements
+	// will be visited.
+	Sources Sources
+
+	// Path is the field path to the current Source Node.
+	Path []string
+}
+
+func (l Walker) Kind() yaml.Kind {
+	for _, s := range l.Sources {
+		if !yaml.IsEmpty(s) {
+			return s.YNode().Kind
+		}
+	}
+	return 0
+}
+
+// GrepFilter implements yaml.GrepFilter
+func (l Walker) Walk() (*yaml.RNode, error) {
+	// invoke the handler for the corresponding node type
+	switch l.Kind() {
+	case yaml.MappingNode:
+		if err := yaml.ErrorIfAnyInvalidAndNonNull(yaml.MappingNode, l.Sources...); err != nil {
+			return nil, err
+		}
+		return l.walkMap()
+	case yaml.SequenceNode:
+		if err := yaml.ErrorIfAnyInvalidAndNonNull(yaml.SequenceNode, l.Sources...); err != nil {
+			return nil, err
+		}
+		if yaml.IsAssociative(l.Sources) {
+			return l.walkAssociativeSequence()
+		} else {
+			return l.walkNonAssociativeSequence()
+		}
+	case yaml.ScalarNode:
+		if err := yaml.ErrorIfAnyInvalidAndNonNull(yaml.ScalarNode, l.Sources...); err != nil {
+			return nil, err
+		}
+		return l.walkScalar()
+	case 0:
+		// walk empty nodes as maps
+		return l.walkMap()
+	default:
+		return nil, nil
+	}
+}
+
+const (
+	DestIndex = iota
+	OriginIndex
+	UpdatedIndex
+)
+
+type Sources []*yaml.RNode
+
+// Dest returns the destination node
+func (s Sources) Dest() *yaml.RNode {
+	if len(s) <= DestIndex {
+		return nil
+	}
+	return s[DestIndex]
+}
+
+// Origin returns the origin node
+func (s Sources) Origin() *yaml.RNode {
+	if len(s) <= OriginIndex {
+		return nil
+	}
+	return s[OriginIndex]
+}
+
+// Updated returns the updated node
+func (s Sources) Updated() *yaml.RNode {
+	if len(s) <= UpdatedIndex {
+		return nil
+	}
+	return s[UpdatedIndex]
+}
+
+func (s Sources) String() string {
+	var values []string
+	for i := range s {
+		str, err := s[i].String()
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "%v\n", err)
+		}
+		values = append(values, str)
+	}
+	return strings.Join(values, "\n")
+}
+
+// setDestNode sets the destination source node
+func (s Sources) setDestNode(node *yaml.RNode, err error) (*yaml.RNode, error) {
+	if err != nil {
+		return nil, err
+	}
+	s[0] = node
+	return node, nil
+}
+
+type FieldSources []*yaml.MapNode
+
+// Dest returns the destination node
+func (s FieldSources) Dest() *yaml.MapNode {
+	if len(s) <= DestIndex {
+		return nil
+	}
+	return s[DestIndex]
+}
+
+// Origin returns the origin node
+func (s FieldSources) Origin() *yaml.MapNode {
+	if len(s) <= OriginIndex {
+		return nil
+	}
+	return s[OriginIndex]
+}
+
+// Updated returns the updated node
+func (s FieldSources) Updated() *yaml.MapNode {
+	if len(s) <= UpdatedIndex {
+		return nil
+	}
+	return s[UpdatedIndex]
+}