kyaml: initial support for yaml and resource manipulation

.travis.yml

@@ -35,6 +35,7 @@ install: true
 
 script:
   - ./travis/pre-commit.sh
+  - ./travis/kyaml-pre-commit.sh
 
 # TBD. Suppressing for now.
 notifications:

kyaml/Makefile

@@ -0,0 +1,29 @@
+# Copyright 2019 The Kubernetes Authors.
+# SPDX-License-Identifier: Apache-2.0
+
+.PHONY: generate fix vet fmt test build tidy
+
+all: generate fix vet fmt test build tidy
+
+fix:
+	go fix ./...
+
+fmt:
+	go fmt ./...
+
+generate:
+	go generate ./...
+
+tidy:
+	go mod tidy
+
+lint:
+	(which $(go env GOPATH)/bin/golangci-lint || go get github.com/golangci/golangci-lint/cmd/golangci-lint@v1.19.1)
+	$(go env GOPATH)/bin/golangci-lint run ./...
+
+test:
+	go test -cover ./...
+
+vet:
+	go vet ./...
+

kyaml/copyutil/copyutil.go

@@ -0,0 +1,128 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+// The copyutil package contains libraries for copying directories of configuration.
+package copyutil
+
+import (
+	"bytes"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"sigs.k8s.io/kustomize/kyaml/sets"
+)
+
+// CopyDir copies a src directory to a dst directory.  CopyDir skips copying the .git directory from the src.
+func CopyDir(src string, dst string) error {
+	return filepath.Walk(src, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		// don't copy the .git dir
+		if path != src {
+			rel := strings.TrimPrefix(path, src)
+			if strings.HasPrefix(rel, string(filepath.Separator)+".git") {
+				return nil
+			}
+		}
+
+		// path is an absolute path, rather than a path relative to src.
+		// e.g. if src is /path/to/package, then path might be /path/to/package/and/sub/dir
+		// we need the path relative to src `and/sub/dir` when we are copying the files to dest.
+		copyTo := strings.TrimPrefix(path, src)
+
+		// make directories that don't exist
+		if info.IsDir() {
+			return os.MkdirAll(filepath.Join(dst, copyTo), info.Mode())
+		}
+
+		// copy file by reading and writing it
+		b, err := ioutil.ReadFile(filepath.Join(src, copyTo))
+		if err != nil {
+			return err
+		}
+		err = ioutil.WriteFile(filepath.Join(dst, copyTo), b, info.Mode())
+		if err != nil {
+			return err
+		}
+
+		return nil
+	})
+}
+
+// Diff returns a list of files that differ between the source and destination.
+//
+// Diff is guaranteed to return a non-empty set if any files differ, but
+// this set is not guaranteed to contain all differing files.
+func Diff(sourceDir, destDir string) (sets.String, error) {
+	// get set of filenames in the package source
+	upstreamFiles := sets.String{}
+	err := filepath.Walk(sourceDir, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+
+		// skip git repo if it exists
+		if strings.Contains(path, ".git") {
+			return nil
+		}
+
+		upstreamFiles.Insert(strings.TrimPrefix(strings.TrimPrefix(path, sourceDir), string(filepath.Separator)))
+		return nil
+	})
+	if err != nil {
+		return sets.String{}, err
+	}
+
+	// get set of filenames in the cloned package
+	localFiles := sets.String{}
+	err = filepath.Walk(destDir, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+
+		// skip git repo if it exists
+		if strings.Contains(path, ".git") {
+			return nil
+		}
+
+		localFiles.Insert(strings.TrimPrefix(strings.TrimPrefix(path, destDir), string(filepath.Separator)))
+		return nil
+	})
+	if err != nil {
+		return sets.String{}, err
+	}
+
+	// verify the source and cloned packages have the same set of filenames
+	diff := upstreamFiles.SymmetricDifference(localFiles)
+
+	// verify file contents match
+	for _, f := range upstreamFiles.Intersection(localFiles).List() {
+		fi, err := os.Stat(filepath.Join(destDir, f))
+		if err != nil {
+			return diff, err
+		}
+		if fi.Mode().IsDir() {
+			// already checked that this directory exists in the local files
+			continue
+		}
+
+		// compare upstreamFiles
+		b1, err := ioutil.ReadFile(filepath.Join(destDir, f))
+		if err != nil {
+			return diff, err
+		}
+		b2, err := ioutil.ReadFile(filepath.Join(sourceDir, f))
+		if err != nil {
+			return diff, err
+		}
+		if !bytes.Equal(b1, b2) {
+			diff.Insert(f)
+		}
+	}
+
+	// return the differing files
+	return diff, nil
+}

kyaml/copyutil/copyutil_test.go

@@ -0,0 +1,199 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package copyutil_test
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	. "sigs.k8s.io/kustomize/kyaml/copyutil"
+)
+
+// TestDiff_identical verifies identical directories return an empty set
+func TestDiff_identical(t *testing.T) {
+	s, err := ioutil.TempDir("", "copyutilsrc")
+	assert.NoError(t, err)
+	d, err := ioutil.TempDir("", "copyutildest")
+	assert.NoError(t, err)
+
+	err = os.Mkdir(filepath.Join(s, "a1"), 0700)
+	assert.NoError(t, err)
+	err = ioutil.WriteFile(
+		filepath.Join(s, "a1", "f.yaml"), []byte(`a`), 0600)
+	assert.NoError(t, err)
+
+	err = os.Mkdir(filepath.Join(d, "a1"), 0700)
+	assert.NoError(t, err)
+	err = ioutil.WriteFile(
+		filepath.Join(d, "a1", "f.yaml"), []byte(`a`), 0600)
+	assert.NoError(t, err)
+
+	diff, err := Diff(s, d)
+	assert.NoError(t, err)
+	assert.Empty(t, diff.List())
+}
+
+// TestDiff_additionalSourceFiles verifies if there are additional files
+// in the source, the diff will contain them
+func TestDiff_additionalSourceFiles(t *testing.T) {
+	s, err := ioutil.TempDir("", "copyutilsrc")
+	assert.NoError(t, err)
+	d, err := ioutil.TempDir("", "copyutildest")
+	assert.NoError(t, err)
+
+	err = os.Mkdir(filepath.Join(s, "a1"), 0700)
+	assert.NoError(t, err)
+
+	err = os.Mkdir(filepath.Join(s, "a2"), 0700)
+	assert.NoError(t, err)
+
+	err = os.Mkdir(filepath.Join(d, "a1"), 0700)
+	assert.NoError(t, err)
+
+	diff, err := Diff(s, d)
+	assert.NoError(t, err)
+	assert.ElementsMatch(t, diff.List(), []string{"a2"})
+}
+
+// TestDiff_additionalDestFiles verifies if there are additional files
+// in the dest, the diff will contain them
+func TestDiff_additionalDestFiles(t *testing.T) {
+	s, err := ioutil.TempDir("", "copyutilsrc")
+	assert.NoError(t, err)
+	d, err := ioutil.TempDir("", "copyutildest")
+	assert.NoError(t, err)
+
+	err = os.Mkdir(filepath.Join(s, "a1"), 0700)
+	assert.NoError(t, err)
+
+	err = os.Mkdir(filepath.Join(d, "a1"), 0700)
+	assert.NoError(t, err)
+
+	err = os.Mkdir(filepath.Join(d, "a2"), 0700)
+	assert.NoError(t, err)
+
+	diff, err := Diff(s, d)
+	assert.NoError(t, err)
+	assert.ElementsMatch(t, diff.List(), []string{"a2"})
+
+}
+
+// TestDiff_srcDestContentsDiffer verifies if the file contents
+// differ between the source and destination, the diff
+// contains the differing files
+func TestDiff_srcDestContentsDiffer(t *testing.T) {
+	s, err := ioutil.TempDir("", "copyutilsrc")
+	assert.NoError(t, err)
+	d, err := ioutil.TempDir("", "copyutildest")
+	assert.NoError(t, err)
+
+	err = os.Mkdir(filepath.Join(s, "a1"), 0700)
+	assert.NoError(t, err)
+	err = ioutil.WriteFile(
+		filepath.Join(s, "a1", "f.yaml"), []byte(`a`), 0600)
+	assert.NoError(t, err)
+
+	err = os.Mkdir(filepath.Join(d, "a1"), 0700)
+	assert.NoError(t, err)
+	err = ioutil.WriteFile(
+		filepath.Join(d, "a1", "f.yaml"), []byte(`b`), 0600)
+	assert.NoError(t, err)
+
+	diff, err := Diff(s, d)
+	assert.NoError(t, err)
+	assert.ElementsMatch(t, diff.List(), []string{"a1/f.yaml"})
+}
+
+// TestDiff_srcDestContentsDifferInDirs verifies if identical files
+// exist in different directories, they are included in the diff
+func TestDiff_srcDestContentsDifferInDirs(t *testing.T) {
+	s, err := ioutil.TempDir("", "copyutilsrc")
+	assert.NoError(t, err)
+	d, err := ioutil.TempDir("", "copyutildest")
+	assert.NoError(t, err)
+
+	err = os.Mkdir(filepath.Join(s, "a1"), 0700)
+	assert.NoError(t, err)
+	err = ioutil.WriteFile(
+		filepath.Join(s, "a1", "f.yaml"), []byte(`a`), 0600)
+	assert.NoError(t, err)
+
+	err = os.Mkdir(filepath.Join(d, "b1"), 0700)
+	assert.NoError(t, err)
+	err = ioutil.WriteFile(
+		filepath.Join(d, "b1", "f.yaml"), []byte(`a`), 0600)
+	assert.NoError(t, err)
+
+	diff, err := Diff(s, d)
+	assert.NoError(t, err)
+	assert.ElementsMatch(t, diff.List(), []string{
+		"a1", "a1/f.yaml", "b1/f.yaml", "b1"})
+}
+
+// TestDiff_skipGitSrc verifies that .git directories in the source
+// are not looked at
+func TestDiff_skipGitSrc(t *testing.T) {
+	s, err := ioutil.TempDir("", "copyutilsrc")
+	assert.NoError(t, err)
+	d, err := ioutil.TempDir("", "copyutildest")
+	assert.NoError(t, err)
+
+	err = os.Mkdir(filepath.Join(s, "a1"), 0700)
+	assert.NoError(t, err)
+	err = ioutil.WriteFile(
+		filepath.Join(s, "a1", "f.yaml"), []byte(`a`), 0600)
+	assert.NoError(t, err)
+
+	// git should be ignored
+	err = os.Mkdir(filepath.Join(s, ".git"), 0700)
+	assert.NoError(t, err)
+	err = ioutil.WriteFile(
+		filepath.Join(s, ".git", "f.yaml"), []byte(`a`), 0600)
+	assert.NoError(t, err)
+
+	err = os.Mkdir(filepath.Join(d, "a1"), 0700)
+	assert.NoError(t, err)
+	err = ioutil.WriteFile(
+		filepath.Join(d, "a1", "f.yaml"), []byte(`a`), 0600)
+	assert.NoError(t, err)
+
+	diff, err := Diff(s, d)
+	assert.NoError(t, err)
+	assert.Empty(t, diff.List())
+}
+
+// TestDiff_skipGitDest verifies that .git directories in the destination
+// are not looked at
+func TestDiff_skipGitDest(t *testing.T) {
+	s, err := ioutil.TempDir("", "copyutilsrc")
+	assert.NoError(t, err)
+	d, err := ioutil.TempDir("", "copyutildest")
+	assert.NoError(t, err)
+
+	err = os.Mkdir(filepath.Join(s, "a1"), 0700)
+	assert.NoError(t, err)
+	err = ioutil.WriteFile(
+		filepath.Join(s, "a1", "f.yaml"), []byte(`a`), 0600)
+	assert.NoError(t, err)
+
+	err = os.Mkdir(filepath.Join(d, "a1"), 0700)
+	assert.NoError(t, err)
+	err = ioutil.WriteFile(
+		filepath.Join(d, "a1", "f.yaml"), []byte(`a`), 0600)
+	assert.NoError(t, err)
+
+	// git should be ignored
+	err = os.Mkdir(filepath.Join(d, ".git"), 0700)
+	assert.NoError(t, err)
+	err = ioutil.WriteFile(
+		filepath.Join(d, ".git", "f.yaml"), []byte(`a`), 0600)
+	assert.NoError(t, err)
+
+	diff, err := Diff(s, d)
+	assert.NoError(t, err)
+	assert.Empty(t, diff.List())
+}

kyaml/go.mod

@@ -0,0 +1,11 @@
+module sigs.k8s.io/kustomize/kyaml
+
+go 1.12
+
+require (
+	github.com/davecgh/go-spew v1.1.1
+	github.com/go-errors/errors v1.0.1
+	github.com/stretchr/testify v1.4.0
+	github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca
+	gopkg.in/yaml.v3 v3.0.0-20191026110619-0b21df46bc1d
+)

kyaml/go.sum

@@ -0,0 +1,18 @@
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/go-errors/errors v1.0.1 h1:LUHzmkK3GUKUrL/1gfBUxAHzcev3apQlezX/+O7ma6w=
+github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca h1:1CFlNzQhALwjS9mBAUkycX616GzgsuYUOCHA5+HSlXI=
+github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v3 v3.0.0-20191026110619-0b21df46bc1d h1:LCPbGQ34PMrwad11aMZ+dbz5SAsq/0ySjRwQ8I9Qwd8=
+gopkg.in/yaml.v3 v3.0.0-20191026110619-0b21df46bc1d/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

kyaml/kio/byteio_reader.go

@@ -0,0 +1,198 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package kio
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"sort"
+	"strings"
+
+	"sigs.k8s.io/kustomize/kyaml/kio/kioutil"
+	"sigs.k8s.io/kustomize/kyaml/yaml"
+)
+
+const (
+	ResourceListKind       = "ResourceList"
+	ResourceListApiVersion = "kyaml.kustomize.dev/v1alpha1"
+)
+
+// ByteReadWriter reads from an input and writes to an output
+type ByteReadWriter struct {
+	// Reader is where ResourceNodes are decoded from.
+	Reader io.Reader
+
+	// Writer is where ResourceNodes are encoded.
+	Writer io.Writer
+
+	// OmitReaderAnnotations will configures Read to skip setting the kyaml.kustomize.dev/kio/index
+	// annotation on Resources as they are Read.
+	OmitReaderAnnotations bool
+
+	// KeepReaderAnnotations if set will keep the Reader specific annotations when writing
+	// the Resources, otherwise they will be cleared.
+	KeepReaderAnnotations bool
+
+	// Style is a style that is set on the Resource Node Document.
+	Style yaml.Style
+
+	FunctionConfig *yaml.RNode
+
+	WrappingApiVersion string
+	WrappingKind       string
+}
+
+func (rw *ByteReadWriter) Read() ([]*yaml.RNode, error) {
+	b := &ByteReader{
+		Reader:                rw.Reader,
+		OmitReaderAnnotations: rw.OmitReaderAnnotations,
+	}
+	val, err := b.Read()
+	rw.FunctionConfig = b.FunctionConfig
+	rw.WrappingApiVersion = b.WrappingApiVersion
+	rw.WrappingKind = b.WrappingKind
+	return val, err
+}
+
+func (rw *ByteReadWriter) Write(nodes []*yaml.RNode) error {
+	return ByteWriter{
+		Writer:                rw.Writer,
+		KeepReaderAnnotations: rw.KeepReaderAnnotations,
+		Style:                 rw.Style,
+		FunctionConfig:        rw.FunctionConfig,
+		WrappingApiVersion:    rw.WrappingApiVersion,
+		WrappingKind:          rw.WrappingKind,
+	}.Write(nodes)
+}
+
+// ByteReader decodes ResourceNodes from bytes.
+// By default, Read will set the kyaml.kustomize.dev/kio/index annotation on each RNode as it
+// is read so they can be written back in the same order.
+type ByteReader struct {
+	// Reader is where ResourceNodes are decoded from.
+	Reader io.Reader
+
+	// OmitReaderAnnotations will configures Read to skip setting the kyaml.kustomize.dev/kio/index
+	// annotation on Resources as they are Read.
+	OmitReaderAnnotations bool
+
+	// SetAnnotations is a map of caller specified annotations to set on resources as they are read
+	// These are independent of the annotations controlled by OmitReaderAnnotations
+	SetAnnotations map[string]string
+
+	FunctionConfig *yaml.RNode
+
+	WrappingApiVersion string
+	WrappingKind       string
+}
+
+var _ Reader = &ByteReader{}
+
+func (r *ByteReader) Read() ([]*yaml.RNode, error) {
+	output := ResourceNodeSlice{}
+
+	// by manually splitting resources -- otherwise the decoder will get the Resource
+	// boundaries wrong for header comments.
+	input := &bytes.Buffer{}
+	_, err := io.Copy(input, r.Reader)
+	if err != nil {
+		return nil, err
+	}
+	values := strings.Split(input.String(), "\n---\n")
+
+	index := 0
+	for i := range values {
+		decoder := yaml.NewDecoder(bytes.NewBufferString(values[i]))
+		node, err := r.decode(index, decoder)
+		if err == io.EOF {
+			continue
+		}
+		if err != nil {
+			return nil, err
+		}
+		if yaml.IsMissingOrNull(node) {
+			// empty value
+			continue
+		}
+
+		// ok if no metadata -- assume not an InputList
+		meta, _ := node.GetMeta()
+
+		// the elements are wrapped in an InputList, unwrap them
+		// don't check apiVersion, we haven't standardized on the domain
+		if (meta.Kind == ResourceListKind || meta.Kind == "List") &&
+			node.Field("items") != nil {
+			r.WrappingKind = meta.Kind
+			r.WrappingApiVersion = meta.ApiVersion
+
+			// unwrap the list
+			fc := node.Field("functionConfig")
+			if fc != nil {
+				r.FunctionConfig = fc.Value
+			}
+
+			items := node.Field("items")
+			if items != nil {
+				for i := range items.Value.Content() {
+					// add items
+					output = append(output, yaml.NewRNode(items.Value.Content()[i]))
+				}
+
+			}
+			continue
+		}
+
+		// add the node to the list
+		output = append(output, node)
+
+		// increment the index annotation value
+		index++
+	}
+	return output, nil
+}
+
+func isEmptyDocument(node *yaml.Node) bool {
+	// node is a Document with no content -- e.g. "---\n---"
+	return node.Kind == yaml.DocumentNode &&
+		node.Content[0].Tag == yaml.NullNodeTag
+}
+
+func (r *ByteReader) decode(index int, decoder *yaml.Decoder) (*yaml.RNode, error) {
+	node := &yaml.Node{}
+	err := decoder.Decode(node)
+	if err == io.EOF {
+		return nil, io.EOF
+	}
+	if err != nil {
+		return nil, err
+	}
+
+	if isEmptyDocument(node) {
+		return nil, nil
+	}
+
+	// set annotations on the read Resources
+	// sort the annotations by key so the output Resources is consistent (otherwise the
+	// annotations will be in a random order)
+	n := yaml.NewRNode(node)
+	if r.SetAnnotations == nil {
+		r.SetAnnotations = map[string]string{}
+	}
+	if !r.OmitReaderAnnotations {
+		r.SetAnnotations[kioutil.IndexAnnotation] = fmt.Sprintf("%d", index)
+	}
+	var keys []string
+	for k := range r.SetAnnotations {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+	for _, k := range keys {
+		_, err = n.Pipe(yaml.SetAnnotation(k, r.SetAnnotations[k]))
+		if err != nil {
+			return nil, err
+		}
+	}
+	return yaml.NewRNode(node), nil
+}

kyaml/kio/byteio_reader_test.go

@@ -0,0 +1,310 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package kio_test
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	. "sigs.k8s.io/kustomize/kyaml/kio"
+)
+
+// getByteReaderTestInput returns test input
+func getByteReaderTestInput(t *testing.T) *bytes.Buffer {
+	b := &bytes.Buffer{}
+	_, err := b.WriteString(`
+---
+a: b # first resource
+c: d
+---
+# second resource
+e: f
+g:
+- h
+---
+---
+i: j
+`)
+	if !assert.NoError(t, err) {
+		assert.FailNow(t, "")
+	}
+	return b
+}
+
+func TestByteReader_Read_wrappedResourceßßList(t *testing.T) {
+	r := &ByteReader{Reader: bytes.NewBufferString(`apiVersion: kyaml.kustomize.dev/v1alpha1
+kind: ResourceList
+functionConfig:
+  foo: bar
+  elems:
+  - a
+  - b
+  - c
+items:
+-  kind: Deployment
+   spec:
+     replicas: 1
+- kind: Service
+  spec:
+    selectors:
+      foo: bar
+`)}
+	nodes, err := r.Read()
+	if !assert.NoError(t, err) {
+		return
+	}
+
+	// verify the contents
+	if !assert.Len(t, nodes, 2) {
+		return
+	}
+	expected := []string{
+		`kind: Deployment
+spec:
+  replicas: 1
+`,
+		`kind: Service
+spec:
+  selectors:
+    foo: bar
+`,
+	}
+	for i := range nodes {
+		if !assert.Equal(t, expected[i], nodes[i].MustString()) {
+			return
+		}
+	}
+
+	// verify the function config
+	assert.Equal(t, `foo: bar
+elems:
+- a
+- b
+- c
+`, r.FunctionConfig.MustString())
+
+	assert.Equal(t, ResourceListKind, r.WrappingKind)
+	assert.Equal(t, ResourceListApiVersion, r.WrappingApiVersion)
+
+}
+
+func TestByteReader_Read_wrappedList(t *testing.T) {
+	r := &ByteReader{Reader: bytes.NewBufferString(`apiVersion: v1
+kind: List
+items:
+-  kind: Deployment
+   spec:
+     replicas: 1
+- kind: Service
+  spec:
+    selectors:
+      foo: bar
+`)}
+	nodes, err := r.Read()
+	if !assert.NoError(t, err) {
+		return
+	}
+
+	// verify the contents
+	if !assert.Len(t, nodes, 2) {
+		return
+	}
+	expected := []string{
+		`kind: Deployment
+spec:
+  replicas: 1
+`,
+		`kind: Service
+spec:
+  selectors:
+    foo: bar
+`,
+	}
+	for i := range nodes {
+		if !assert.Equal(t, expected[i], nodes[i].MustString()) {
+			return
+		}
+	}
+
+	// verify the function config
+	assert.Nil(t, r.FunctionConfig)
+	assert.Equal(t, "List", r.WrappingKind)
+	assert.Equal(t, "v1", r.WrappingApiVersion)
+}
+
+// TestByteReader_Read tests the default Read behavior
+// - Resources are read into a slice
+// - ReaderAnnotations are set on the ResourceNodes
+func TestByteReader_Read(t *testing.T) {
+	nodes, err := (&ByteReader{Reader: getByteReaderTestInput(t)}).Read()
+	if !assert.NoError(t, err) {
+		return
+	}
+
+	if !assert.Len(t, nodes, 3) {
+		return
+	}
+	expected := []string{
+		`a: b # first resource
+c: d
+metadata:
+  annotations:
+    kyaml.kustomize.dev/kio/index: 0
+`,
+		`# second resource
+e: f
+g:
+- h
+metadata:
+  annotations:
+    kyaml.kustomize.dev/kio/index: 1
+`,
+		`i: j
+metadata:
+  annotations:
+    kyaml.kustomize.dev/kio/index: 2
+`,
+	}
+	for i := range nodes {
+		val, err := nodes[i].String()
+		if !assert.NoError(t, err) {
+			return
+		}
+		if !assert.Equal(t, expected[i], val) {
+			return
+		}
+	}
+}
+
+// TestByteReader_Read_omitReaderAnnotations tests
+// - Resources are read into a slice
+// - ReaderAnnotations are not set on the ResourceNodes
+func TestByteReader_Read_omitReaderAnnotations(t *testing.T) {
+	nodes, err := (&ByteReader{
+		Reader:                getByteReaderTestInput(t),
+		OmitReaderAnnotations: true}).Read()
+	if !assert.NoError(t, err) {
+		return
+	}
+
+	// should have parsed 3 resources
+	if !assert.Len(t, nodes, 3) {
+		return
+	}
+	expected := []string{
+		"a: b # first resource\nc: d\n",
+		"# second resource\ne: f\ng:\n- h\n",
+		"i: j\n",
+	}
+	for i := range nodes {
+		val, err := nodes[i].String()
+		if !assert.NoError(t, err) {
+			return
+		}
+		if !assert.Equal(t, expected[i], val) {
+			return
+		}
+	}
+}
+
+// TestByteReader_Read_omitReaderAnnotations tests
+// - Resources are read into a slice
+// - ReaderAnnotations are NOT set on the ResourceNodes
+// - Additional annotations ARE set on the ResourceNodes
+func TestByteReader_Read_setAnnotationsOmitReaderAnnotations(t *testing.T) {
+	nodes, err := (&ByteReader{
+		Reader:                getByteReaderTestInput(t),
+		SetAnnotations:        map[string]string{"foo": "bar"},
+		OmitReaderAnnotations: true,
+	}).Read()
+	if !assert.NoError(t, err) {
+		return
+	}
+
+	if !assert.Len(t, nodes, 3) {
+		return
+	}
+	expected := []string{
+		`a: b # first resource
+c: d
+metadata:
+  annotations:
+    foo: bar
+`,
+		`# second resource
+e: f
+g:
+- h
+metadata:
+  annotations:
+    foo: bar
+`,
+		`i: j
+metadata:
+  annotations:
+    foo: bar
+`,
+	}
+	for i := range nodes {
+		val, err := nodes[i].String()
+		if !assert.NoError(t, err) {
+			return
+		}
+		if !assert.Equal(t, expected[i], val) {
+			return
+		}
+	}
+}
+
+// TestByteReader_Read_omitReaderAnnotations tests
+// - Resources are read into a slice
+// - ReaderAnnotations ARE set on the ResourceNodes
+// - Additional annotations ARE set on the ResourceNodes
+func TestByteReader_Read_setAnnotations(t *testing.T) {
+	nodes, err := (&ByteReader{
+		Reader:         getByteReaderTestInput(t),
+		SetAnnotations: map[string]string{"foo": "bar"},
+	}).Read()
+	if !assert.NoError(t, err) {
+		return
+	}
+
+	if !assert.Len(t, nodes, 3) {
+		return
+	}
+	expected := []string{
+		`a: b # first resource
+c: d
+metadata:
+  annotations:
+    foo: bar
+    kyaml.kustomize.dev/kio/index: 0
+`,
+		`# second resource
+e: f
+g:
+- h
+metadata:
+  annotations:
+    foo: bar
+    kyaml.kustomize.dev/kio/index: 1
+`,
+		`i: j
+metadata:
+  annotations:
+    foo: bar
+    kyaml.kustomize.dev/kio/index: 2
+`,
+	}
+	for i := range nodes {
+		val, err := nodes[i].String()
+		if !assert.NoError(t, err) {
+			return
+		}
+		if !assert.Equal(t, expected[i], val) {
+			return
+		}
+	}
+}

kyaml/kio/byteio_writer.go

@@ -0,0 +1,122 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package kio
+
+import (
+	"io"
+
+	"sigs.k8s.io/kustomize/kyaml/kio/kioutil"
+	"sigs.k8s.io/kustomize/kyaml/yaml"
+)
+
+// Writer writes ResourceNodes to bytes.
+type ByteWriter struct {
+	// Writer is where ResourceNodes are encoded.
+	Writer io.Writer
+
+	// KeepReaderAnnotations if set will keep the Reader specific annotations when writing
+	// the Resources, otherwise they will be cleared.
+	KeepReaderAnnotations bool
+
+	// ClearAnnotations is a list of annotations to clear when writing the Resources.
+	ClearAnnotations []string
+
+	// Style is a style that is set on the Resource Node Document.
+	Style yaml.Style
+
+	// FunctionConfig is the function config for an ResourceList.  If non-nil
+	// wrap the results in an ResourceList.
+	FunctionConfig *yaml.RNode
+
+	// WrappingKind if set will cause ByteWriter to wrap the Resources in
+	// an 'items' field in this kind.  e.g. if WrappingKind is 'List',
+	// ByteWriter will wrap the Resources in a List .items field.
+	WrappingKind string
+
+	// WrappingApiVersion is the apiVersion for WrappingKind
+	WrappingApiVersion string
+
+	// Sort if set, will cause ByteWriter to sort the the nodes before writing them.
+	Sort bool
+}
+
+var _ Writer = ByteWriter{}
+
+func (w ByteWriter) Write(nodes []*yaml.RNode) error {
+	if w.Sort {
+		if err := kioutil.SortNodes(nodes); err != nil {
+			return err
+		}
+	}
+
+	encoder := yaml.NewEncoder(w.Writer)
+	defer encoder.Close()
+	for i := range nodes {
+
+		// clean resources by removing annotations set by the Reader
+		if !w.KeepReaderAnnotations {
+			_, err := nodes[i].Pipe(yaml.ClearAnnotation(kioutil.IndexAnnotation))
+			if err != nil {
+				return err
+			}
+		}
+		for _, a := range w.ClearAnnotations {
+			_, err := nodes[i].Pipe(yaml.ClearAnnotation(a))
+			if err != nil {
+				return err
+			}
+		}
+
+		// TODO(pwittrock): factor this into a a common module for pruning empty values
+		_, err := nodes[i].Pipe(yaml.Lookup("metadata"), yaml.FieldClearer{
+			Name: "annotations", IfEmpty: true})
+		if err != nil {
+			return err
+		}
+		_, err = nodes[i].Pipe(yaml.FieldClearer{Name: "metadata", IfEmpty: true})
+		if err != nil {
+			return err
+		}
+
+		if w.Style != 0 {
+			nodes[i].YNode().Style = w.Style
+		}
+	}
+
+	// don't wrap the elements
+	if w.WrappingKind == "" {
+		for i := range nodes {
+			err := encoder.Encode(nodes[i].Document())
+			if err != nil {
+				return err
+			}
+		}
+		return nil
+	}
+	// wrap the elements in a list
+	items := &yaml.Node{Kind: yaml.SequenceNode}
+	list := &yaml.Node{
+		Kind:  yaml.MappingNode,
+		Style: w.Style,
+		Content: []*yaml.Node{
+			{Kind: yaml.ScalarNode, Value: "apiVersion"},
+			{Kind: yaml.ScalarNode, Value: w.WrappingApiVersion},
+			{Kind: yaml.ScalarNode, Value: "kind"},
+			{Kind: yaml.ScalarNode, Value: w.WrappingKind},
+			{Kind: yaml.ScalarNode, Value: "items"}, items,
+		}}
+	if w.FunctionConfig != nil {
+		list.Content = append(list.Content,
+			&yaml.Node{Kind: yaml.ScalarNode, Value: "functionConfig"},
+			w.FunctionConfig.YNode())
+	}
+	doc := &yaml.Node{
+		Kind:    yaml.DocumentNode,
+		Style:   yaml.FoldedStyle,
+		Content: []*yaml.Node{list}}
+	for i := range nodes {
+		items.Content = append(items.Content, nodes[i].YNode())
+	}
+	return encoder.Encode(doc)
+}

kyaml/kio/byteio_writer_test.go

@@ -0,0 +1,284 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package kio
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"sigs.k8s.io/kustomize/kyaml/yaml"
+)
+
+// TestByteWriter_Write_withoutAnnotations tests:
+// - Resource Config ordering is preserved if no annotations are present
+func TestByteWriter_Write_wrapped(t *testing.T) {
+	node1, err := yaml.Parse(`a: b #first
+`)
+	if !assert.NoError(t, err) {
+		return
+	}
+	node2, err := yaml.Parse(`c: d # second
+`)
+	if !assert.NoError(t, err) {
+		return
+	}
+	node3, err := yaml.Parse(`e: f
+g:
+  h:
+  - i # has a list
+  - j
+`)
+	if !assert.NoError(t, err) {
+		return
+	}
+
+	buff := &bytes.Buffer{}
+	err = ByteWriter{
+		Sort:               true,
+		Writer:             buff,
+		FunctionConfig:     node3,
+		WrappingKind:       ResourceListKind,
+		WrappingApiVersion: ResourceListApiVersion}.
+		Write([]*yaml.RNode{node2, node1})
+	if !assert.NoError(t, err) {
+		return
+	}
+	assert.Equal(t, `apiVersion: kyaml.kustomize.dev/v1alpha1
+kind: ResourceList
+items:
+- c: d # second
+- a: b #first
+functionConfig:
+  e: f
+  g:
+    h:
+    - i # has a list
+    - j
+`, buff.String())
+}
+
+// TestByteWriter_Write_withoutAnnotations tests:
+// - Resource Config ordering is preserved if no annotations are present
+func TestByteWriter_Write_withoutAnnotations(t *testing.T) {
+	node1, err := yaml.Parse(`a: b #first
+`)
+	if !assert.NoError(t, err) {
+		return
+	}
+	node2, err := yaml.Parse(`c: d # second
+`)
+	if !assert.NoError(t, err) {
+		return
+	}
+	node3, err := yaml.Parse(`e: f
+g:
+  h:
+  - i # has a list
+  - j
+`)
+	if !assert.NoError(t, err) {
+		return
+	}
+
+	buff := &bytes.Buffer{}
+	err = ByteWriter{Writer: buff}.
+		Write([]*yaml.RNode{node2, node3, node1})
+	if !assert.NoError(t, err) {
+		return
+	}
+	assert.Equal(t, `c: d # second
+---
+e: f
+g:
+  h:
+  - i # has a list
+  - j
+---
+a: b #first
+`, buff.String())
+}
+
+// TestByteWriter_Write_withAnnotationsKeepAnnotations tests:
+// - Resource Config is sorted by annotations if present
+// - IndexAnnotations are retained
+func TestByteWriter_Write_withAnnotationsKeepAnnotations(t *testing.T) {
+	node1, err := yaml.Parse(`a: b #first
+metadata:
+  annotations:
+    kyaml.kustomize.dev/kio/index: 0
+    kyaml.kustomize.dev/kio/path: "a/b/a_test.yaml"
+`)
+	if !assert.NoError(t, err) {
+		return
+	}
+	node2, err := yaml.Parse(`c: d # second
+metadata:
+  annotations:
+    kyaml.kustomize.dev/kio/index: 1
+    kyaml.kustomize.dev/kio/path: "a/b/a_test.yaml"
+`)
+	if !assert.NoError(t, err) {
+		return
+	}
+	node3, err := yaml.Parse(`e: f
+g:
+  h:
+  - i # has a list
+  - j
+metadata:
+  annotations:
+    kyaml.kustomize.dev/kio/index: 0
+    kyaml.kustomize.dev/kio/path: "a/b/b_test.yaml"
+`)
+	if !assert.NoError(t, err) {
+		return
+	}
+
+	buff := &bytes.Buffer{}
+	err = ByteWriter{Sort: true, Writer: buff, KeepReaderAnnotations: true}.
+		Write([]*yaml.RNode{node2, node3, node1})
+	if !assert.NoError(t, err) {
+		return
+	}
+	assert.Equal(t, `a: b #first
+metadata:
+  annotations:
+    kyaml.kustomize.dev/kio/index: 0
+    kyaml.kustomize.dev/kio/path: "a/b/a_test.yaml"
+---
+c: d # second
+metadata:
+  annotations:
+    kyaml.kustomize.dev/kio/index: 1
+    kyaml.kustomize.dev/kio/path: "a/b/a_test.yaml"
+---
+e: f
+g:
+  h:
+  - i # has a list
+  - j
+metadata:
+  annotations:
+    kyaml.kustomize.dev/kio/index: 0
+    kyaml.kustomize.dev/kio/path: "a/b/b_test.yaml"
+`, buff.String())
+}
+
+// TestByteWriter_Write_withAnnotations tests:
+// - Resource Config is sorted by annotations if present
+// - IndexAnnotations are pruned
+func TestByteWriter_Write_withAnnotations(t *testing.T) {
+	node1, err := yaml.Parse(`a: b #first
+metadata:
+  annotations:
+    kyaml.kustomize.dev/kio/index: 0
+    kyaml.kustomize.dev/kio/path: "a/b/a_test.yaml"
+`)
+	if !assert.NoError(t, err) {
+		return
+	}
+	node2, err := yaml.Parse(`c: d # second
+metadata:
+  annotations:
+    kyaml.kustomize.dev/kio/index: 1
+    kyaml.kustomize.dev/kio/path: "a/b/a_test.yaml"
+`)
+	if !assert.NoError(t, err) {
+		return
+	}
+	node3, err := yaml.Parse(`e: f
+g:
+  h:
+  - i # has a list
+  - j
+metadata:
+  annotations:
+    kyaml.kustomize.dev/kio/index: 0
+    kyaml.kustomize.dev/kio/path: "a/b/b_test.yaml"
+`)
+	if !assert.NoError(t, err) {
+		return
+	}
+
+	buff := &bytes.Buffer{}
+	err = ByteWriter{Sort: true, Writer: buff}.
+		Write([]*yaml.RNode{node2, node3, node1})
+	if !assert.NoError(t, err) {
+		return
+	}
+	assert.Equal(t, `a: b #first
+metadata:
+  annotations:
+    kyaml.kustomize.dev/kio/path: "a/b/a_test.yaml"
+---
+c: d # second
+metadata:
+  annotations:
+    kyaml.kustomize.dev/kio/path: "a/b/a_test.yaml"
+---
+e: f
+g:
+  h:
+  - i # has a list
+  - j
+metadata:
+  annotations:
+    kyaml.kustomize.dev/kio/path: "a/b/b_test.yaml"
+`, buff.String())
+}
+
+// TestByteWriter_Write_partialValues tests:
+// - Resource Config is sorted when annotations are present on some but not all ResourceNodes
+func TestByteWriter_Write_partialAnnotations(t *testing.T) {
+	node1, err := yaml.Parse(`a: b #first
+metadata:
+  annotations:
+    kyaml.kustomize.dev/kio/path: "a/b/a_test.yaml"
+`)
+	if !assert.NoError(t, err) {
+		return
+	}
+	node2, err := yaml.Parse(`c: d # second
+metadata:
+  annotations:
+    kyaml.kustomize.dev/kio/index: 1
+    kyaml.kustomize.dev/kio/path: "a/b/a_test.yaml"
+`)
+	if !assert.NoError(t, err) {
+		return
+	}
+	node3, err := yaml.Parse(`e: f
+g:
+  h:
+  - i # has a list
+  - j
+`)
+	if !assert.NoError(t, err) {
+		return
+	}
+
+	buff := &bytes.Buffer{}
+	rw := ByteWriter{Sort: true, Writer: buff}
+	err = rw.Write([]*yaml.RNode{node2, node3, node1})
+	if !assert.NoError(t, err) {
+		return
+	}
+	assert.Equal(t, `e: f
+g:
+  h:
+  - i # has a list
+  - j
+---
+a: b #first
+metadata:
+  annotations:
+    kyaml.kustomize.dev/kio/path: "a/b/a_test.yaml"
+---
+c: d # second
+metadata:
+  annotations:
+    kyaml.kustomize.dev/kio/path: "a/b/a_test.yaml"
+`, buff.String())
+}

kyaml/kio/example_test.go

@@ -0,0 +1,110 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package kio_test
+
+import (
+	"bytes"
+	"log"
+	"os"
+
+	"sigs.k8s.io/kustomize/kyaml/kio"
+	"sigs.k8s.io/kustomize/kyaml/yaml"
+)
+
+func Example() {
+	input := bytes.NewReader([]byte(`apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: nginx
+  labels:
+    app: nginx
+spec:
+  replicas: 3
+  selector:
+    matchLabels:
+      app: nginx
+  template:
+    metadata:
+      labels:
+        app: nginx
+    spec:
+      containers:
+      - name: nginx
+        image: nginx:1.7.9
+        ports:
+        - containerPort: 80
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: nginx
+spec:
+  selector:
+    app: nginx
+  ports:
+    - protocol: TCP
+      port: 80
+      targetPort: 80
+`))
+
+	// setAnnotationFn
+	setAnnotationFn := kio.FilterFunc(func(operand []*yaml.RNode) ([]*yaml.RNode, error) {
+		for i := range operand {
+			resource := operand[i]
+			_, err := resource.Pipe(yaml.SetAnnotation("foo", "bar"))
+			if err != nil {
+				return nil, err
+			}
+		}
+		return operand, nil
+	})
+
+	err := kio.Pipeline{
+		Inputs:  []kio.Reader{&kio.ByteReader{Reader: input}},
+		Filters: []kio.Filter{setAnnotationFn},
+		Outputs: []kio.Writer{kio.ByteWriter{Writer: os.Stdout}},
+	}.Execute()
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	// Output:
+	// apiVersion: apps/v1
+	// kind: Deployment
+	// metadata:
+	//   name: nginx
+	//   labels:
+	//     app: nginx
+	//   annotations:
+	//     foo: bar
+	// spec:
+	//   replicas: 3
+	//   selector:
+	//     matchLabels:
+	//       app: nginx
+	//   template:
+	//     metadata:
+	//       labels:
+	//         app: nginx
+	//     spec:
+	//       containers:
+	//       - name: nginx
+	//         image: nginx:1.7.9
+	//         ports:
+	//         - containerPort: 80
+	// ---
+	// apiVersion: v1
+	// kind: Service
+	// metadata:
+	//   name: nginx
+	//   annotations:
+	//     foo: bar
+	// spec:
+	//   selector:
+	//     app: nginx
+	//   ports:
+	//   - protocol: TCP
+	//     port: 80
+	//     targetPort: 80
+}

kyaml/kio/filters/container.go

@@ -0,0 +1,171 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package filters
+
+import (
+	"bytes"
+	"os"
+	"os/exec"
+	"regexp"
+	"strings"
+
+	"sigs.k8s.io/kustomize/kyaml/kio"
+
+	"sigs.k8s.io/kustomize/kyaml/yaml"
+)
+
+// GrepFilter filters Resources using a container image.
+// The container must start a process that reads the list of
+// input Resources from stdin, reads the Configuration from the env
+// API_CONFIG, and writes the filtered Resources to stdout.
+// If there is a error or validation failure, the process must exit
+// non-zero.
+// The full set of environment variables from the parent process
+// are passed to the container.
+type ContainerFilter struct {
+	// Image is the container image to use to create a container.
+	Image string `yaml:"image,omitempty"`
+
+	// Config is the API configuration for the container and passed through the
+	// API_CONFIG env var to the container.
+	// Typically a Kubernetes style Resource Config.
+	Config *yaml.RNode `yaml:"config,omitempty"`
+
+	// args may be specified by tests to override how a container is spawned
+	args []string
+
+	checkInput func(string)
+}
+
+// GrepFilter implements kio.GrepFilter
+func (c *ContainerFilter) Filter(input []*yaml.RNode) ([]*yaml.RNode, error) {
+	// get the command to filter the Resources
+	cmd, err := c.getCommand()
+	if err != nil {
+		return nil, err
+	}
+
+	in := &bytes.Buffer{}
+	out := &bytes.Buffer{}
+
+	// write the input
+	err = kio.ByteWriter{
+		WrappingApiVersion: kio.ResourceListApiVersion,
+		WrappingKind:       kio.ResourceListKind,
+		Writer:             in, KeepReaderAnnotations: true, FunctionConfig: c.Config}.Write(input)
+	if err != nil {
+		return nil, err
+	}
+
+	// capture the command stdout for the return value
+	r := &kio.ByteReader{Reader: out}
+
+	// do the filtering
+	if c.checkInput != nil {
+		c.checkInput(in.String())
+	}
+	cmd.Stdin = in
+	cmd.Stdout = out
+	if err := cmd.Run(); err != nil {
+		return nil, err
+	}
+
+	return r.Read()
+}
+
+// getArgs returns the command + args to run to spawn the container
+func (c *ContainerFilter) getArgs() []string {
+	// run the container using docker.  this is simpler than using the docker
+	// libraries, and ensures things like auth work the same as if the container
+	// was run from the cli.
+	args := []string{"docker", "run",
+		"--rm",                                              // delete the container afterward
+		"-i", "-a", "STDIN", "-a", "STDOUT", "-a", "STDERR", // attach stdin, stdout, stderr
+
+		// added security options
+		"--network", "none", // disable the network
+		"--user", "nobody", // run as nobody
+		// don't make fs readonly because things like heredoc rely on writing tmp files
+		"--security-opt=no-new-privileges", // don't allow the user to escalate privileges
+	}
+
+	// export the local environment vars to the container
+	for _, pair := range os.Environ() {
+		args = append(args, "-e", strings.Split(pair, "=")[0])
+	}
+	return append(args, c.Image)
+
+}
+
+// getCommand returns a command which will apply the GrepFilter using the container image
+func (c *ContainerFilter) getCommand() (*exec.Cmd, error) {
+	// encode the filter command API configuration
+	cfg := &bytes.Buffer{}
+	if err := func() error {
+		e := yaml.NewEncoder(cfg)
+		defer e.Close()
+		// make it fit on a single line
+		c.Config.YNode().Style = yaml.FlowStyle
+		return e.Encode(c.Config.YNode())
+	}(); err != nil {
+		return nil, err
+	}
+
+	if len(c.args) == 0 {
+		c.args = c.getArgs()
+	}
+
+	cmd := exec.Command(c.args[0], c.args[1:]...)
+	cmd.Stderr = os.Stderr
+	cmd.Env = os.Environ()
+
+	// set stderr for err messaging
+	return cmd, nil
+}
+
+// IsReconcilerFilter filters Resources based on whether or not they are Reconciler Resource.
+// Resources with an apiVersion starting with '*.gcr.io', 'gcr.io' or 'docker.io' are considered
+// Reconciler Resources.
+type IsReconcilerFilter struct {
+	// ExcludeReconcilers if set to true, then Reconcilers will be excluded -- e.g.
+	// Resources with a reconcile container through the apiVersion (gcr.io prefix) or
+	// through the annotations
+	ExcludeReconcilers bool `yaml:"excludeReconcilers,omitempty"`
+
+	// IncludeNonReconcilers if set to true, the NonReconciler will be included.
+	IncludeNonReconcilers bool `yaml:"includeNonReconcilers,omitempty"`
+}
+
+// Filter implements kio.Filter
+func (c *IsReconcilerFilter) Filter(inputs []*yaml.RNode) ([]*yaml.RNode, error) {
+	var out []*yaml.RNode
+	for i := range inputs {
+		isContainerResource := GetContainerName(inputs[i]) != ""
+		if isContainerResource && !c.ExcludeReconcilers {
+			out = append(out, inputs[i])
+		}
+		if !isContainerResource && c.IncludeNonReconcilers {
+			out = append(out, inputs[i])
+		}
+	}
+	return out, nil
+}
+
+// GetContainerName returns the container image for an API if one exists
+func GetContainerName(n *yaml.RNode) string {
+	meta, _ := n.GetMeta()
+	container := meta.Annotations["kyaml.kustomize.dev/container"]
+	if container != "" {
+		return container
+	}
+
+	if match.MatchString(meta.ApiVersion) {
+		return meta.ApiVersion
+	}
+
+	return ""
+}
+
+// match specifies the set of apiVersions to recognize as being container images
+var match = regexp.MustCompile(`(docker\.io|.*\.?gcr\.io)/.*(:.*)?`)

kyaml/kio/filters/container_test.go

@@ -0,0 +1,281 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package filters
+
+import (
+	"bytes"
+	"os"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"sigs.k8s.io/kustomize/kyaml/kio"
+	"sigs.k8s.io/kustomize/kyaml/yaml"
+)
+
+func TestFilter_command(t *testing.T) {
+	cfg, err := yaml.Parse(`apiversion: apps/v1
+kind: Deployment
+metadata:
+  name: foo
+`)
+	if !assert.NoError(t, err) {
+		return
+	}
+	instance := &ContainerFilter{
+		Image:  "example.com:version",
+		Config: cfg,
+	}
+	os.Setenv("KYAML_TEST", "FOO")
+	cmd, err := instance.getCommand()
+	if !assert.NoError(t, err) {
+		return
+	}
+
+	expected := []string{
+		"docker", "run",
+		"--rm",
+		"-i", "-a", "STDIN", "-a", "STDOUT", "-a", "STDERR",
+		"--network", "none",
+		"--user", "nobody",
+		"--security-opt=no-new-privileges",
+	}
+	for _, e := range os.Environ() {
+		// the process env
+		expected = append(expected, "-e", strings.Split(e, "=")[0])
+	}
+	expected = append(expected, "example.com:version")
+	assert.Equal(t, expected, cmd.Args)
+
+	foundKyaml := false
+	for _, e := range cmd.Env {
+		// verify the command has the right environment variables to pass to the container
+		split := strings.Split(e, "=")
+		if split[0] == "KYAML_TEST" {
+			assert.Equal(t, "FOO", split[1])
+			foundKyaml = true
+		}
+	}
+	assert.True(t, foundKyaml)
+}
+
+func TestFilter_Filter(t *testing.T) {
+	cfg, err := yaml.Parse(`apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: foo
+`)
+	if !assert.NoError(t, err) {
+		return
+	}
+
+	input, err := (&kio.ByteReader{Reader: bytes.NewBufferString(`
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: deployment-foo
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: service-foo
+`)}).Read()
+	if !assert.NoError(t, err) {
+		return
+	}
+
+	called := false
+	result, err := (&ContainerFilter{
+		Image:  "example.com:version",
+		Config: cfg,
+		args:   []string{"sed", "s/Deployment/StatefulSet/g"},
+		checkInput: func(s string) {
+			called = true
+			if !assert.Equal(t, `apiVersion: kyaml.kustomize.dev/v1alpha1
+kind: ResourceList
+items:
+- apiVersion: apps/v1
+  kind: Deployment
+  metadata:
+    name: deployment-foo
+    annotations:
+      kyaml.kustomize.dev/kio/index: 0
+- apiVersion: v1
+  kind: Service
+  metadata:
+    name: service-foo
+    annotations:
+      kyaml.kustomize.dev/kio/index: 1
+functionConfig: {apiVersion: apps/v1, kind: Deployment, metadata: {name: foo}}
+`, s) {
+				t.FailNow()
+			}
+		},
+	}).Filter(input)
+	if !assert.NoError(t, err) {
+		return
+	}
+	if !assert.True(t, called) {
+		return
+	}
+
+	b := &bytes.Buffer{}
+	err = kio.ByteWriter{Writer: b, KeepReaderAnnotations: true}.Write(result)
+	if !assert.NoError(t, err) {
+		return
+	}
+
+	assert.Equal(t, `apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: deployment-foo
+  annotations:
+    kyaml.kustomize.dev/kio/index: 0
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: service-foo
+  annotations:
+    kyaml.kustomize.dev/kio/index: 1
+`, b.String())
+}
+
+func TestFilter_Filter_noChange(t *testing.T) {
+	cfg, err := yaml.Parse(`apiversion: apps/v1
+kind: Deployment
+metadata:
+  name: foo
+`)
+	if !assert.NoError(t, err) {
+		return
+	}
+
+	input, err := (&kio.ByteReader{Reader: bytes.NewBufferString(`
+apiversion: apps/v1
+kind: Deployment
+metadata:
+  name: deployment-foo
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: service-foo
+`)}).Read()
+	if !assert.NoError(t, err) {
+		return
+	}
+
+	called := false
+	result, err := (&ContainerFilter{
+		Image:  "example.com:version",
+		Config: cfg,
+		args:   []string{"sh", "-c", "cat <&0"},
+		checkInput: func(s string) {
+			called = true
+			if !assert.Equal(t, `apiVersion: kyaml.kustomize.dev/v1alpha1
+kind: ResourceList
+items:
+- apiversion: apps/v1
+  kind: Deployment
+  metadata:
+    name: deployment-foo
+    annotations:
+      kyaml.kustomize.dev/kio/index: 0
+- apiVersion: v1
+  kind: Service
+  metadata:
+    name: service-foo
+    annotations:
+      kyaml.kustomize.dev/kio/index: 1
+functionConfig: {apiversion: apps/v1, kind: Deployment, metadata: {name: foo}}
+`, s) {
+				t.FailNow()
+			}
+		},
+	}).Filter(input)
+	if !assert.NoError(t, err) {
+		return
+	}
+	if !assert.True(t, called) {
+		return
+	}
+
+	b := &bytes.Buffer{}
+	err = kio.ByteWriter{Writer: b, KeepReaderAnnotations: true}.Write(result)
+	if !assert.NoError(t, err) {
+		return
+	}
+
+	assert.Equal(t, `apiversion: apps/v1
+kind: Deployment
+metadata:
+  name: deployment-foo
+  annotations:
+    kyaml.kustomize.dev/kio/index: 0
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: service-foo
+  annotations:
+    kyaml.kustomize.dev/kio/index: 1
+`, b.String())
+}
+
+func Test_GetContainerName(t *testing.T) {
+	// make sure gcr.io works
+	n, err := yaml.Parse(`apiVersion: gcr.io/foo/bar:something
+kind: MyThing
+`)
+	if !assert.NoError(t, err) {
+		return
+	}
+	c := GetContainerName(n)
+	assert.Equal(t, "gcr.io/foo/bar:something", c)
+
+	// make sure regional gcr.io works
+	n, err = yaml.Parse(`apiVersion: us.gcr.io/foo/bar:something
+kind: MyThing
+`)
+	if !assert.NoError(t, err) {
+		return
+	}
+	c = GetContainerName(n)
+	assert.Equal(t, "us.gcr.io/foo/bar:something", c)
+
+	// container from annotation
+	n, err = yaml.Parse(`apiVersion: v1
+kind: MyThing
+metadata:
+  annotations:
+    kyaml.kustomize.dev/container: gcr.io/foo/bar:something
+`)
+	if !assert.NoError(t, err) {
+		return
+	}
+	c = GetContainerName(n)
+	assert.Equal(t, "gcr.io/foo/bar:something", c)
+
+	// doesn't have a container
+	n, err = yaml.Parse(`apiVersion: v1
+kind: MyThing
+metadata:
+`)
+	if !assert.NoError(t, err) {
+		return
+	}
+	c = GetContainerName(n)
+	assert.Equal(t, "", c)
+
+	// make sure docker.io works
+	n, err = yaml.Parse(`apiVersion: docker.io/foo/bar:something
+kind: MyThing
+`)
+	if !assert.NoError(t, err) {
+		return
+	}
+	c = GetContainerName(n)
+	assert.Equal(t, "docker.io/foo/bar:something", c)
+}

kyaml/kio/filters/filters.go

@@ -0,0 +1,198 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package filters
+
+import (
+	"fmt"
+	"sort"
+	"strings"
+
+	"sigs.k8s.io/kustomize/kyaml/kio"
+	"sigs.k8s.io/kustomize/kyaml/kio/kioutil"
+	"sigs.k8s.io/kustomize/kyaml/yaml"
+)
+
+// Filters are the list of known filters for unmarshalling a filter into a concrete
+// implementation.
+var Filters = map[string]func() kio.Filter{
+	"FileSetter":    func() kio.Filter { return &FileSetter{} },
+	"FormatFilter":  func() kio.Filter { return &FormatFilter{} },
+	"GrepFilter":    func() kio.Filter { return GrepFilter{} },
+	"MatchModifier": func() kio.Filter { return &MatchModifyFilter{} },
+	"Modifier":      func() kio.Filter { return &Modifier{} },
+}
+
+// filter wraps a kio.filter so that it can be unmarshalled from yaml.
+type KFilter struct {
+	kio.Filter
+}
+
+func (t KFilter) MarshalYAML() (interface{}, error) {
+	return t.Filter, nil
+}
+
+func (t *KFilter) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	i := map[string]interface{}{}
+	if err := unmarshal(i); err != nil {
+		return err
+	}
+	meta := &yaml.ResourceMeta{}
+	if err := unmarshal(meta); err != nil {
+		return err
+	}
+	if filter, found := Filters[meta.Kind]; !found {
+		var knownFilters []string
+		for k := range Filters {
+			knownFilters = append(knownFilters, k)
+		}
+		sort.Strings(knownFilters)
+		return fmt.Errorf("unsupported filter Kind %v:  may be one of: [%s]",
+			meta, strings.Join(knownFilters, ","))
+	} else {
+		t.Filter = filter()
+	}
+	return unmarshal(t.Filter)
+}
+
+// Modifier modifies the input Resources by invoking the provided pipeline.
+// Modifier will return any Resources for which the pipeline does not return an error.
+type Modifier struct {
+	Kind string `yaml:"kind,omitempty"`
+
+	Filters yaml.YFilters `yaml:"pipeline,omitempty"`
+}
+
+var _ kio.Filter = &Modifier{}
+
+func (f Modifier) Filter(input []*yaml.RNode) ([]*yaml.RNode, error) {
+	for i := range input {
+		if _, err := input[i].Pipe(f.Filters.Filters()...); err != nil {
+			return nil, err
+		}
+	}
+	return input, nil
+}
+
+type MatchModifyFilter struct {
+	Kind string `yaml:"kind,omitempty"`
+
+	MatchFilters []yaml.YFilters `yaml:"match,omitempty"`
+
+	ModifyFilters yaml.YFilters `yaml:"modify,omitempty"`
+}
+
+var _ kio.Filter = &MatchModifyFilter{}
+
+func (f MatchModifyFilter) Filter(input []*yaml.RNode) ([]*yaml.RNode, error) {
+	var matches = input
+	var err error
+	for _, filter := range f.MatchFilters {
+		matches, err = MatchFilter{Filters: filter}.Filter(matches)
+		if err != nil {
+			return nil, err
+		}
+	}
+	_, err = Modifier{Filters: f.ModifyFilters}.Filter(matches)
+	if err != nil {
+		return nil, err
+	}
+	return input, nil
+}
+
+type MatchFilter struct {
+	Kind string `yaml:"kind,omitempty"`
+
+	Filters yaml.YFilters `yaml:"pipeline,omitempty"`
+}
+
+var _ kio.Filter = &MatchFilter{}
+
+func (f MatchFilter) Filter(input []*yaml.RNode) ([]*yaml.RNode, error) {
+	var output []*yaml.RNode
+	for i := range input {
+		if v, err := input[i].Pipe(f.Filters.Filters()...); err != nil {
+			return nil, err
+		} else if v == nil {
+			continue
+		}
+		output = append(output, input[i])
+	}
+	return output, nil
+}
+
+type FilenameFmtVerb string
+
+const (
+	// KindFmt substitutes kind
+	KindFmt FilenameFmtVerb = "%k"
+
+	// NameFmt substitutes metadata.name
+	NameFmt FilenameFmtVerb = "%n"
+
+	// NamespaceFmt substitutes metdata.namespace
+	NamespaceFmt FilenameFmtVerb = "%s"
+)
+
+// FileSetter sets the file name and mode annotations on Resources.
+type FileSetter struct {
+	Kind string `yaml:"kind,omitempty"`
+
+	// FilenamePattern is the pattern to use for generating filenames.  FilenameFmtVerb
+	// FielnameFmtVerbs may be specified to substitute Resource metadata into the filename.
+	FilenamePattern string `yaml:"filenamePattern,omitempty"`
+
+	// Mode is the filemode to write.
+	Mode string `yaml:"mode,omitempty"`
+
+	// Override will override the existing filename if it is set on the pattern.
+	// Otherwise the existing filename is kept.
+	Override bool `yaml:"override,omitempty"`
+}
+
+var _ kio.Filter = &FileSetter{}
+
+const DefaultFilenamePattern = "%n_%k.yaml"
+
+func (f *FileSetter) Filter(input []*yaml.RNode) ([]*yaml.RNode, error) {
+	if f.Mode == "" {
+		f.Mode = fmt.Sprintf("%d", 0600)
+	}
+	if f.FilenamePattern == "" {
+		f.FilenamePattern = DefaultFilenamePattern
+	}
+
+	resources := map[string][]*yaml.RNode{}
+	for i := range input {
+		m, err := input[i].GetMeta()
+		if err != nil {
+			return nil, err
+		}
+		file := f.FilenamePattern
+		file = strings.Replace(file, string(KindFmt), strings.ToLower(m.Kind), -1)
+		file = strings.Replace(file, string(NameFmt), strings.ToLower(m.Name), -1)
+		file = strings.Replace(file, string(NamespaceFmt), strings.ToLower(m.Namespace), -1)
+
+		if _, found := m.Annotations[kioutil.PathAnnotation]; !found || f.Override {
+			if _, err := input[i].Pipe(yaml.SetAnnotation(kioutil.PathAnnotation, file)); err != nil {
+				return nil, err
+			}
+		}
+		resources[file] = append(resources[file], input[i])
+	}
+
+	var output []*yaml.RNode
+	for i := range resources {
+		if err := kioutil.SortNodes(resources[i]); err != nil {
+			return nil, err
+		}
+		for j := range resources[i] {
+			if _, err := resources[i][j].Pipe(
+				yaml.SetAnnotation(kioutil.IndexAnnotation, fmt.Sprintf("%d", j))); err != nil {
+				return nil, err
+			}
+			output = append(output, resources[i][j])
+		}
+	}
+	return output, nil
+}

kyaml/kio/filters/filters_test.go

@@ -0,0 +1,170 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package filters_test
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	. "sigs.k8s.io/kustomize/kyaml/kio"
+	. "sigs.k8s.io/kustomize/kyaml/kio/filters"
+)
+
+var r = `
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: foo1
+  namespace: bar
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: foo2
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: foo2
+  namespace: bar
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: foo1
+`
+
+func TestFileSetter_Filter(t *testing.T) {
+	in := bytes.NewBufferString(r)
+	out := &bytes.Buffer{}
+	err := Pipeline{
+		Inputs:  []Reader{&ByteReader{Reader: in}},
+		Filters: []Filter{&FileSetter{}},
+		Outputs: []Writer{ByteWriter{Sort: true, Writer: out}},
+	}.Execute()
+	if !assert.NoError(t, err) {
+		return
+	}
+	assert.Equal(t, `apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: foo1
+  namespace: bar
+  annotations:
+    kyaml.kustomize.dev/kio/path: foo1_deployment.yaml
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: foo1
+  annotations:
+    kyaml.kustomize.dev/kio/path: foo1_service.yaml
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: foo2
+  annotations:
+    kyaml.kustomize.dev/kio/path: foo2_deployment.yaml
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: foo2
+  namespace: bar
+  annotations:
+    kyaml.kustomize.dev/kio/path: foo2_service.yaml
+`, out.String())
+}
+
+func TestFileSetter_Filter_pattern(t *testing.T) {
+	in := bytes.NewBufferString(r)
+	out := &bytes.Buffer{}
+	err := Pipeline{
+		Inputs: []Reader{&ByteReader{Reader: in}},
+		Filters: []Filter{&FileSetter{
+			FilenamePattern: "%n_%s_%k.yaml",
+		}},
+		Outputs: []Writer{ByteWriter{Sort: true, Writer: out}},
+	}.Execute()
+	if !assert.NoError(t, err) {
+		return
+	}
+	assert.Equal(t, `apiVersion: v1
+kind: Service
+metadata:
+  name: foo1
+  annotations:
+    kyaml.kustomize.dev/kio/path: foo1__service.yaml
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: foo1
+  namespace: bar
+  annotations:
+    kyaml.kustomize.dev/kio/path: foo1_bar_deployment.yaml
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: foo2
+  annotations:
+    kyaml.kustomize.dev/kio/path: foo2__deployment.yaml
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: foo2
+  namespace: bar
+  annotations:
+    kyaml.kustomize.dev/kio/path: foo2_bar_service.yaml
+`, out.String())
+}
+
+func TestFileSetter_Filter_empty(t *testing.T) {
+	in := bytes.NewBufferString(r)
+	out := &bytes.Buffer{}
+	err := Pipeline{
+		Inputs: []Reader{&ByteReader{Reader: in}},
+		Filters: []Filter{&FileSetter{
+			FilenamePattern: "resource.yaml",
+		}},
+		Outputs: []Writer{ByteWriter{Writer: out}},
+	}.Execute()
+	if !assert.NoError(t, err) {
+		return
+	}
+	assert.Equal(t, `apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: foo1
+  namespace: bar
+  annotations:
+    kyaml.kustomize.dev/kio/path: resource.yaml
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: foo2
+  annotations:
+    kyaml.kustomize.dev/kio/path: resource.yaml
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: foo2
+  namespace: bar
+  annotations:
+    kyaml.kustomize.dev/kio/path: resource.yaml
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: foo1
+  annotations:
+    kyaml.kustomize.dev/kio/path: resource.yaml
+`, out.String())
+}

kyaml/kio/filters/fmtr.go

@@ -0,0 +1,220 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+// Package yamlfmt contains libraries for formatting yaml files containing
+// Kubernetes Resource configuration.
+//
+// Yaml files are formatted by:
+// - Sorting fields and map values
+// - Sorting unordered lists for whitelisted types
+// - Applying a canonical yaml Style
+//
+// Fields are ordered using a relative ordering applied to commonly
+// encountered Resource fields.  All Resources,  including non-builtin
+// Resources such as CRDs, share the same field precedence.
+//
+// Fields that do not appear in the explicit ordering are ordered
+// lexicographically.
+//
+// A subset of well known known unordered lists are sorted by element field
+// values.
+package filters
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"sort"
+
+	"sigs.k8s.io/kustomize/kyaml/kio"
+	"sigs.k8s.io/kustomize/kyaml/yaml"
+)
+
+// FormatInput returns the formatted input.
+func FormatInput(input io.Reader) (*bytes.Buffer, error) {
+	buff := &bytes.Buffer{}
+	err := kio.Pipeline{
+		Inputs:  []kio.Reader{&kio.ByteReader{Reader: input}},
+		Filters: []kio.Filter{FormatFilter{}},
+		Outputs: []kio.Writer{kio.ByteWriter{Writer: buff}},
+	}.Execute()
+
+	return buff, err
+}
+
+// FormatFileOrDirectory reads the file or directory and formats each file's
+// contents by writing it back to the file.
+func FormatFileOrDirectory(path string) error {
+	return kio.Pipeline{
+		Inputs: []kio.Reader{kio.LocalPackageReader{
+			PackagePath: path,
+		}},
+		Filters: []kio.Filter{FormatFilter{}},
+		Outputs: []kio.Writer{kio.LocalPackageWriter{PackagePath: path}},
+	}.Execute()
+}
+
+type FormatFilter struct{}
+
+var _ kio.Filter = FormatFilter{}
+
+func (f FormatFilter) Filter(slice []*yaml.RNode) ([]*yaml.RNode, error) {
+	for i := range slice {
+		kindNode, err := slice[i].Pipe(yaml.Get("kind"))
+		if err != nil {
+			return nil, err
+		}
+		if kindNode == nil {
+			continue
+		}
+		apiVersionNode, err := slice[i].Pipe(yaml.Get("apiVersion"))
+		if err != nil {
+			return nil, err
+		}
+		if apiVersionNode == nil {
+			continue
+		}
+		kind, apiVersion := kindNode.YNode().Value, apiVersionNode.YNode().Value
+		err = (&formatter{apiVersion: apiVersion, kind: kind}).fmtNode(slice[i].YNode(), "")
+		if err != nil {
+			return nil, err
+		}
+	}
+	return slice, nil
+}
+
+type formatter struct {
+	apiVersion string
+	kind       string
+}
+
+// fmtNode recursively formats the Document Contents.
+func (f *formatter) fmtNode(n *yaml.Node, path string) error {
+	n.Style = 0
+	// sort the order of mapping fields
+	if n.Kind == yaml.MappingNode {
+		sort.Sort(sortedMapContents(*n))
+	}
+
+	// sort the order of sequence elements if it is whitelisted
+	if n.Kind == yaml.SequenceNode {
+		if yaml.WhitelistedListSortKinds.Has(f.kind) &&
+			yaml.WhitelistedListSortApis.Has(f.apiVersion) {
+			if sortField, found := yaml.WhitelistedListSortFields[path]; found {
+				sort.Sort(sortedSeqContents{Node: *n, sortField: sortField})
+			}
+		}
+	}
+	for i := range n.Content {
+		p := path
+		if n.Kind == yaml.MappingNode && i%2 == 1 {
+			p = fmt.Sprintf("%s.%s", path, n.Content[i-1].Value)
+		}
+		err := f.fmtNode(n.Content[i], p)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// sortedMapContents sorts the Contents field of a MappingNode by the field names using a statically
+// defined field precedence, and falling back on lexicographical sorting
+type sortedMapContents yaml.Node
+
+func (s sortedMapContents) Len() int {
+	return len(s.Content) / 2
+}
+func (s sortedMapContents) Swap(i, j int) {
+	// yaml MappingNode Contents are a list of field names followed by
+	// field values, rather than a list of field <name, value> pairs.
+	// increment.
+	//
+	// e.g. ["field1Name", "field1Value", "field2Name", "field2Value"]
+	iFieldNameIndex := i * 2
+	jFieldNameIndex := j * 2
+	iFieldValueIndex := iFieldNameIndex + 1
+	jFieldValueIndex := jFieldNameIndex + 1
+
+	// swap field names
+	s.Content[iFieldNameIndex], s.Content[jFieldNameIndex] =
+		s.Content[jFieldNameIndex], s.Content[iFieldNameIndex]
+
+	// swap field values
+	s.Content[iFieldValueIndex], s.Content[jFieldValueIndex] = s.
+		Content[jFieldValueIndex], s.Content[iFieldValueIndex]
+}
+func (s sortedMapContents) Less(i, j int) bool {
+	iFieldNameIndex := i * 2
+	jFieldNameIndex := j * 2
+	iFieldName := s.Content[iFieldNameIndex].Value
+	jFieldName := s.Content[jFieldNameIndex].Value
+
+	// order by their precedence values looked up from the index
+	iOrder, foundI := yaml.FieldOrder[iFieldName]
+	jOrder, foundJ := yaml.FieldOrder[jFieldName]
+	if foundI && foundJ {
+		return iOrder < jOrder
+	}
+
+	// known fields come before unknown fields
+	if foundI {
+		return true
+	}
+	if foundJ {
+		return false
+	}
+
+	// neither field is known, sort them lexicographically
+	return iFieldName < jFieldName
+}
+
+// sortedSeqContents sorts the Contents field of a SequenceNode by the value of
+// the elements sortField.
+// e.g. it will sort spec.template.spec.containers by the value of the container `name` field
+type sortedSeqContents struct {
+	yaml.Node
+	sortField string
+}
+
+func (s sortedSeqContents) Len() int {
+	return len(s.Content)
+}
+func (s sortedSeqContents) Swap(i, j int) {
+	s.Content[i], s.Content[j] = s.Content[j], s.Content[i]
+}
+func (s sortedSeqContents) Less(i, j int) bool {
+	// primitive lists -- sort by the element's primitive values
+	if s.sortField == "" {
+		iValue := s.Content[i].Value
+		jValue := s.Content[j].Value
+		return iValue < jValue
+	}
+
+	// map lists -- sort by the element's sortField values
+	var iValue, jValue string
+	for a := range s.Content[i].Content {
+		if a%2 != 0 {
+			continue // not a fieldNameIndex
+		}
+		// locate the index of the sortField field
+		if s.Content[i].Content[a].Value == s.sortField {
+			// a is the yaml node for the field key, a+1 is the node for the field value
+			iValue = s.Content[i].Content[a+1].Value
+		}
+	}
+	for a := range s.Content[j].Content {
+		if a%2 != 0 {
+			continue // not a fieldNameIndex
+		}
+
+		// locate the index of the sortField field
+		if s.Content[j].Content[a].Value == s.sortField {
+			// a is the yaml node for the field key, a+1 is the node for the field value
+			jValue = s.Content[j].Content[a+1].Value
+		}
+	}
+
+	// compare the field values
+	return iValue < jValue
+}

kyaml/kio/filters/fmtr_test.go

@@ -0,0 +1,623 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package filters_test
+
+import (
+	"bytes"
+	"io"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	. "sigs.k8s.io/kustomize/kyaml/kio/filters"
+	"sigs.k8s.io/kustomize/kyaml/kio/filters/testyaml"
+)
+
+// TestFormatInput_configMap verifies a ConfigMap yaml is formatted correctly
+func TestFormatInput_configMap(t *testing.T) {
+	y := `
+
+
+# this formatting is intentionally weird
+
+apiVersion: v1
+# this is data
+data:
+  # this is color
+  color: purple
+  # that was color
+
+  # this is textmode
+  textmode: "true"
+  # this is how
+  how: fairlyNice
+
+
+
+kind: ConfigMap
+
+
+metadata:
+  selfLink: /api/v1/namespaces/default/configmaps/config-multi-env-files
+  namespace: default
+  creationTimestamp: 2017-12-27T18:38:34Z
+  name: config-multi-env-files
+  resourceVersion: "810136"
+  uid: 252c4572-eb35-11e7-887b-42010a8002b8  # keep no trailing linefeed`
+
+	expected := `# this formatting is intentionally weird
+
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: config-multi-env-files
+  namespace: default
+  creationTimestamp: 2017-12-27T18:38:34Z
+  resourceVersion: "810136"
+  selfLink: /api/v1/namespaces/default/configmaps/config-multi-env-files
+  uid: 252c4572-eb35-11e7-887b-42010a8002b8 # keep no trailing linefeed
+# this is data
+data:
+  # this is color
+  color: purple
+  # that was color
+
+  # this is how
+  how: fairlyNice
+  # this is textmode
+  textmode: "true"
+`
+
+	s, err := FormatInput(strings.NewReader(y))
+	assert.NoError(t, err)
+	assert.Equal(t, expected, s.String())
+}
+
+// TestFormatInput_deployment verifies a Deployment yaml is formatted correctly
+func TestFormatInput_deployment(t *testing.T) {
+	y := `
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: nginx-deployment
+  labels:
+    app: nginx
+spec:
+  selector:
+    matchLabels:
+      app: nginx
+  replicas: 3
+  template:
+    metadata:
+      labels:
+        app: nginx
+    spec:
+      containers:
+      # this is a container
+      - ports:
+        # this is a port
+        - containerPort: 80
+        name: b-nginx
+        image: nginx:1.7.9
+      # this is another container
+      - name: a-nginx
+        image: nginx:1.7.9
+        ports:
+        - containerPort: 80
+`
+	expected := `apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: nginx-deployment
+  labels:
+    app: nginx
+spec:
+  replicas: 3
+  selector:
+    matchLabels:
+      app: nginx
+  template:
+    metadata:
+      labels:
+        app: nginx
+    spec:
+      containers:
+      - # this is another container
+        name: a-nginx
+        image: nginx:1.7.9
+        ports:
+        - containerPort: 80
+      - name: b-nginx
+        image: nginx:1.7.9
+        # this is a container
+        ports:
+        - # this is a port
+          containerPort: 80
+`
+	s, err := FormatInput(strings.NewReader(y))
+	assert.NoError(t, err)
+	assert.Equal(t, expected, s.String())
+}
+
+// TestFormatInput_service verifies a Service yaml is formatted correctly
+func TestFormatInput_service(t *testing.T) {
+
+	y := `
+apiVersion: v1
+kind: Service
+metadata:
+  name: my-service
+spec:
+  selector:
+    app: MyApp
+  ports:
+    - protocol: TCP
+      port: 80
+      targetPort: 9376
+`
+	expected := `apiVersion: v1
+kind: Service
+metadata:
+  name: my-service
+spec:
+  selector:
+    app: MyApp
+  ports:
+  - protocol: TCP
+    port: 80
+    targetPort: 9376
+`
+	s, err := FormatInput(strings.NewReader(y))
+	assert.NoError(t, err)
+	assert.Equal(t, expected, s.String())
+}
+
+// TestFormatInput_service verifies a Service yaml is formatted correctly
+func TestFormatInput_validatingWebhookConfiguration(t *testing.T) {
+
+	y := `
+apiVersion: admissionregistration.k8s.io/v1beta1
+kind: ValidatingWebhookConfiguration
+metadata:
+  name: <name of this configuration object>
+webhooks:
+- name: <webhook name, e.g., pod-policy.example.io>
+  rules:
+  - apiGroups:
+    - ""
+    apiVersions:
+    - v1
+    operations:
+      - UPDATE # this list is indented by 2
+      - CREATE
+      - CONNECT
+    resources:
+    - pods # this list is not indented by 2
+    scope: "Namespaced"
+  clientConfig:
+    service:
+      namespace: <namespace of the front-end service>
+      name: <name of the front-end service>
+    caBundle: <pem encoded ca cert that signs the server cert used by the webhook>
+  admissionReviewVersions:
+  - v1beta1
+  timeoutSeconds: 1
+`
+	expected := `apiVersion: admissionregistration.k8s.io/v1beta1
+kind: ValidatingWebhookConfiguration
+metadata:
+  name: <name of this configuration object>
+webhooks:
+- name: <webhook name, e.g., pod-policy.example.io>
+  admissionReviewVersions:
+  - v1beta1
+  clientConfig:
+    service:
+      name: <name of the front-end service>
+      namespace: <namespace of the front-end service>
+    caBundle: <pem encoded ca cert that signs the server cert used by the webhook>
+  rules:
+  - resources:
+    - pods # this list is not indented by 2
+    apiGroups:
+    - ""
+    apiVersions:
+    - v1
+    operations:
+    - CONNECT
+    - CREATE
+    - UPDATE # this list is indented by 2
+    scope: Namespaced
+  timeoutSeconds: 1
+`
+	s, err := FormatInput(strings.NewReader(y))
+	assert.NoError(t, err)
+	assert.Equal(t, expected, s.String())
+}
+
+// TestFormatInput_unKnownType verifies an unknown type yaml is formatted correctly
+func TestFormatInput_unKnownType(t *testing.T) {
+	y := `
+spec:
+  template:
+    spec:
+      # these shouldn't be sorted because the type isn't whitelisted
+      containers:
+      - name: b
+      - name: a
+  replicas: 1
+status:
+  conditions:
+  - 3
+  - 1
+  - 2
+other:
+  b: a1
+  a: b1
+apiVersion: example.com/v1beta1
+kind: MyType
+`
+
+	expected := `apiVersion: example.com/v1beta1
+kind: MyType
+spec:
+  replicas: 1
+  template:
+    spec:
+      # these shouldn't be sorted because the type isn't whitelisted
+      containers:
+      - name: b
+      - name: a
+status:
+  conditions:
+  - 3
+  - 1
+  - 2
+other:
+  a: b1
+  b: a1
+`
+	s, err := FormatInput(strings.NewReader(y))
+	assert.NoError(t, err)
+	assert.Equal(t, expected, s.String())
+}
+
+// TestFormatInput_deployment verifies a Deployment yaml is formatted correctly
+func TestFormatInput_resources(t *testing.T) {
+	input := &bytes.Buffer{}
+	_, err := io.Copy(input, bytes.NewReader(testyaml.UnformattedYaml1))
+	assert.NoError(t, err)
+	_, err = io.Copy(input, strings.NewReader("---\n"))
+	assert.NoError(t, err)
+	_, err = io.Copy(input, bytes.NewReader(testyaml.UnformattedYaml2))
+	assert.NoError(t, err)
+
+	expectedOutput := &bytes.Buffer{}
+	_, err = io.Copy(expectedOutput, bytes.NewReader(testyaml.FormattedYaml1))
+	assert.NoError(t, err)
+	_, err = io.Copy(expectedOutput, strings.NewReader("---\n"))
+	assert.NoError(t, err)
+	_, err = io.Copy(expectedOutput, bytes.NewReader(testyaml.FormattedYaml2))
+	assert.NoError(t, err)
+
+	s, err := FormatInput(input)
+	assert.NoError(t, err)
+	assert.Equal(t, expectedOutput.String(), s.String())
+}
+
+//
+func TestFormatInput_failMissingKind(t *testing.T) {
+	y := `
+spec:
+  template:
+    spec:
+      containers:
+      - b
+      - a
+  replicas: 1
+status:
+  conditions:
+  - 3
+  - 1
+  - 2
+other:
+  b: a1
+  a: b1
+apiVersion: example.com/v1beta1
+`
+
+	b, err := FormatInput(strings.NewReader(y))
+	assert.NoError(t, err)
+	assert.Equal(t, strings.TrimLeft(y, "\n"), b.String())
+}
+
+func TestFormatInput_failMissingApiVersion(t *testing.T) {
+	y := `
+spec:
+  template:
+    spec:
+      containers:
+      - a
+      - b
+  replicas: 1
+status:
+  conditions:
+  - 3
+  - 1
+  - 2
+other:
+  b: a1
+  a: b1
+kind: MyKind
+`
+
+	b, err := FormatInput(strings.NewReader(y))
+	assert.NoError(t, err)
+	assert.Equal(t, strings.TrimLeft(y, "\n"), b.String())
+}
+
+func TestFormatInput_failUnmarshal(t *testing.T) {
+	y := `
+spec:
+  template:
+    spec:
+      containers:
+      - a
+      - b
+  replicas: 1
+status:
+  conditions:
+  - 3
+  - 1
+  - 2
+other:
+	b: a1
+	a: b1
+kind: MyKind
+apiVersion: example.com/v1beta1
+`
+
+	_, err := FormatInput(strings.NewReader(y))
+	assert.EqualError(t, err, "yaml: line 15: found character that cannot start any token")
+}
+
+// TestFormatFileOrDirectory_yamlExtFile verifies that FormatFileOrDirectory will format a file
+// with a .yaml extension.
+func TestFormatFileOrDirectory_yamlExtFile(t *testing.T) {
+	// write the unformatted file
+	f, err := ioutil.TempFile("", "yamlfmt*.yaml")
+	if !assert.NoError(t, err) {
+		return
+	}
+	defer os.Remove(f.Name())
+	err = ioutil.WriteFile(f.Name(), testyaml.UnformattedYaml1, 0600)
+	if !assert.NoError(t, err) {
+		return
+	}
+
+	// format the file
+	err = FormatFileOrDirectory(f.Name())
+	if !assert.NoError(t, err) {
+		return
+	}
+
+	// check the result is formatted
+	b, err := ioutil.ReadFile(f.Name())
+	if !assert.NoError(t, err) {
+		return
+	}
+	assert.Equal(t, string(testyaml.FormattedYaml1), string(b))
+}
+
+func TestFormatFileOrDirectory_multipleYamlEntries(t *testing.T) {
+	// write the unformatted file
+	f, err := ioutil.TempFile("", "yamlfmt*.yaml")
+	assert.NoError(t, err)
+	defer os.Remove(f.Name())
+	err = ioutil.WriteFile(f.Name(),
+		[]byte(string(testyaml.UnformattedYaml1)+"---\n"+string(testyaml.UnformattedYaml2)), 0600)
+	assert.NoError(t, err)
+
+	// format the file
+	err = FormatFileOrDirectory(f.Name())
+	assert.NoError(t, err)
+
+	// check the result is formatted
+	b, err := ioutil.ReadFile(f.Name())
+	assert.NoError(t, err)
+	assert.Equal(t, string(testyaml.FormattedYaml1)+"---\n"+string(testyaml.FormattedYaml2), string(b))
+}
+
+// TestFormatFileOrDirectory_ymlExtFile verifies that FormatFileOrDirectory will format a file
+// with a .yml extension.
+func TestFormatFileOrDirectory_ymlExtFile(t *testing.T) {
+	// write the unformatted file
+	f, err := ioutil.TempFile("", "yamlfmt*.yml")
+	assert.NoError(t, err)
+	defer os.Remove(f.Name())
+	err = ioutil.WriteFile(f.Name(), testyaml.UnformattedYaml1, 0600)
+	assert.NoError(t, err)
+
+	// format the file
+	err = FormatFileOrDirectory(f.Name())
+	assert.NoError(t, err)
+
+	// check the result is formatted
+	b, err := ioutil.ReadFile(f.Name())
+	assert.NoError(t, err)
+	assert.Equal(t, string(testyaml.FormattedYaml1), string(b))
+}
+
+// TestFormatFileOrDirectory_skipYamlExtFileWithJson verifies that the json content is formatted
+// as yaml
+func TestFormatFileOrDirectory_YamlExtFileWithJson(t *testing.T) {
+	// write the unformatted JSON file contents
+	f, err := ioutil.TempFile("", "yamlfmt*.yaml")
+	assert.NoError(t, err)
+	defer os.Remove(f.Name())
+	err = ioutil.WriteFile(f.Name(), testyaml.UnformattedJson1, 0600)
+	assert.NoError(t, err)
+
+	// format the file
+	err = FormatFileOrDirectory(f.Name())
+	assert.NoError(t, err)
+
+	// check the result is formatted as yaml
+	b, err := ioutil.ReadFile(f.Name())
+	assert.NoError(t, err)
+	assert.Equal(t, string(testyaml.FormattedYaml1), string(b))
+}
+
+// TestFormatFileOrDirectory_partialKubernetesYamlFile verifies that if a yaml file contains both
+// Kubernetes and non-Kubernetes documents, it will only format the Kubernetes documents
+func TestFormatFileOrDirectory_partialKubernetesYamlFile(t *testing.T) {
+	// write the unformatted file
+	f, err := ioutil.TempFile("", "yamlfmt*.yaml")
+	assert.NoError(t, err)
+	defer os.Remove(f.Name())
+	err = ioutil.WriteFile(f.Name(), []byte(string(testyaml.UnformattedYaml1)+`---
+status:
+  conditions:
+  - 3
+  - 1
+  - 2
+spec: a
+---
+`+string(testyaml.UnformattedYaml2)), 0600)
+	assert.NoError(t, err)
+
+	// format the file
+	err = FormatFileOrDirectory(f.Name())
+	assert.NoError(t, err)
+
+	// check the result is  NOT formatted
+	b, err := ioutil.ReadFile(f.Name())
+	assert.NoError(t, err)
+	assert.Equal(t, string(testyaml.FormattedYaml1)+`---
+status:
+  conditions:
+  - 3
+  - 1
+  - 2
+spec: a
+---
+`+string(testyaml.FormattedYaml2), string(b))
+}
+
+// TestFormatFileOrDirectory_nonKubernetesYamlFile verifies that if a yaml file does not contain
+// kubernetes
+func TestFormatFileOrDirectory_skipNonKubernetesYamlFile(t *testing.T) {
+	// write the unformatted JSON file contents
+	f, err := ioutil.TempFile("", "yamlfmt*.yaml")
+	assert.NoError(t, err)
+	defer os.Remove(f.Name())
+	err = ioutil.WriteFile(f.Name(), []byte(`
+status:
+  conditions:
+  - 3
+  - 1
+  - 2
+spec: a
+`), 0600)
+	assert.NoError(t, err)
+
+	// format the file
+	err = FormatFileOrDirectory(f.Name())
+	assert.NoError(t, err)
+
+	// check the result is formatted as yaml
+	b, err := ioutil.ReadFile(f.Name())
+	assert.NoError(t, err)
+	assert.Equal(t, `status:
+  conditions:
+  - 3
+  - 1
+  - 2
+spec: a
+`, string(b))
+}
+
+// TestFormatFileOrDirectory_jsonFile should not fmt the file even though it contains yaml.
+func TestFormatFileOrDirectory_skipJsonExtFile(t *testing.T) {
+	f, err := ioutil.TempFile("", "yamlfmt*.json")
+	assert.NoError(t, err)
+	defer os.Remove(f.Name())
+	err = ioutil.WriteFile(f.Name(), testyaml.UnformattedYaml1, 0600)
+	assert.NoError(t, err)
+
+	err = FormatFileOrDirectory(f.Name())
+	assert.NoError(t, err)
+
+	b, err := ioutil.ReadFile(f.Name())
+	assert.NoError(t, err)
+
+	assert.Equal(t, string(testyaml.UnformattedYaml1), string(b))
+}
+
+// TestFormatFileOrDirectory_directory verifies that yaml files will be formatted,
+// and other files will be ignored
+func TestFormatFileOrDirectory_directory(t *testing.T) {
+	d, err := ioutil.TempDir("", "yamlfmt")
+	assert.NoError(t, err)
+
+	err = os.Mkdir(filepath.Join(d, "config"), 0700)
+	assert.NoError(t, err)
+
+	err = ioutil.WriteFile(filepath.Join(d, "c1.yaml"), testyaml.UnformattedYaml1, 0600)
+	assert.NoError(t, err)
+
+	err = ioutil.WriteFile(filepath.Join(d, "config", "c2.yaml"), testyaml.UnformattedYaml2, 0600)
+	assert.NoError(t, err)
+
+	err = ioutil.WriteFile(filepath.Join(d, "README.md"), []byte(`# Markdown`), 0600)
+	assert.NoError(t, err)
+
+	err = FormatFileOrDirectory(d)
+	assert.NoError(t, err)
+
+	b, err := ioutil.ReadFile(filepath.Join(d, "c1.yaml"))
+	assert.NoError(t, err)
+	assert.Equal(t, string(testyaml.FormattedYaml1), string(b))
+
+	b, err = ioutil.ReadFile(filepath.Join(d, "config", "c2.yaml"))
+	assert.NoError(t, err)
+	assert.Equal(t, string(testyaml.FormattedYaml2), string(b))
+
+	b, err = ioutil.ReadFile(filepath.Join(d, "README.md"))
+	assert.NoError(t, err)
+	assert.Equal(t, `# Markdown`, string(b))
+
+	// verify no additional files were created
+	files := []string{
+		".", "c1.yaml", "README.md", "config", filepath.Join("config", "c2.yaml")}
+	err = filepath.Walk(d, func(path string, info os.FileInfo, err error) error {
+		assert.NoError(t, err)
+		path, err = filepath.Rel(d, path)
+		assert.NoError(t, err)
+		assert.Contains(t, files, path)
+		return nil
+	})
+	assert.NoError(t, err)
+}
+
+// TestFormatFileOrDirectory_trimWhiteSpace verifies that trailling and leading whitespace is
+// trimmed
+func TestFormatFileOrDirectory_trimWhiteSpace(t *testing.T) {
+	f, err := ioutil.TempFile("", "yamlfmt*.yaml")
+	assert.NoError(t, err)
+	defer os.Remove(f.Name())
+	err = ioutil.WriteFile(f.Name(), []byte("\n\n"+string(testyaml.UnformattedYaml1)+"\n\n"), 0600)
+	assert.NoError(t, err)
+
+	err = FormatFileOrDirectory(f.Name())
+	assert.NoError(t, err)
+
+	b, err := ioutil.ReadFile(f.Name())
+	assert.NoError(t, err)
+
+	assert.Equal(t, string(testyaml.FormattedYaml1), string(b))
+}

kyaml/kio/filters/grep.go

@@ -0,0 +1,117 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package filters
+
+import (
+	"regexp"
+	"strings"
+
+	"sigs.k8s.io/kustomize/kyaml/kio"
+	"sigs.k8s.io/kustomize/kyaml/yaml"
+)
+
+type GrepType int
+
+const (
+	Regexp GrepType = 1 << iota
+	GreaterThanEq
+	GreaterThan
+	LessThan
+	LessThanEq
+)
+
+// GrepFilter filters RNodes with a matching field
+type GrepFilter struct {
+	Path        []string `yaml:"path,omitempty"`
+	Value       string   `yaml:"value,omitempty"`
+	MatchType   GrepType `yaml:"matchType,omitempty"`
+	InvertMatch bool     `yaml:"invertMatch,omitempty"`
+	Compare     func(a, b string) (int, error)
+}
+
+var _ kio.Filter = GrepFilter{}
+
+func (f GrepFilter) Filter(input []*yaml.RNode) ([]*yaml.RNode, error) {
+	// compile the regular expression 1 time if we are matching using regex
+	var reg *regexp.Regexp
+	var err error
+	if f.MatchType == Regexp || f.MatchType == 0 {
+		reg, err = regexp.Compile(f.Value)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	var output kio.ResourceNodeSlice
+	for i := range input {
+		node := input[i]
+		val, err := node.Pipe(&yaml.PathMatcher{Path: f.Path})
+		if err != nil {
+			return nil, err
+		}
+		if val == nil || len(val.Content()) == 0 {
+			if f.InvertMatch {
+				output = append(output, input[i])
+			}
+			continue
+		}
+		found := false
+		err = val.VisitElements(func(elem *yaml.RNode) error {
+			// get the value
+			var str string
+			if f.MatchType == Regexp {
+				style := elem.YNode().Style
+				defer func() { elem.YNode().Style = style }()
+				elem.YNode().Style = yaml.FlowStyle
+				str, err = elem.String()
+				if err != nil {
+					return err
+				}
+				str = strings.TrimSpace(strings.Replace(str, `"`, "", -1))
+			} else {
+				// if not regexp, then it needs to parse into a quantity and comments will
+				// break that
+				str = elem.YNode().Value
+				if str == "" {
+					return nil
+				}
+			}
+
+			if f.MatchType == Regexp || f.MatchType == 0 {
+				if reg.MatchString(str) {
+					found = true
+				}
+				return nil
+			}
+
+			comp, err := f.Compare(str, f.Value)
+			if err != nil {
+				return err
+			}
+
+			if f.MatchType == GreaterThan && comp > 0 {
+				found = true
+			}
+			if f.MatchType == GreaterThanEq && comp >= 0 {
+				found = true
+			}
+			if f.MatchType == LessThan && comp < 0 {
+				found = true
+			}
+			if f.MatchType == LessThanEq && comp <= 0 {
+				found = true
+			}
+			return nil
+		})
+		if err != nil {
+			return nil, err
+		}
+		if found == f.InvertMatch {
+			continue
+		}
+
+		output = append(output, input[i])
+	}
+	return output, nil
+}

kyaml/kio/filters/grep_test.go

@@ -0,0 +1,157 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package filters_test
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"sigs.k8s.io/kustomize/kyaml/kio"
+	. "sigs.k8s.io/kustomize/kyaml/kio/filters"
+	"sigs.k8s.io/kustomize/kyaml/yaml"
+)
+
+func TestGrepFilter_Filter(t *testing.T) {
+	in := `kind: Deployment
+metadata:
+  labels:
+    app: nginx2
+  name: foo
+  annotations:
+    app: nginx2
+spec:
+  replicas: 1
+---
+kind: Deployment
+metadata:
+  labels:
+    app: nginx
+  annotations:
+    app: nginx
+  name: bar
+spec:
+  replicas: 3
+---
+kind: Service
+metadata:
+  name: foo
+  annotations:
+    app: nginx
+spec:
+  selector:
+    app: nginx
+`
+	out := &bytes.Buffer{}
+	err := kio.Pipeline{
+		Inputs:  []kio.Reader{&kio.ByteReader{Reader: bytes.NewBufferString(in)}},
+		Filters: []kio.Filter{GrepFilter{Path: []string{"metadata", "name"}, Value: "foo"}},
+		Outputs: []kio.Writer{kio.ByteWriter{Writer: out}},
+	}.Execute()
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
+
+	if !assert.Equal(t, `kind: Deployment
+metadata:
+  labels:
+    app: nginx2
+  name: foo
+  annotations:
+    app: nginx2
+spec:
+  replicas: 1
+---
+kind: Service
+metadata:
+  name: foo
+  annotations:
+    app: nginx
+spec:
+  selector:
+    app: nginx
+`, out.String()) {
+		t.FailNow()
+	}
+
+	out = &bytes.Buffer{}
+	err = kio.Pipeline{
+		Inputs:  []kio.Reader{&kio.ByteReader{Reader: bytes.NewBufferString(in)}},
+		Filters: []kio.Filter{GrepFilter{Path: []string{"kind"}, Value: "Deployment"}},
+		Outputs: []kio.Writer{kio.ByteWriter{Writer: out}},
+	}.Execute()
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
+	if !assert.Equal(t, `kind: Deployment
+metadata:
+  labels:
+    app: nginx2
+  name: foo
+  annotations:
+    app: nginx2
+spec:
+  replicas: 1
+---
+kind: Deployment
+metadata:
+  labels:
+    app: nginx
+  annotations:
+    app: nginx
+  name: bar
+spec:
+  replicas: 3
+`, out.String()) {
+		t.FailNow()
+	}
+
+	out = &bytes.Buffer{}
+	err = kio.Pipeline{
+		Inputs:  []kio.Reader{&kio.ByteReader{Reader: bytes.NewBufferString(in)}},
+		Filters: []kio.Filter{GrepFilter{Path: []string{"spec", "replicas"}, Value: "3"}},
+		Outputs: []kio.Writer{kio.ByteWriter{Writer: out}},
+	}.Execute()
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
+	if !assert.Equal(t, `kind: Deployment
+metadata:
+  labels:
+    app: nginx
+  annotations:
+    app: nginx
+  name: bar
+spec:
+  replicas: 3
+`, out.String()) {
+		t.FailNow()
+	}
+
+	out = &bytes.Buffer{}
+	err = kio.Pipeline{
+		Inputs:  []kio.Reader{&kio.ByteReader{Reader: bytes.NewBufferString(in)}},
+		Filters: []kio.Filter{GrepFilter{Path: []string{"spec", "not-present"}, Value: "3"}},
+		Outputs: []kio.Writer{kio.ByteWriter{Writer: out}},
+	}.Execute()
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
+	if !assert.Equal(t, ``, out.String()) {
+		t.FailNow()
+	}
+}
+
+func TestGrepFilter_init(t *testing.T) {
+	assert.Equal(t, GrepFilter{}, Filters["GrepFilter"]())
+}
+
+func TestGrepFilter_error(t *testing.T) {
+	v, err := GrepFilter{Path: []string{"metadata", "name"},
+		Value: "foo"}.Filter([]*yaml.RNode{{}})
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
+	assert.Nil(t, v)
+}

kyaml/kio/filters/merge.go

@@ -0,0 +1,76 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+// Package merge contains libraries for merging Resources and Patches
+package filters
+
+import (
+	"sigs.k8s.io/kustomize/kyaml/yaml"
+	"sigs.k8s.io/kustomize/kyaml/yaml/merge2"
+)
+
+// GrepFilter merges Resources with the Group/Version/Kind/Namespace/Name together using
+// a 2-way merge strategy.
+//
+// - Fields set to null in the source will be cleared from the destination
+// - Fields with matching keys will be merged recursively
+// - Lists with an associative key (e.g. name) will have their elements merged using the key
+// - List without an associative key will have the dest list replaced by the source list
+type MergeFilter struct {
+	Reverse bool
+}
+
+type mergeKey struct {
+	apiVersion string
+	kind       string
+	namespace  string
+	name       string
+}
+
+// GrepFilter implements kio.GrepFilter by merge Resources with the same G/V/K/NS/N
+func (c MergeFilter) Filter(input []*yaml.RNode) ([]*yaml.RNode, error) {
+	// invert the merge precedence
+	if c.Reverse {
+		for i, j := 0, len(input)-1; i < j; i, j = i+1, j-1 {
+			input[i], input[j] = input[j], input[i]
+		}
+	}
+
+	// index the Resources by G/V/K/NS/N
+	index := map[mergeKey][]*yaml.RNode{}
+	for i := range input {
+		meta, err := input[i].GetMeta()
+		if err != nil {
+			return nil, err
+		}
+		key := mergeKey{
+			apiVersion: meta.ApiVersion,
+			kind:       meta.Kind,
+			namespace:  meta.Namespace,
+			name:       meta.Name,
+		}
+		index[key] = append(index[key], input[i])
+	}
+
+	// merge each of the G/V/K/NS/N lists
+	var output []*yaml.RNode
+	var err error
+	for k := range index {
+		var merged *yaml.RNode
+		resources := index[k]
+		for i := range resources {
+			patch := resources[i]
+			if merged == nil {
+				// first resources, don't merge it
+				merged = resources[i]
+			} else {
+				merged, err = merge2.Merge(patch, merged)
+				if err != nil {
+					return nil, err
+				}
+			}
+		}
+		output = append(output, merged)
+	}
+	return output, nil
+}

kyaml/kio/filters/merge3.go

@@ -0,0 +1,167 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package filters
+
+import (
+	"fmt"
+
+	"sigs.k8s.io/kustomize/kyaml/kio"
+	"sigs.k8s.io/kustomize/kyaml/yaml"
+	"sigs.k8s.io/kustomize/kyaml/yaml/merge3"
+)
+
+const (
+	mergeSourceAnnotation = "kyaml.kustomize.dev/merge-source"
+	mergeSourceOriginal   = "original"
+	mergeSourceUpdated    = "updated"
+	mergeSourceDest       = "dest"
+)
+
+// Merge3 performs a 3-way merge on the original, updated, and destination packages.
+type Merge3 struct {
+	OriginalPath   string
+	UpdatedPath    string
+	DestPath       string
+	MatchFilesGlob []string
+}
+
+func (m Merge3) Merge() error {
+	// Read the destination package.  The ReadWriter will take take of deleting files
+	// for removed resources.
+	var inputs []kio.Reader
+	dest := &kio.LocalPackageReadWriter{
+		PackagePath:    m.DestPath,
+		MatchFilesGlob: m.MatchFilesGlob,
+		SetAnnotations: map[string]string{mergeSourceAnnotation: mergeSourceDest},
+	}
+	inputs = append(inputs, dest)
+
+	// Read the original package
+	inputs = append(inputs, kio.LocalPackageReader{
+		PackagePath:    m.OriginalPath,
+		MatchFilesGlob: m.MatchFilesGlob,
+		SetAnnotations: map[string]string{mergeSourceAnnotation: mergeSourceOriginal},
+	})
+
+	// Read the updated package
+	inputs = append(inputs, kio.LocalPackageReader{
+		PackagePath:    m.UpdatedPath,
+		MatchFilesGlob: m.MatchFilesGlob,
+		SetAnnotations: map[string]string{mergeSourceAnnotation: mergeSourceUpdated},
+	})
+
+	return kio.Pipeline{
+		Inputs:  inputs,
+		Filters: []kio.Filter{m, FormatFilter{}}, // format the merged output
+		Outputs: []kio.Writer{dest},
+	}.Execute()
+}
+
+// Filter combines Resources with the same GVK + N + NS into tuples, and then merges them
+func (m Merge3) Filter(nodes []*yaml.RNode) ([]*yaml.RNode, error) {
+	// index the nodes by their identity
+	tl := tuples{}
+	for i := range nodes {
+		if err := tl.add(nodes[i]); err != nil {
+			return nil, err
+		}
+	}
+
+	// iterate over the inputs, merging as needed
+	var output []*yaml.RNode
+	for i := range tl.list {
+		t := tl.list[i]
+		switch {
+		case t.original == nil && t.updated == nil && t.dest != nil:
+			// added locally -- keep dest
+			output = append(output, t.dest)
+		case t.original == nil && t.updated != nil && t.dest == nil:
+			// added in the update -- add update
+			output = append(output, t.updated)
+		case t.original != nil && t.updated == nil:
+			// deleted in the update
+		// don't include the resource in the output
+		case t.original != nil && t.dest == nil:
+			// deleted locally
+			// don't include the resource in the output
+		default:
+			// dest and updated are non-nil -- merge them
+			node, err := t.merge()
+			if err != nil {
+				return nil, err
+			}
+			if node != nil {
+				output = append(output, node)
+			}
+		}
+	}
+	return output, nil
+}
+
+// tuples combines nodes with the same GVK + N + NS
+type tuples struct {
+	list []*tuple
+}
+
+// add adds a node to the list, combining it with an existing matching Resource if found
+func (ts *tuples) add(node *yaml.RNode) error {
+	nodeMeta, err := node.GetMeta()
+	if err != nil {
+		return err
+	}
+	for i := range ts.list {
+		t := ts.list[i]
+		if t.meta.Name == nodeMeta.Name && t.meta.Namespace == nodeMeta.Namespace &&
+			t.meta.ApiVersion == nodeMeta.ApiVersion && t.meta.Kind == nodeMeta.Kind {
+			return t.add(node)
+		}
+	}
+	t := &tuple{meta: nodeMeta}
+	if err := t.add(node); err != nil {
+		return err
+	}
+	ts.list = append(ts.list, t)
+	return nil
+}
+
+// tuple wraps an original, updated, and dest tuple for a given Resource
+type tuple struct {
+	meta     yaml.ResourceMeta
+	original *yaml.RNode
+	updated  *yaml.RNode
+	dest     *yaml.RNode
+}
+
+// add sets the corresponding tuple field for the node
+func (t *tuple) add(node *yaml.RNode) error {
+	meta, err := node.GetMeta()
+	if err != nil {
+		return err
+	}
+	switch meta.Annotations[mergeSourceAnnotation] {
+	case mergeSourceDest:
+		if t.dest != nil {
+			return fmt.Errorf("dest source already specified")
+		}
+		t.dest = node
+	case mergeSourceOriginal:
+		if t.original != nil {
+			return fmt.Errorf("original source already specified")
+		}
+		t.original = node
+	case mergeSourceUpdated:
+		if t.updated != nil {
+			return fmt.Errorf("updated source already specified")
+		}
+		t.updated = node
+	default:
+		return fmt.Errorf("no source annotation for Resource")
+	}
+	return nil
+}
+
+// merge performs a 3-way merge on the tuple
+func (t *tuple) merge() (*yaml.RNode, error) {
+	return merge3.Merge(t.dest, t.original, t.updated)
+}

kyaml/kio/filters/merge3_test.go

@@ -0,0 +1,56 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package filters_test
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"runtime"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"sigs.k8s.io/kustomize/kyaml/copyutil"
+	"sigs.k8s.io/kustomize/kyaml/kio/filters"
+)
+
+func TestMerge3_Merge(t *testing.T) {
+	_, datadir, _, ok := runtime.Caller(0)
+	if !assert.True(t, ok) {
+		t.FailNow()
+	}
+	datadir = filepath.Join(filepath.Dir(datadir), "testdata")
+
+	// setup the local directory
+	dir, err := ioutil.TempDir("", "kyaml-test")
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
+	defer os.RemoveAll(dir)
+
+	if !assert.NoError(t, copyutil.CopyDir(
+		filepath.Join(datadir, "dataset1-localupdates"),
+		filepath.Join(dir, "dataset1"))) {
+		t.FailNow()
+	}
+
+	err = filters.Merge3{
+		OriginalPath: filepath.Join(datadir, "dataset1"),
+		UpdatedPath:  filepath.Join(datadir, "dataset1-remoteupdates"),
+		DestPath:     filepath.Join(dir, "dataset1"),
+	}.Merge()
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
+
+	diffs, err := copyutil.Diff(
+		filepath.Join(dir, "dataset1"),
+		filepath.Join(datadir, "dataset1-expected"))
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
+	if !assert.Empty(t, diffs.List()) {
+		t.FailNow()
+	}
+}

kyaml/kio/filters/modify.go

@@ -0,0 +1,4 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package filters

kyaml/kio/filters/stripcomments.go

@@ -0,0 +1,43 @@
+// Copyright 2019 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package filters
+
+import (
+	"sigs.k8s.io/kustomize/kyaml/kio"
+	"sigs.k8s.io/kustomize/kyaml/yaml"
+)
+
+type StripCommentsFilter struct{}
+
+var _ kio.Filter = StripCommentsFilter{}
+
+func (f StripCommentsFilter) Filter(slice []*yaml.RNode) ([]*yaml.RNode, error) {
+	for i := range slice {
+		stripComments(slice[i].YNode())
+	}
+	return slice, nil
+}
+
+func stripComments(node *yaml.Node) {
+	if node == nil {
+		return
+	}
+	node.HeadComment = ""
+	node.LineComment = ""
+	node.FootComment = ""
+	for i := range node.Content {
+		stripComments(node.Content[i])
+	}
+}

kyaml/kio/filters/testdata/dataset1-expected/java/java-configmap.resource.yaml

@@ -0,0 +1,11 @@
+# Copyright 2019 The Kubernetes Authors.
+# SPDX-License-Identifier: Apache-2.0
+# 
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: app-config
+  labels:
+    app.kubernetes.io/component: undefined
+    app.kubernetes.io/instance: undefined
+data: {}

kyaml/kio/filters/testdata/dataset1-expected/java/java-deployment.resource.yaml

@@ -0,0 +1,43 @@
+# Copyright 2019 The Kubernetes Authors.
+# SPDX-License-Identifier: Apache-2.0
+# 
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: app
+  labels:
+    app: java
+    new-local: label
+    new-remote: label
+spec:
+  replicas: 3
+  selector:
+    matchLabels:
+      app: java
+  template:
+    metadata:
+      labels:
+        app: java
+    spec:
+      restartPolicy: Always
+      containers:
+      - name: app
+        image: gcr.io/project/app:version
+        command:
+        - java
+        - -jar
+        - /app.jar
+        - otherstuff
+        args:
+        - foo
+        ports:
+        - containerPort: 8080
+        envFrom:
+        - configMapRef:
+            name: app-config
+        env:
+        - name: JAVA_OPTS
+          value: -XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap
+            -Djava.security.egd=file:/dev/./urandom
+        imagePullPolicy: Always
+  minReadySeconds: 20

kyaml/kio/filters/testdata/dataset1-expected/java/java-service.resource.yaml

@@ -0,0 +1,16 @@
+# Copyright 2019 The Kubernetes Authors.
+# SPDX-License-Identifier: Apache-2.0
+# 
+apiVersion: v1
+kind: Service
+metadata:
+  name: app
+  labels:
+    app: java
+spec:
+  selector:
+    app: java
+  ports:
+  - name: "8080"
+    port: 8080
+    targetPort: 8080

kyaml/kio/filters/testdata/dataset1-expected/mysql/mysql-configmap.resource.yaml

@@ -0,0 +1,23 @@
+# Copyright 2019 The Kubernetes Authors.
+# SPDX-License-Identifier: Apache-2.0
+# 
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: mysql
+  labels:
+    app: mysql
+  annotations:
+    file/index: "0"
+    file/path: mysql-configmap.resource.yaml
+    package/name: mysql
+    package/original-name: mysql
+data:
+  master.cnf: |
+    # Apply this config only on the master.
+    [mysqld]
+    log-bin
+  slave.cnf: |
+    # Apply this config only on slaves.
+    [mysqld]
+    super-read-only

kyaml/kio/filters/testdata/dataset1-expected/mysql/mysql-service.resource.yaml

@@ -0,0 +1,29 @@
+# Copyright 2019 The Kubernetes Authors.
+# SPDX-License-Identifier: Apache-2.0
+# 
+apiVersion: v1
+kind: Service
+metadata:
+  name: mysql
+  labels:
+    app: mysql
+spec:
+  selector:
+    app: mysql
+  ports:
+  - name: mysql
+    port: 3306
+  clusterIP: None
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: mysql-read
+  labels:
+    app: mysql
+spec:
+  selector:
+    app: mysql
+  ports:
+  - name: mysql
+    port: 3306

kyaml/kio/filters/testdata/dataset1-expected/mysql/mysql-statefulset.resource.yaml

@@ -0,0 +1,173 @@
+# Copyright 2019 The Kubernetes Authors.
+# SPDX-License-Identifier: Apache-2.0
+# 
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: mysql
+spec:
+  replicas: 3
+  selector:
+    matchLabels:
+      app: mysql
+  template:
+    metadata:
+      labels:
+        app: mysql
+    spec:
+      initContainers:
+      - name: init-mysql
+        image: mysql:5.7
+        command:
+        - bash
+        - -c
+        - |
+          set -ex
+          # Generate mysql server-id from pod ordinal index.
+          [[ `hostname` =~ -([0-9]+)$ ]] || exit 1
+          ordinal=${BASH_REMATCH[1]}
+          echo [mysqld] > /mnt/conf.d/server-id.cnf
+          # Add an offset to avoid reserved server-id=0 value.
+          echo server-id=$((100 + $ordinal)) >> /mnt/conf.d/server-id.cnf
+          # Copy appropriate conf.d files from config-map to emptyDir.
+          if [[ $ordinal -eq 0 ]]; then
+            cp /mnt/config-map/master.cnf /mnt/conf.d/
+          else
+            cp /mnt/config-map/slave.cnf /mnt/conf.d/
+          fi
+        volumeMounts:
+        - name: conf
+          mountPath: /mnt/conf.d
+        - name: config-map
+          mountPath: /mnt/config-map
+      - name: clone-mysql
+        image: gcr.io/google-samples/xtrabackup:1.0
+        command:
+        - bash
+        - -c
+        - |
+          set -ex
+          # Skip the clone if data already exists.
+          [[ -d /var/lib/mysql/mysql ]] && exit 0
+          # Skip the clone on master (ordinal index 0).
+          [[ `hostname` =~ -([0-9]+)$ ]] || exit 1
+          ordinal=${BASH_REMATCH[1]}
+          [[ $ordinal -eq 0 ]] && exit 0
+          # Clone data from previous peer.
+          ncat --recv-only mysql-$(($ordinal-1)).mysql 3307 | xbstream -x -C /var/lib/mysql
+          # Prepare the backup.
+          xtrabackup --prepare --target-dir=/var/lib/mysql
+        volumeMounts:
+        - name: data
+          mountPath: /var/lib/mysql
+          subPath: mysql
+        - name: conf
+          mountPath: /etc/mysql/conf.d
+      containers:
+      - name: mysql
+        image: mysql:5.7
+        ports:
+        - name: mysql
+          containerPort: 3306
+        env:
+        - name: MYSQL_ALLOW_EMPTY_PASSWORD
+          value: "1"
+        resources:
+          requests:
+            cpu: 500m
+            memory: 1Gi
+        volumeMounts:
+        - name: data
+          mountPath: /var/lib/mysql
+          subPath: mysql
+        - name: conf
+          mountPath: /etc/mysql/conf.d
+        livenessProbe:
+          exec:
+            command:
+            - mysqladmin
+            - ping
+          initialDelaySeconds: 30
+          periodSeconds: 10
+          timeoutSeconds: 5
+        readinessProbe:
+          exec:
+            command:
+            - mysql
+            - -h
+            - 127.0.0.1
+            - -e
+            - SELECT 1
+          initialDelaySeconds: 5
+          periodSeconds: 2
+          timeoutSeconds: 1
+      - name: xtrabackup
+        image: gcr.io/google-samples/xtrabackup:1.0
+        command:
+        - bash
+        - -c
+        - |
+          set -ex
+          cd /var/lib/mysql
+          # Determine binlog position of cloned data, if any.
+          if [[ -f xtrabackup_slave_info ]]; then
+            # XtraBackup already generated a partial "CHANGE MASTER TO" query
+            # because we're cloning from an existing slave.
+            mv xtrabackup_slave_info change_master_to.sql.in
+            # Ignore xtrabackup_binlog_info in this case (it's useless).
+            rm -f xtrabackup_binlog_info
+          elif [[ -f xtrabackup_binlog_info ]]; then
+            # We're cloning directly from master. Parse binlog position.
+            [[ `cat xtrabackup_binlog_info` =~ ^(.*?)[[:space:]]+(.*?)$ ]] || exit 1
+            rm xtrabackup_binlog_info
+            echo "CHANGE MASTER TO MASTER_LOG_FILE='${BASH_REMATCH[1]}',\
+                  MASTER_LOG_POS=${BASH_REMATCH[2]}" > change_master_to.sql.in
+          fi
+          # Check if we need to complete a clone by starting replication.
+          if [[ -f change_master_to.sql.in ]]; then
+            echo "Waiting for mysqld to be ready (accepting connections)"
+            until mysql -h 127.0.0.1 -e "SELECT 1"; do sleep 1; done
+            echo "Initializing replication from clone position"
+            # In case of container restart, attempt this at-most-once.
+            mv change_master_to.sql.in change_master_to.sql.orig
+            mysql -h 127.0.0.1 <<EOF
+          $(<change_master_to.sql.orig),
+            MASTER_HOST='mysql-0.mysql',
+            MASTER_USER='root',
+            MASTER_PASSWORD='',
+            MASTER_CONNECT_RETRY=10;
+          START SLAVE;
+          EOF
+          fi
+          # Start a server to send backups when requested by peers.
+          exec ncat --listen --keep-open --send-only --max-conns=1 3307 -c \
+            "xtrabackup --backup --slave-info --stream=xbstream --host=127.0.0.1 --user=root"
+        ports:
+        - name: xtrabackup
+          containerPort: 3307
+        resources:
+          requests:
+            cpu: 100m
+            memory: 100Mi
+        volumeMounts:
+        - name: data
+          mountPath: /var/lib/mysql
+          subPath: mysql
+        - name: conf
+          mountPath: /etc/mysql/conf.d
+      volumes:
+      - name: conf
+        emptyDir: {}
+      - name: config-map
+        configMap:
+          name: mysql
+  volumeClaimTemplates:
+  - metadata:
+      name: data
+    spec:
+      resources:
+        requests:
+          storage: 10Gi
+      accessModes:
+      - ReadWriteOnce
+  serviceName: mysql

kyaml/kio/filters/testdata/dataset1-expected/wordpress/wordpress-service.resource.yaml

@@ -0,0 +1,31 @@
+# Copyright 2019 The Kubernetes Authors.
+# SPDX-License-Identifier: Apache-2.0
+# 
+apiVersion: v1
+kind: Service
+metadata:
+  name: wordpress
+  labels:
+    app: wordpress
+spec:
+  type: NodePort
+  selector:
+    app: wordpress
+    tier: frontend
+  ports:
+  - port: 80
+    nodePort: 30000
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: wordpress-identity
+  labels:
+    app: wordpress-identity
+spec:
+  selector:
+    app: wordpress
+    tier: frontend
+  ports:
+  - port: 80
+    nodePort: 30000

kyaml/kio/filters/testdata/dataset1-expected/wordpress/wordpress-statefulset.resource.yaml

@@ -0,0 +1,42 @@
+# Copyright 2019 The Kubernetes Authors.
+# SPDX-License-Identifier: Apache-2.0
+# 
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: wordpress
+  labels:
+    app: wordpress
+spec:
+  selector:
+    matchLabels:
+      app: wordpress
+      tier: frontend
+  template:
+    metadata:
+      labels:
+        app: wordpress
+        tier: frontend
+    spec:
+      containers:
+      - name: wordpress
+        image: buddy/wordpress:latest
+        ports:
+        - name: wordpress
+          containerPort: 80
+        env:
+        - name: WORDPRESS_DB_HOST
+          value: wordpress-mysql
+        - name: WORDPRESS_DB_PASSWORD
+          valueFrom:
+            secretKeyRef:
+              name: mysql-pass
+              key: password
+        volumeMounts:
+        - name: wordpress-persistent-storage
+          mountPath: /var/www/html
+      volumes:
+      - name: wordpress-persistent-storage
+        persistentVolumeClaim:
+          claimName: wp-pv-claim
+  serviceName: wordpress-identity

kyaml/kio/filters/testdata/dataset1-localupdates/java/java-configmap.resource.yaml

@@ -0,0 +1,11 @@
+# Copyright 2019 The Kubernetes Authors.
+# SPDX-License-Identifier: Apache-2.0
+# 
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: app-config
+  labels:
+    app.kubernetes.io/component: undefined
+    app.kubernetes.io/instance: undefined
+data: {}

kyaml/kio/filters/testdata/dataset1-localupdates/java/java-deployment.resource.yaml

@@ -0,0 +1,41 @@
+# Copyright 2019 The Kubernetes Authors.
+# SPDX-License-Identifier: Apache-2.0
+# 
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: app
+  labels:
+    app: java
+    new-local: label
+spec:
+  replicas: 2
+  selector:
+    matchLabels:
+      app: java
+  template:
+    metadata:
+      labels:
+        app: java
+    spec:
+      restartPolicy: Always
+      containers:
+      - name: app
+        image: gcr.io/project/app:version
+        command:
+        - java
+        - -jar
+        - /app.jar
+        args:
+        - foo
+        ports:
+        - containerPort: 8080
+        envFrom:
+        - configMapRef:
+            name: app-config
+        env:
+        - name: JAVA_OPTS
+          value: -XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap
+            -Djava.security.egd=file:/dev/./urandom
+        imagePullPolicy: Always
+  minReadySeconds: 20

kyaml/kio/filters/testdata/dataset1-localupdates/java/java-service.resource.yaml

@@ -0,0 +1,16 @@
+# Copyright 2019 The Kubernetes Authors.
+# SPDX-License-Identifier: Apache-2.0
+# 
+apiVersion: v1
+kind: Service
+metadata:
+  name: app
+  labels:
+    app: java
+spec:
+  selector:
+    app: java
+  ports:
+  - name: "8080"
+    port: 8080
+    targetPort: 8080

kyaml/kio/filters/testdata/dataset1-localupdates/wordpress/wordpress-service.resource.yaml

@@ -0,0 +1,31 @@
+# Copyright 2019 The Kubernetes Authors.
+# SPDX-License-Identifier: Apache-2.0
+# 
+apiVersion: v1
+kind: Service
+metadata:
+  name: wordpress
+  labels:
+    app: wordpress
+spec:
+  type: NodePort
+  selector:
+    app: wordpress
+    tier: frontend
+  ports:
+  - port: 80
+    nodePort: 30000
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: wordpress-identity
+  labels:
+    app: wordpress-identity
+spec:
+  selector:
+    app: wordpress
+    tier: frontend
+  ports:
+  - port: 80
+    nodePort: 30000

kyaml/kio/filters/testdata/dataset1-localupdates/wordpress/wordpress-statefulset.resource.yaml

@@ -0,0 +1,42 @@
+# Copyright 2019 The Kubernetes Authors.
+# SPDX-License-Identifier: Apache-2.0
+# 
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: wordpress
+  labels:
+    app: wordpress
+spec:
+  selector:
+    matchLabels:
+      app: wordpress
+      tier: frontend
+  template:
+    metadata:
+      labels:
+        app: wordpress
+        tier: frontend
+    spec:
+      containers:
+      - name: wordpress
+        image: buddy/wordpress:latest
+        ports:
+        - name: wordpress
+          containerPort: 80
+        env:
+        - name: WORDPRESS_DB_HOST
+          value: wordpress-mysql
+        - name: WORDPRESS_DB_PASSWORD
+          valueFrom:
+            secretKeyRef:
+              name: mysql-pass
+              key: password
+        volumeMounts:
+        - name: wordpress-persistent-storage
+          mountPath: /var/www/html
+      volumes:
+      - name: wordpress-persistent-storage
+        persistentVolumeClaim:
+          claimName: wp-pv-claim
+  serviceName: wordpress-identity

kyaml/kio/filters/testdata/dataset1-remoteupdates/java/java-configmap.resource.yaml

@@ -0,0 +1,11 @@
+# Copyright 2019 The Kubernetes Authors.
+# SPDX-License-Identifier: Apache-2.0
+# 
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: app-config
+  labels:
+    app.kubernetes.io/component: undefined
+    app.kubernetes.io/instance: undefined
+data: {}

kyaml/kio/filters/testdata/dataset1-remoteupdates/java/java-deployment.resource.yaml

@@ -0,0 +1,40 @@
+# Copyright 2019 The Kubernetes Authors.
+# SPDX-License-Identifier: Apache-2.0
+# 
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: app
+  labels:
+    app: java
+    new-remote: label
+spec:
+  replicas: 3
+  selector:
+    matchLabels:
+      app: java
+  template:
+    metadata:
+      labels:
+        app: java
+    spec:
+      restartPolicy: Always
+      containers:
+      - name: app
+        image: gcr.io/project/app:version
+        command:
+        - java
+        - -jar
+        - /app.jar
+        - otherstuff
+        ports:
+        - containerPort: 8080
+        envFrom:
+        - configMapRef:
+            name: app-config
+        env:
+        - name: JAVA_OPTS
+          value: -XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap
+            -Djava.security.egd=file:/dev/./urandom
+        imagePullPolicy: Always
+  minReadySeconds: 5

kyaml/kio/filters/testdata/dataset1-remoteupdates/java/java-service.resource.yaml

@@ -0,0 +1,16 @@
+# Copyright 2019 The Kubernetes Authors.
+# SPDX-License-Identifier: Apache-2.0
+# 
+apiVersion: v1
+kind: Service
+metadata:
+  name: app
+  labels:
+    app: java
+spec:
+  selector:
+    app: java
+  ports:
+  - name: "8080"
+    port: 8080
+    targetPort: 8080

kyaml/kio/filters/testdata/dataset1-remoteupdates/mysql/mysql-configmap.resource.yaml

@@ -0,0 +1,23 @@
+# Copyright 2019 The Kubernetes Authors.
+# SPDX-License-Identifier: Apache-2.0
+# 
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: mysql
+  labels:
+    app: mysql
+  annotations:
+    file/index: "0"
+    file/path: mysql-configmap.resource.yaml
+    package/name: mysql
+    package/original-name: mysql
+data:
+  master.cnf: |
+    # Apply this config only on the master.
+    [mysqld]
+    log-bin
+  slave.cnf: |
+    # Apply this config only on slaves.
+    [mysqld]
+    super-read-only

kyaml/kio/filters/testdata/dataset1-remoteupdates/mysql/mysql-service.resource.yaml

@@ -0,0 +1,29 @@
+# Copyright 2019 The Kubernetes Authors.
+# SPDX-License-Identifier: Apache-2.0
+# 
+apiVersion: v1
+kind: Service
+metadata:
+  name: mysql
+  labels:
+    app: mysql
+spec:
+  selector:
+    app: mysql
+  ports:
+  - name: mysql
+    port: 3306
+  clusterIP: None
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: mysql-read
+  labels:
+    app: mysql
+spec:
+  selector:
+    app: mysql
+  ports:
+  - name: mysql
+    port: 3306

kyaml/kio/filters/testdata/dataset1-remoteupdates/mysql/mysql-statefulset.resource.yaml

@@ -0,0 +1,173 @@
+# Copyright 2019 The Kubernetes Authors.
+# SPDX-License-Identifier: Apache-2.0
+# 
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: mysql
+spec:
+  replicas: 3
+  selector:
+    matchLabels:
+      app: mysql
+  template:
+    metadata:
+      labels:
+        app: mysql
+    spec:
+      initContainers:
+      - name: init-mysql
+        image: mysql:5.7
+        command:
+        - bash
+        - -c
+        - |
+          set -ex
+          # Generate mysql server-id from pod ordinal index.
+          [[ `hostname` =~ -([0-9]+)$ ]] || exit 1
+          ordinal=${BASH_REMATCH[1]}
+          echo [mysqld] > /mnt/conf.d/server-id.cnf
+          # Add an offset to avoid reserved server-id=0 value.
+          echo server-id=$((100 + $ordinal)) >> /mnt/conf.d/server-id.cnf
+          # Copy appropriate conf.d files from config-map to emptyDir.
+          if [[ $ordinal -eq 0 ]]; then
+            cp /mnt/config-map/master.cnf /mnt/conf.d/
+          else
+            cp /mnt/config-map/slave.cnf /mnt/conf.d/
+          fi
+        volumeMounts:
+        - name: conf
+          mountPath: /mnt/conf.d
+        - name: config-map
+          mountPath: /mnt/config-map
+      - name: clone-mysql
+        image: gcr.io/google-samples/xtrabackup:1.0
+        command:
+        - bash
+        - -c
+        - |
+          set -ex
+          # Skip the clone if data already exists.
+          [[ -d /var/lib/mysql/mysql ]] && exit 0
+          # Skip the clone on master (ordinal index 0).
+          [[ `hostname` =~ -([0-9]+)$ ]] || exit 1
+          ordinal=${BASH_REMATCH[1]}
+          [[ $ordinal -eq 0 ]] && exit 0
+          # Clone data from previous peer.
+          ncat --recv-only mysql-$(($ordinal-1)).mysql 3307 | xbstream -x -C /var/lib/mysql
+          # Prepare the backup.
+          xtrabackup --prepare --target-dir=/var/lib/mysql
+        volumeMounts:
+        - name: data
+          mountPath: /var/lib/mysql
+          subPath: mysql
+        - name: conf
+          mountPath: /etc/mysql/conf.d
+      containers:
+      - name: mysql
+        image: mysql:5.7
+        ports:
+        - name: mysql
+          containerPort: 3306
+        env:
+        - name: MYSQL_ALLOW_EMPTY_PASSWORD
+          value: "1"
+        resources:
+          requests:
+            cpu: 500m
+            memory: 1Gi
+        volumeMounts:
+        - name: data
+          mountPath: /var/lib/mysql
+          subPath: mysql
+        - name: conf
+          mountPath: /etc/mysql/conf.d
+        livenessProbe:
+          exec:
+            command:
+            - mysqladmin
+            - ping
+          initialDelaySeconds: 30
+          periodSeconds: 10
+          timeoutSeconds: 5
+        readinessProbe:
+          exec:
+            command:
+            - mysql
+            - -h
+            - 127.0.0.1
+            - -e
+            - SELECT 1
+          initialDelaySeconds: 5
+          periodSeconds: 2
+          timeoutSeconds: 1
+      - name: xtrabackup
+        image: gcr.io/google-samples/xtrabackup:1.0
+        command:
+        - bash
+        - -c
+        - |
+          set -ex
+          cd /var/lib/mysql
+          # Determine binlog position of cloned data, if any.
+          if [[ -f xtrabackup_slave_info ]]; then
+            # XtraBackup already generated a partial "CHANGE MASTER TO" query
+            # because we're cloning from an existing slave.
+            mv xtrabackup_slave_info change_master_to.sql.in
+            # Ignore xtrabackup_binlog_info in this case (it's useless).
+            rm -f xtrabackup_binlog_info
+          elif [[ -f xtrabackup_binlog_info ]]; then
+            # We're cloning directly from master. Parse binlog position.
+            [[ `cat xtrabackup_binlog_info` =~ ^(.*?)[[:space:]]+(.*?)$ ]] || exit 1
+            rm xtrabackup_binlog_info
+            echo "CHANGE MASTER TO MASTER_LOG_FILE='${BASH_REMATCH[1]}',\
+                  MASTER_LOG_POS=${BASH_REMATCH[2]}" > change_master_to.sql.in
+          fi
+          # Check if we need to complete a clone by starting replication.
+          if [[ -f change_master_to.sql.in ]]; then
+            echo "Waiting for mysqld to be ready (accepting connections)"
+            until mysql -h 127.0.0.1 -e "SELECT 1"; do sleep 1; done
+            echo "Initializing replication from clone position"
+            # In case of container restart, attempt this at-most-once.
+            mv change_master_to.sql.in change_master_to.sql.orig
+            mysql -h 127.0.0.1 <<EOF
+          $(<change_master_to.sql.orig),
+            MASTER_HOST='mysql-0.mysql',
+            MASTER_USER='root',
+            MASTER_PASSWORD='',
+            MASTER_CONNECT_RETRY=10;
+          START SLAVE;
+          EOF
+          fi
+          # Start a server to send backups when requested by peers.
+          exec ncat --listen --keep-open --send-only --max-conns=1 3307 -c \
+            "xtrabackup --backup --slave-info --stream=xbstream --host=127.0.0.1 --user=root"
+        ports:
+        - name: xtrabackup
+          containerPort: 3307
+        resources:
+          requests:
+            cpu: 100m
+            memory: 100Mi
+        volumeMounts:
+        - name: data
+          mountPath: /var/lib/mysql
+          subPath: mysql
+        - name: conf
+          mountPath: /etc/mysql/conf.d
+      volumes:
+      - name: conf
+        emptyDir: {}
+      - name: config-map
+        configMap:
+          name: mysql
+  volumeClaimTemplates:
+  - metadata:
+      name: data
+    spec:
+      resources:
+        requests:
+          storage: 10Gi
+      accessModes:
+      - ReadWriteOnce
+  serviceName: mysql

kyaml/kio/filters/testdata/dataset1/java/java-configmap.resource.yaml

@@ -0,0 +1,11 @@
+# Copyright 2019 The Kubernetes Authors.
+# SPDX-License-Identifier: Apache-2.0
+# 
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: app-config
+  labels:
+    app.kubernetes.io/component: undefined
+    app.kubernetes.io/instance: undefined
+data: {}

kyaml/kio/filters/testdata/dataset1/java/java-deployment.resource.yaml

@@ -0,0 +1,38 @@
+# Copyright 2019 The Kubernetes Authors.
+# SPDX-License-Identifier: Apache-2.0
+# 
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: app
+  labels:
+    app: java
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: java
+  template:
+    metadata:
+      labels:
+        app: java
+    spec:
+      restartPolicy: Always
+      containers:
+      - name: app
+        image: gcr.io/project/app:version
+        command:
+        - java
+        - -jar
+        - /app.jar
+        ports:
+        - containerPort: 8080
+        envFrom:
+        - configMapRef:
+            name: app-config
+        env:
+        - name: JAVA_OPTS
+          value: -XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap
+            -Djava.security.egd=file:/dev/./urandom
+        imagePullPolicy: Always
+  minReadySeconds: 5

kyaml/kio/filters/testdata/dataset1/java/java-service.resource.yaml

@@ -0,0 +1,16 @@
+# Copyright 2019 The Kubernetes Authors.
+# SPDX-License-Identifier: Apache-2.0
+# 
+apiVersion: v1
+kind: Service
+metadata:
+  name: app
+  labels:
+    app: java
+spec:
+  selector:
+    app: java
+  ports:
+  - name: "8080"
+    port: 8080
+    targetPort: 8080

kyaml/kio/filters/testdata/dataset1/rails/rails-configmap.resource.yaml

@@ -0,0 +1,11 @@
+# Copyright 2019 The Kubernetes Authors.
+# SPDX-License-Identifier: Apache-2.0
+# 
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: rails-app-config
+  labels:
+    app.kubernetes.io/component: undefined
+    app.kubernetes.io/instance: undefined
+data: {}

kyaml/kio/filters/testdata/dataset1/rails/rails-deployment.resource.yaml

@@ -0,0 +1,36 @@
+# Copyright 2019 The Kubernetes Authors.
+# SPDX-License-Identifier: Apache-2.0
+# 
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: rails-app
+  labels:
+    app: rails
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: rails
+  template:
+    metadata:
+      labels:
+        app: rails
+    spec:
+      restartPolicy: Always
+      containers:
+      - name: rails
+        image: gcr.io/project/app:version
+        command:
+        - rails
+        ports:
+        - containerPort: 8080
+        envFrom:
+        - configMapRef:
+            name: rails-app-config
+        env:
+        - name: JAVA_OPTS
+          value: -XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap
+            -Djava.security.egd=file:/dev/./urandom
+        imagePullPolicy: Always
+  minReadySeconds: 5

kyaml/kio/filters/testdata/dataset1/rails/rails-service.resource.yaml

@@ -0,0 +1,16 @@
+# Copyright 2019 The Kubernetes Authors.
+# SPDX-License-Identifier: Apache-2.0
+# 
+apiVersion: v1
+kind: Service
+metadata:
+  name: rails-app
+  labels:
+    app: rails
+spec:
+  selector:
+    app: rails
+  ports:
+  - name: "8080"
+    port: 8080
+    targetPort: 8080

kyaml/kio/filters/testyaml/testyaml.go

@@ -0,0 +1,57 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+// Package testyaml contains test data and libraries for formatting
+// Kubernetes configuration
+package testyaml
+
+var UnformattedYaml1 = []byte(`
+spec: a
+status:
+  conditions:
+  - 3
+  - 1
+  - 2
+apiVersion: example.com/v1beta1
+kind: MyType
+`)
+
+var UnformattedYaml2 = []byte(`
+spec2: a
+status2:
+  conditions:
+  - 3
+  - 1
+  - 2
+apiVersion: example.com/v1beta1
+kind: MyType2
+`)
+
+var UnformattedJson1 = []byte(`
+{
+  "spec": "a",
+  "status": {"conditions": [3, 1, 2]},
+  "apiVersion": "example.com/v1beta1",
+  "kind": "MyType"
+}
+`)
+
+var FormattedYaml1 = []byte(`apiVersion: example.com/v1beta1
+kind: MyType
+spec: a
+status:
+  conditions:
+  - 3
+  - 1
+  - 2
+`)
+
+var FormattedYaml2 = []byte(`apiVersion: example.com/v1beta1
+kind: MyType2
+spec2: a
+status2:
+  conditions:
+  - 3
+  - 1
+  - 2
+`)

kyaml/kio/kio.go

@@ -0,0 +1,106 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+// Package kio contains low-level libraries for reading, modifying and writing
+// Resource Configuration and packages.
+package kio
+
+import (
+	"sigs.k8s.io/kustomize/kyaml/yaml"
+)
+
+// Reader reads ResourceNodes. Analogous to io.Reader.
+type Reader interface {
+	Read() ([]*yaml.RNode, error)
+}
+
+// ResourceNodeSlice is a collection of ResourceNodes.
+// While ResourceNodeSlice has no inherent constraints on ordering or uniqueness, specific
+// Readers, Filters or Writers may have constraints.
+type ResourceNodeSlice []*yaml.RNode
+
+var _ Reader = ResourceNodeSlice{}
+
+func (o ResourceNodeSlice) Read() ([]*yaml.RNode, error) {
+	return o, nil
+}
+
+// Writer writes ResourceNodes. Analogous to io.Writer.
+type Writer interface {
+	Write([]*yaml.RNode) error
+}
+
+type WriterFunc func([]*yaml.RNode) error
+
+func (fn WriterFunc) Write(o []*yaml.RNode) error {
+	return fn(o)
+}
+
+// GrepFilter modifies a collection of Resource Configuration by returning the modified slice.
+// When possible, Filters should be serializable to yaml so that they can be described
+// declaratively as data.
+//
+// Analogous to http://www.linfo.org/filters.html
+type Filter interface {
+	Filter([]*yaml.RNode) ([]*yaml.RNode, error)
+}
+
+// FilterFunc can be used to implement GrepFilter by defining a function.
+type FilterFunc func([]*yaml.RNode) ([]*yaml.RNode, error)
+
+func (fn FilterFunc) Filter(o []*yaml.RNode) ([]*yaml.RNode, error) {
+	return fn(o)
+}
+
+// Pipeline reads Resource Configuration from a set of Inputs, applies some
+// transformations, and writes the results to a set of Outputs.
+//
+// Analogous to http://www.linfo.org/pipes.html
+type Pipeline struct {
+	// Inputs provide sources for Resource Configuration to be read.
+	Inputs []Reader `yaml:"inputs,omitempty"`
+
+	// Filters are transformations applied to the Resource Configuration.
+	// They are applied in the order they are specified.
+	// Analogous to http://www.linfo.org/filters.html
+	Filters []Filter `yaml:"filters,omitempty"`
+
+	// Outputs are where the transformed Resource Configuration is written.
+	Outputs []Writer `yaml:"outputs,omitempty"`
+}
+
+// Execute implements the Pipeline pipeline.
+func (p Pipeline) Execute() error {
+	var result []*yaml.RNode
+
+	// read from the inputs
+	for _, i := range p.Inputs {
+		nodes, err := i.Read()
+		if err != nil {
+			return err
+		}
+		result = append(result, nodes...)
+	}
+	if len(result) == 0 {
+		// no inputs to operate on
+		return nil
+	}
+
+	// apply operations
+	var err error
+	for i := range p.Filters {
+		op := p.Filters[i]
+		result, err = op.Filter(result)
+		if len(result) == 0 || err != nil {
+			return err
+		}
+	}
+
+	// write to the outputs
+	for _, o := range p.Outputs {
+		if err := o.Write(result); err != nil {
+			return err
+		}
+	}
+	return nil
+}

kyaml/kio/kio_test.go

@@ -0,0 +1,28 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package kio_test
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	. "sigs.k8s.io/kustomize/kyaml/kio"
+)
+
+func TestPipe(t *testing.T) {
+	p := Pipeline{
+		Inputs:  []Reader{},
+		Filters: []Filter{},
+		Outputs: []Writer{},
+	}
+
+	err := p.Execute()
+	if !assert.NoError(t, err) {
+		assert.FailNow(t, err.Error())
+	}
+}
+
+func TestSlice_Write(t *testing.T) {
+
+}

kyaml/kio/kioutil/kioutil.go

@@ -0,0 +1,111 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package kioutil
+
+import (
+	"fmt"
+	"sort"
+	"strconv"
+
+	"sigs.k8s.io/kustomize/kyaml/yaml"
+)
+
+type AnnotationKey = string
+
+const (
+	// IndexAnnotation records the index of a specific resource in a file or input stream.
+	IndexAnnotation AnnotationKey = "kyaml.kustomize.dev/kio/index"
+
+	// PathAnnotation records the path to the file the Resource was read from
+	PathAnnotation AnnotationKey = "kyaml.kustomize.dev/kio/path"
+
+	// PackageAnnotation records the name of the package the Resource was read from
+	PackageAnnotation AnnotationKey = "kyaml.kustomize.dev/kio/package"
+)
+
+func GetFileAnnotations(rn *yaml.RNode) (string, string, error) {
+	meta, err := rn.GetMeta()
+	if err != nil {
+		return "", "", err
+	}
+	path := meta.Annotations[PathAnnotation]
+	index := meta.Annotations[IndexAnnotation]
+	return path, index, nil
+}
+
+// ErrorIfMissingAnnotation validates the provided annotations are present on the given resources
+func ErrorIfMissingAnnotation(nodes []*yaml.RNode, keys ...AnnotationKey) error {
+	for _, key := range keys {
+		for _, node := range nodes {
+			val, err := node.Pipe(yaml.GetAnnotation(key))
+			if err != nil {
+				return err
+			}
+			if val == nil {
+				return fmt.Errorf("missing package annotation %s", key)
+			}
+		}
+	}
+	return nil
+}
+
+// SortNodes sorts nodes in place:
+// - by PathAnnotation annotation
+// - by IndexAnnotation annotation
+func SortNodes(nodes []*yaml.RNode) error {
+	var err error
+	// use stable sort to keep ordering of equal elements
+	sort.SliceStable(nodes, func(i, j int) bool {
+		if err != nil {
+			return false
+		}
+		var iMeta, jMeta yaml.ResourceMeta
+		if iMeta, _ = nodes[i].GetMeta(); err != nil {
+			return false
+		}
+		if jMeta, _ = nodes[j].GetMeta(); err != nil {
+			return false
+		}
+
+		iValue := iMeta.Annotations[PathAnnotation]
+		jValue := jMeta.Annotations[PathAnnotation]
+		if iValue != jValue {
+			return iValue < jValue
+		}
+
+		iValue = iMeta.Annotations[IndexAnnotation]
+		jValue = jMeta.Annotations[IndexAnnotation]
+
+		// put resource config without an index first
+		if iValue == jValue {
+			return false
+		}
+		if iValue == "" {
+			return true
+		}
+		if jValue == "" {
+			return false
+		}
+
+		// sort by index
+		var iIndex, jIndex int
+		iIndex, err = strconv.Atoi(iValue)
+		if err != nil {
+			err = fmt.Errorf("unable to parse kyaml.kustomize.dev/kio/index %s :%v", iValue, err)
+			return false
+		}
+		jIndex, err = strconv.Atoi(jValue)
+		if err != nil {
+			err = fmt.Errorf("unable to parse kyaml.kustomize.dev/kio/index %s :%v", jValue, err)
+			return false
+		}
+		if iIndex != jIndex {
+			return iValue < jValue
+		}
+
+		// elements are equal
+		return false
+	})
+	return err
+}

kyaml/kio/pkgio_reader.go

@@ -0,0 +1,281 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package kio
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"sigs.k8s.io/kustomize/kyaml/kio/kioutil"
+	"sigs.k8s.io/kustomize/kyaml/sets"
+	"sigs.k8s.io/kustomize/kyaml/yaml"
+)
+
+// requiredResourcePackageAnnotations are annotations that are required to write resources back to
+// files.
+var requiredResourcePackageAnnotations = []string{kioutil.IndexAnnotation, kioutil.PathAnnotation}
+
+type PackageBuffer struct {
+	Nodes []*yaml.RNode
+}
+
+func (r *PackageBuffer) Read() ([]*yaml.RNode, error) {
+	return r.Nodes, nil
+}
+
+func (r *PackageBuffer) Write(nodes []*yaml.RNode) error {
+	r.Nodes = nodes
+	return nil
+}
+
+type LocalPackageReadWriter struct {
+	Kind string `yaml:"kind,omitempty"`
+
+	KeepReaderAnnotations bool `yaml:"keepReaderAnnotations,omitempty"`
+
+	// PackagePath is the path to the package directory.
+	PackagePath string `yaml:"path,omitempty"`
+
+	// PackageFileName is the name of file containing package metadata.
+	// It will be used to identify package.
+	PackageFileName string `yaml:"packageFileName,omitempty"`
+
+	// MatchFilesGlob configures Read to only read Resources from files matching any of the
+	// provided patterns.
+	// Defaults to ["*.yaml", "*.yml"] if empty.  To match all files specify ["*"].
+	MatchFilesGlob []string `yaml:"matchFilesGlob,omitempty"`
+
+	// IncludeSubpackages will configure Read to read Resources from subpackages.
+	// Subpackages are identified by presence of PackageFileName.
+	IncludeSubpackages bool `yaml:"includeSubpackages,omitempty"`
+
+	// ErrorIfNonResources will configure Read to throw an error if yaml missing missing
+	// apiVersion or kind is read.
+	ErrorIfNonResources bool `yaml:"errorIfNonResources,omitempty"`
+
+	// OmitReaderAnnotations will cause the reader to skip annotating Resources with the file
+	// path and mode.
+	OmitReaderAnnotations bool `yaml:"omitReaderAnnotations,omitempty"`
+
+	// SetAnnotations are annotations to set on the Resources as they are read.
+	SetAnnotations map[string]string `yaml:"setAnnotations,omitempty"`
+
+	// NoDeleteFiles if set to true, LocalPackageReadWriter won't delete any files
+	NoDeleteFiles bool `yaml:"noDeleteFiles,omitempty"`
+
+	files sets.String
+}
+
+func (r *LocalPackageReadWriter) Read() ([]*yaml.RNode, error) {
+	nodes, err := LocalPackageReader{
+		PackagePath:         r.PackagePath,
+		MatchFilesGlob:      r.MatchFilesGlob,
+		IncludeSubpackages:  r.IncludeSubpackages,
+		ErrorIfNonResources: r.ErrorIfNonResources,
+		SetAnnotations:      r.SetAnnotations,
+	}.Read()
+	if err != nil {
+		return nil, err
+	}
+	// keep track of all the files
+	if !r.NoDeleteFiles {
+		r.files, err = r.getFiles(nodes)
+		if err != nil {
+			return nil, err
+		}
+	}
+	return nodes, nil
+}
+
+func (r *LocalPackageReadWriter) Write(nodes []*yaml.RNode) error {
+	newFiles, err := r.getFiles(nodes)
+	if err != nil {
+		return err
+	}
+	var clear []string
+	for k := range r.SetAnnotations {
+		clear = append(clear, k)
+	}
+	err = LocalPackageWriter{
+		PackagePath:           r.PackagePath,
+		ClearAnnotations:      clear,
+		KeepReaderAnnotations: r.KeepReaderAnnotations,
+	}.Write(nodes)
+	if err != nil {
+		return err
+	}
+	deleteFiles := r.files.Difference(newFiles)
+	for f := range deleteFiles {
+		if err = os.Remove(filepath.Join(r.PackagePath, f)); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (r *LocalPackageReadWriter) getFiles(nodes []*yaml.RNode) (sets.String, error) {
+	val := sets.String{}
+	for _, n := range nodes {
+		path, _, err := kioutil.GetFileAnnotations(n)
+		if err != nil {
+			return nil, err
+		}
+		val.Insert(path)
+	}
+	return val, nil
+}
+
+// LocalPackageReader reads ResourceNodes from a local package.
+type LocalPackageReader struct {
+	Kind string `yaml:"kind,omitempty"`
+
+	// PackagePath is the path to the package directory.
+	PackagePath string `yaml:"path,omitempty"`
+
+	// PackageFileName is the name of file containing package metadata.
+	// It will be used to identify package.
+	PackageFileName string `yaml:"packageFileName,omitempty"`
+
+	// MatchFilesGlob configures Read to only read Resources from files matching any of the
+	// provided patterns.
+	// Defaults to ["*.yaml", "*.yml"] if empty.  To match all files specify ["*"].
+	MatchFilesGlob []string `yaml:"matchFilesGlob,omitempty"`
+
+	// IncludeSubpackages will configure Read to read Resources from subpackages.
+	// Subpackages are identified by presence of PackageFileName.
+	IncludeSubpackages bool `yaml:"includeSubpackages,omitempty"`
+
+	// ErrorIfNonResources will configure Read to throw an error if yaml missing missing
+	// apiVersion or kind is read.
+	ErrorIfNonResources bool `yaml:"errorIfNonResources,omitempty"`
+
+	// OmitReaderAnnotations will cause the reader to skip annotating Resources with the file
+	// path and mode.
+	OmitReaderAnnotations bool `yaml:"omitReaderAnnotations,omitempty"`
+
+	// SetAnnotations are annotations to set on the Resources as they are read.
+	SetAnnotations map[string]string `yaml:"setAnnotations,omitempty"`
+}
+
+var _ Reader = LocalPackageReader{}
+
+var defaultMatch = []string{"*.yaml", "*.yml"}
+
+// Read reads the Resources.
+func (r LocalPackageReader) Read() ([]*yaml.RNode, error) {
+	if r.PackagePath == "" {
+		return nil, fmt.Errorf("must specify package path")
+	}
+	if len(r.MatchFilesGlob) == 0 {
+		r.MatchFilesGlob = defaultMatch
+	}
+
+	var operand ResourceNodeSlice
+	var pathRelativeTo string
+	r.PackagePath = filepath.Clean(r.PackagePath)
+	err := filepath.Walk(r.PackagePath, func(
+		path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+
+		// is this the user specified path?
+		if path == r.PackagePath {
+			if info.IsDir() {
+				// skip the root package directory
+				pathRelativeTo = r.PackagePath
+				return nil
+			}
+
+			// user specified path is a file rather than a directory.
+			// make its path relative to its parent so it can be written to another file.
+			pathRelativeTo = filepath.Dir(r.PackagePath)
+		}
+
+		// check if we should skip the directory or file
+		if info.IsDir() {
+			return r.shouldSkipDir(path, info)
+		}
+		if match, err := r.shouldSkipFile(path, info); err != nil {
+			return err
+		} else if !match {
+			// skip this file
+			return nil
+		}
+
+		// get the relative path to file within the package so we can write the files back out
+		// to another location.
+		path, err = filepath.Rel(pathRelativeTo, path)
+		if err != nil {
+			return err
+		}
+
+		r.initReaderAnnotations(path, info)
+		nodes, err := r.readFile(filepath.Join(pathRelativeTo, path), info)
+		if err != nil {
+			return err
+		}
+		operand = append(operand, nodes...)
+		return nil
+	})
+	return operand, err
+}
+
+// readFile reads the ResourceNodes from a file
+func (r *LocalPackageReader) readFile(path string, info os.FileInfo) ([]*yaml.RNode, error) {
+	f, err := os.Open(path)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+	rr := &ByteReader{
+		Reader:                f,
+		OmitReaderAnnotations: r.OmitReaderAnnotations,
+		SetAnnotations:        r.SetAnnotations,
+	}
+	return rr.Read()
+}
+
+// shouldSkipFile returns true if the file should be skipped
+func (r *LocalPackageReader) shouldSkipFile(path string, info os.FileInfo) (bool, error) {
+	// check if the files are in scope
+	for _, g := range r.MatchFilesGlob {
+		if match, err := filepath.Match(g, info.Name()); err != nil {
+			return false, err
+		} else if match {
+			return true, nil
+		}
+	}
+	return false, nil
+}
+
+// initReaderAnnotations adds the LocalPackageReader Annotations to r.SetAnnotations
+func (r *LocalPackageReader) initReaderAnnotations(path string, info os.FileInfo) {
+	if r.SetAnnotations == nil {
+		r.SetAnnotations = map[string]string{}
+	}
+	if !r.OmitReaderAnnotations {
+		r.SetAnnotations[kioutil.PackageAnnotation] = filepath.Dir(path)
+		r.SetAnnotations[kioutil.PathAnnotation] = path
+	}
+}
+
+// shouldSkipDir returns a filepath.SkipDir if the directory should be skipped
+func (r *LocalPackageReader) shouldSkipDir(path string, info os.FileInfo) error {
+	if r.PackageFileName == "" {
+		return nil
+	}
+	// check if this is a subpackage
+	_, err := os.Stat(filepath.Join(path, r.PackageFileName))
+	if os.IsNotExist(err) {
+		return nil
+	} else if err != nil {
+		return err
+	}
+	if !r.IncludeSubpackages {
+		return filepath.SkipDir
+	}
+	return nil
+}

kyaml/kio/pkgio_reader_test.go

@@ -0,0 +1,486 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package kio_test
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	. "sigs.k8s.io/kustomize/kyaml/kio"
+	// "sigs.k8s.io/kustomize/kyaml/testutil"
+)
+
+// setup creates directories and files for testing
+type setup struct {
+	// root is the tmp directory
+	root string
+}
+
+// setupDirectories creates directories for reading test configuration from
+func setupDirectories(t *testing.T, dirs ...string) setup {
+	d, err := ioutil.TempDir("", "kyaml-test")
+	if !assert.NoError(t, err) {
+		assert.FailNow(t, err.Error())
+	}
+	err = os.Chdir(d)
+	if !assert.NoError(t, err) {
+		assert.FailNow(t, err.Error())
+	}
+	for _, s := range dirs {
+		err = os.MkdirAll(s, 0700)
+		if !assert.NoError(t, err) {
+			assert.FailNow(t, err.Error())
+		}
+	}
+	return setup{root: d}
+}
+
+// writeFile writes a file under the test directory
+func (s setup) writeFile(t *testing.T, path string, value []byte) {
+	err := os.MkdirAll(filepath.Dir(filepath.Join(s.root, path)), 0700)
+	if !assert.NoError(t, err) {
+		assert.FailNow(t, err.Error())
+	}
+	err = ioutil.WriteFile(filepath.Join(s.root, path), value, 0600)
+	if !assert.NoError(t, err) {
+		assert.FailNow(t, err.Error())
+	}
+}
+
+// clean deletes the test config
+func (s setup) clean() {
+	os.RemoveAll(s.root)
+}
+
+var readFileA = []byte(`---
+a: b #first
+---
+c: d # second
+`)
+
+var readFileB = []byte(`# second thing
+e: f
+g:
+  h:
+  - i # has a list
+  - j
+`)
+
+var pkgFile = []byte(``)
+
+func TestLocalPackageReader_Read_empty(t *testing.T) {
+	var r LocalPackageReader
+	nodes, err := r.Read()
+	if assert.Error(t, err) {
+		assert.Contains(t, err.Error(), "must specify package path")
+	}
+	assert.Nil(t, nodes)
+
+}
+
+func TestLocalPackageReader_Read_pkg(t *testing.T) {
+	s := setupDirectories(t, filepath.Join("a", "b"), filepath.Join("a", "c"))
+	defer s.clean()
+	s.writeFile(t, filepath.Join("a_test.yaml"), readFileA)
+	s.writeFile(t, filepath.Join("b_test.yaml"), readFileB)
+
+	paths := []struct {
+		path string
+	}{
+		{path: "./"},
+		{path: s.root},
+	}
+	for _, p := range paths {
+		rfr := LocalPackageReader{PackagePath: p.path}
+		nodes, err := rfr.Read()
+		if !assert.NoError(t, err) {
+			return
+		}
+
+		if !assert.Len(t, nodes, 3) {
+			return
+		}
+		expected := []string{
+			`a: b #first
+metadata:
+  annotations:
+    kyaml.kustomize.dev/kio/index: 0
+    kyaml.kustomize.dev/kio/package: .
+    kyaml.kustomize.dev/kio/path: a_test.yaml
+`,
+			`c: d # second
+metadata:
+  annotations:
+    kyaml.kustomize.dev/kio/index: 1
+    kyaml.kustomize.dev/kio/package: .
+    kyaml.kustomize.dev/kio/path: a_test.yaml
+`,
+			`# second thing
+e: f
+g:
+  h:
+  - i # has a list
+  - j
+metadata:
+  annotations:
+    kyaml.kustomize.dev/kio/index: 0
+    kyaml.kustomize.dev/kio/package: .
+    kyaml.kustomize.dev/kio/path: b_test.yaml
+`,
+		}
+		for i := range nodes {
+			val, err := nodes[i].String()
+			if !assert.NoError(t, err) {
+				return
+			}
+			if !assert.Equal(t, expected[i], val) {
+				return
+			}
+		}
+	}
+}
+
+func TestLocalPackageReader_Read_file(t *testing.T) {
+	s := setupDirectories(t, filepath.Join("a", "b"), filepath.Join("a", "c"))
+	defer s.clean()
+	s.writeFile(t, filepath.Join("a_test.yaml"), readFileA)
+	s.writeFile(t, filepath.Join("b_test.yaml"), readFileB)
+
+	paths := []struct {
+		path string
+	}{
+		{path: "./"},
+		{path: s.root},
+	}
+	for _, p := range paths {
+		rfr := LocalPackageReader{PackagePath: filepath.Join(p.path, "a_test.yaml")}
+		nodes, err := rfr.Read()
+		if !assert.NoError(t, err) {
+			return
+		}
+
+		if !assert.Len(t, nodes, 2) {
+			return
+		}
+		expected := []string{
+			`a: b #first
+metadata:
+  annotations:
+    kyaml.kustomize.dev/kio/index: 0
+    kyaml.kustomize.dev/kio/package: .
+    kyaml.kustomize.dev/kio/path: a_test.yaml
+`,
+			`c: d # second
+metadata:
+  annotations:
+    kyaml.kustomize.dev/kio/index: 1
+    kyaml.kustomize.dev/kio/package: .
+    kyaml.kustomize.dev/kio/path: a_test.yaml
+`,
+		}
+		for i := range nodes {
+			val, err := nodes[i].String()
+			if !assert.NoError(t, err) {
+				return
+			}
+			if !assert.Equal(t, expected[i], val) {
+				return
+			}
+		}
+	}
+}
+
+func TestLocalPackageReader_Read_pkgOmitAnnotations(t *testing.T) {
+	s := setupDirectories(t, filepath.Join("a", "b"), filepath.Join("a", "c"))
+	defer s.clean()
+	s.writeFile(t, filepath.Join("a_test.yaml"), readFileA)
+	s.writeFile(t, filepath.Join("b_test.yaml"), readFileB)
+
+	paths := []struct {
+		path string
+	}{
+		{path: "./"},
+		{path: s.root},
+	}
+	for _, p := range paths {
+
+		// empty path
+		rfr := LocalPackageReader{PackagePath: p.path, OmitReaderAnnotations: true}
+		nodes, err := rfr.Read()
+		if !assert.NoError(t, err) {
+			return
+		}
+
+		if !assert.Len(t, nodes, 3) {
+			return
+		}
+		expected := []string{
+			`a: b #first
+`,
+			`c: d # second
+`,
+			`# second thing
+e: f
+g:
+  h:
+  - i # has a list
+  - j
+`,
+		}
+		for i := range nodes {
+			val, err := nodes[i].String()
+			if !assert.NoError(t, err) {
+				return
+			}
+			if !assert.Equal(t, expected[i], val) {
+				return
+			}
+		}
+	}
+}
+
+func TestLocalPackageReader_Read_nestedDirs(t *testing.T) {
+	s := setupDirectories(t, filepath.Join("a", "b"), filepath.Join("a", "c"))
+	defer s.clean()
+	s.writeFile(t, filepath.Join("a", "b", "a_test.yaml"), readFileA)
+	s.writeFile(t, filepath.Join("a", "b", "b_test.yaml"), readFileB)
+
+	paths := []struct {
+		path string
+	}{
+		{path: "./"},
+		{path: s.root},
+	}
+	for _, p := range paths {
+		// empty path
+		rfr := LocalPackageReader{PackagePath: p.path}
+		nodes, err := rfr.Read()
+		if !assert.NoError(t, err) {
+			assert.FailNow(t, err.Error())
+		}
+
+		if !assert.Len(t, nodes, 3) {
+			return
+		}
+		expected := []string{
+			`a: b #first
+metadata:
+  annotations:
+    kyaml.kustomize.dev/kio/index: 0
+    kyaml.kustomize.dev/kio/package: a/b
+    kyaml.kustomize.dev/kio/path: a/b/a_test.yaml
+`,
+			`c: d # second
+metadata:
+  annotations:
+    kyaml.kustomize.dev/kio/index: 1
+    kyaml.kustomize.dev/kio/package: a/b
+    kyaml.kustomize.dev/kio/path: a/b/a_test.yaml
+`,
+			`# second thing
+e: f
+g:
+  h:
+  - i # has a list
+  - j
+metadata:
+  annotations:
+    kyaml.kustomize.dev/kio/index: 0
+    kyaml.kustomize.dev/kio/package: a/b
+    kyaml.kustomize.dev/kio/path: a/b/b_test.yaml
+`,
+		}
+		for i := range nodes {
+			val, err := nodes[i].String()
+			if !assert.NoError(t, err) {
+				return
+			}
+			if !assert.Equal(t, expected[i], val) {
+				return
+			}
+		}
+	}
+}
+
+func TestLocalPackageReader_Read_matchRegex(t *testing.T) {
+	s := setupDirectories(t, filepath.Join("a", "b"), filepath.Join("a", "c"))
+	defer s.clean()
+	s.writeFile(t, filepath.Join("a", "b", "a_test.yaml"), readFileA)
+	s.writeFile(t, filepath.Join("a", "b", "b_test.yaml"), readFileB)
+
+	// empty path
+	rfr := LocalPackageReader{PackagePath: s.root, MatchFilesGlob: []string{`a*.yaml`}}
+	nodes, err := rfr.Read()
+	if !assert.NoError(t, err) {
+		assert.FailNow(t, err.Error())
+	}
+
+	if !assert.Len(t, nodes, 2) {
+		assert.FailNow(t, "wrong number items")
+	}
+
+	val, err := nodes[0].String()
+	assert.NoError(t, err)
+	assert.Equal(t, `a: b #first
+metadata:
+  annotations:
+    kyaml.kustomize.dev/kio/index: 0
+    kyaml.kustomize.dev/kio/package: a/b
+    kyaml.kustomize.dev/kio/path: a/b/a_test.yaml
+`, val)
+
+	val, err = nodes[1].String()
+	assert.NoError(t, err)
+	assert.Equal(t, `c: d # second
+metadata:
+  annotations:
+    kyaml.kustomize.dev/kio/index: 1
+    kyaml.kustomize.dev/kio/package: a/b
+    kyaml.kustomize.dev/kio/path: a/b/a_test.yaml
+`, val)
+}
+
+func TestLocalPackageReader_Read_skipSubpackage(t *testing.T) {
+	s := setupDirectories(t, filepath.Join("a", "b"), filepath.Join("a", "c"))
+	defer s.clean()
+	s.writeFile(t, filepath.Join("a", "b", "a_test.yaml"), readFileA)
+	s.writeFile(t, filepath.Join("a", "c", "c_test.yaml"), readFileB)
+	s.writeFile(t, filepath.Join("a", "c", "pkgFile"), pkgFile)
+
+	// empty path
+	rfr := LocalPackageReader{PackagePath: s.root, PackageFileName: "pkgFile"}
+	nodes, err := rfr.Read()
+	if !assert.NoError(t, err) {
+		assert.FailNow(t, err.Error())
+	}
+
+	if !assert.Len(t, nodes, 2) {
+		assert.FailNow(t, "wrong number items")
+	}
+
+	val, err := nodes[0].String()
+	assert.NoError(t, err)
+	assert.Equal(t, `a: b #first
+metadata:
+  annotations:
+    kyaml.kustomize.dev/kio/index: 0
+    kyaml.kustomize.dev/kio/package: a/b
+    kyaml.kustomize.dev/kio/path: a/b/a_test.yaml
+`, val)
+
+	val, err = nodes[1].String()
+	assert.NoError(t, err)
+	assert.Equal(t, `c: d # second
+metadata:
+  annotations:
+    kyaml.kustomize.dev/kio/index: 1
+    kyaml.kustomize.dev/kio/package: a/b
+    kyaml.kustomize.dev/kio/path: a/b/a_test.yaml
+`, val)
+}
+
+func TestLocalPackageReader_Read_includeSubpackage(t *testing.T) {
+	s := setupDirectories(t, filepath.Join("a", "b"), filepath.Join("a", "c"))
+	defer s.clean()
+	s.writeFile(t, filepath.Join("a", "b", "a_test.yaml"), readFileA)
+	s.writeFile(t, filepath.Join("a", "c", "c_test.yaml"), readFileB)
+	s.writeFile(t, filepath.Join("a", "c", "pkgFile"), pkgFile)
+
+	// empty path
+	rfr := LocalPackageReader{PackagePath: s.root, IncludeSubpackages: true, PackageFileName: "pkgFile"}
+	nodes, err := rfr.Read()
+	if !assert.NoError(t, err) {
+		assert.FailNow(t, err.Error())
+	}
+
+	if !assert.Len(t, nodes, 3) {
+		assert.FailNow(t, "wrong number items")
+	}
+	val, err := nodes[0].String()
+	assert.NoError(t, err)
+	assert.Equal(t, `a: b #first
+metadata:
+  annotations:
+    kyaml.kustomize.dev/kio/index: 0
+    kyaml.kustomize.dev/kio/package: a/b
+    kyaml.kustomize.dev/kio/path: a/b/a_test.yaml
+`, val)
+
+	val, err = nodes[1].String()
+	assert.NoError(t, err)
+	assert.Equal(t, `c: d # second
+metadata:
+  annotations:
+    kyaml.kustomize.dev/kio/index: 1
+    kyaml.kustomize.dev/kio/package: a/b
+    kyaml.kustomize.dev/kio/path: a/b/a_test.yaml
+`, val)
+
+	val, err = nodes[2].String()
+	assert.NoError(t, err)
+	assert.Equal(t, `# second thing
+e: f
+g:
+  h:
+  - i # has a list
+  - j
+metadata:
+  annotations:
+    kyaml.kustomize.dev/kio/index: 0
+    kyaml.kustomize.dev/kio/package: a/c
+    kyaml.kustomize.dev/kio/path: a/c/c_test.yaml
+`, val)
+}
+
+// func TestLocalPackageReaderWriter_DeleteFiles(t *testing.T) {
+// 	g, _, clean := testutil.SetupDefaultRepoAndWorkspace(t)
+// 	defer clean()
+// 	if !assert.NoError(t, os.Chdir(g.RepoDirectory)) {
+// 		return
+// 	}
+//
+// 	rw := LocalPackageReadWriter{PackagePath: "."}
+// 	nodes, err := rw.Read()
+// 	if !assert.NoError(t, err) {
+// 		t.FailNow()
+// 	}
+// 	_, err = os.Stat(filepath.Join("java", "java-deployment.resource.yaml"))
+// 	if !assert.NoError(t, err) {
+// 		t.FailNow()
+// 	}
+//
+// 	// delete one of the nodes
+// 	var newNodes []*yaml.RNode
+// 	for i := range nodes {
+// 		meta, err := nodes[i].GetMeta()
+// 		if !assert.NoError(t, err) {
+// 			t.FailNow()
+// 		}
+// 		if meta.Name == "app" && meta.Kind == "Deployment" {
+// 			continue
+// 		}
+// 		newNodes = append(newNodes, nodes[i])
+// 	}
+//
+// 	if !assert.NoError(t, rw.Write(newNodes)) {
+// 		t.FailNow()
+// 	}
+//
+// 	_, err = os.Stat(filepath.Join("java", "java-deployment.resource.yaml"))
+// 	if !assert.Error(t, err) {
+// 		t.FailNow()
+// 	}
+//
+// 	diff, err := copyutil.Diff(filepath.Join(g.DatasetDirectory, testutil.Dataset1), ".")
+// 	if !assert.NoError(t, err) {
+// 		t.FailNow()
+// 	}
+//
+// 	assert.ElementsMatch(t,
+// 		diff.List(),
+// 		[]string{filepath.Join("java", "java-deployment.resource.yaml")})
+// }

kyaml/kio/pkgio_writer.go

@@ -0,0 +1,152 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package kio
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"sigs.k8s.io/kustomize/kyaml/kio/kioutil"
+	"sigs.k8s.io/kustomize/kyaml/yaml"
+)
+
+// LocalPackageWriter writes ResourceNodes to a filesystem
+type LocalPackageWriter struct {
+	Kind string `yaml:"kind,omitempty"`
+
+	// PackagePath is the path to the package directory.
+	PackagePath string `yaml:"path,omitempty"`
+
+	// KeepReaderAnnotations if set will retain the annotations set by LocalPackageReader
+	KeepReaderAnnotations bool `yaml:"keepReaderAnnotations,omitempty"`
+
+	// ClearAnnotations will clear annotations before writing the resources
+	ClearAnnotations []string `yaml:"clearAnnotations,omitempty"`
+}
+
+var _ Writer = LocalPackageWriter{}
+
+func (r LocalPackageWriter) Write(nodes []*yaml.RNode) error {
+	if err := kioutil.ErrorIfMissingAnnotation(nodes, requiredResourcePackageAnnotations...); err != nil {
+		return err
+	}
+
+	if s, err := os.Stat(r.PackagePath); err != nil {
+		return err
+	} else if !s.IsDir() {
+		// if the user specified input isn't a directory, the package is the directory of the
+		// target
+		r.PackagePath = filepath.Dir(r.PackagePath)
+	}
+
+	// setup indexes for writing Resources back to files
+	if err := r.errorIfMissingRequiredAnnotation(nodes); err != nil {
+		return err
+	}
+	outputFiles, err := r.indexByFilePath(nodes)
+	if err != nil {
+		return err
+	}
+	for k := range outputFiles {
+		if err = kioutil.SortNodes(outputFiles[k]); err != nil {
+			return err
+		}
+	}
+
+	if !r.KeepReaderAnnotations {
+		r.ClearAnnotations = append(r.ClearAnnotations, kioutil.PackageAnnotation)
+		r.ClearAnnotations = append(r.ClearAnnotations, kioutil.PathAnnotation)
+	}
+
+	// validate outputs before writing any
+	for path := range outputFiles {
+		outputPath := filepath.Join(r.PackagePath, path)
+		if st, err := os.Stat(outputPath); !os.IsNotExist(err) {
+			if err != nil {
+				return err
+			}
+			if st.IsDir() {
+				return fmt.Errorf("kyaml.kustomize.dev/kio/path cannot be a directory: %s", path)
+			}
+		}
+
+		err = os.MkdirAll(filepath.Dir(outputPath), 0700)
+		if err != nil {
+			return err
+		}
+	}
+
+	// write files
+	for path := range outputFiles {
+		outputPath := filepath.Join(r.PackagePath, path)
+		err = os.MkdirAll(filepath.Dir(filepath.Join(r.PackagePath, path)), 0700)
+		if err != nil {
+			return err
+		}
+
+		f, err := os.OpenFile(outputPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, os.FileMode(0600))
+		if err != nil {
+			return err
+		}
+		if err := func() error {
+			defer f.Close()
+			w := ByteWriter{
+				Writer:                f,
+				KeepReaderAnnotations: r.KeepReaderAnnotations,
+				ClearAnnotations:      r.ClearAnnotations,
+			}
+			if err = w.Write(outputFiles[path]); err != nil {
+				return err
+			}
+			return nil
+		}(); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (r LocalPackageWriter) errorIfMissingRequiredAnnotation(nodes []*yaml.RNode) error {
+	for i := range nodes {
+		for _, s := range requiredResourcePackageAnnotations {
+			key, err := nodes[i].Pipe(yaml.GetAnnotation(s))
+			if err != nil {
+				return err
+			}
+			if key == nil || key.YNode() == nil || key.YNode().Value == "" {
+				return fmt.Errorf(
+					"resources must be annotated with %s to be written to files", s)
+			}
+		}
+	}
+	return nil
+}
+
+func (r LocalPackageWriter) indexByFilePath(nodes []*yaml.RNode) (map[string][]*yaml.RNode, error) {
+
+	outputFiles := map[string][]*yaml.RNode{}
+	for i := range nodes {
+		// parse the file write path
+		node := nodes[i]
+		value, err := node.Pipe(yaml.GetAnnotation(kioutil.PathAnnotation))
+		if err != nil {
+			// this should never happen if errorIfMissingRequiredAnnotation was run
+			return nil, err
+		}
+		path := value.YNode().Value
+		outputFiles[path] = append(outputFiles[path], node)
+
+		if filepath.IsAbs(path) {
+			return nil, fmt.Errorf("package paths may not be absolute paths")
+		}
+		if strings.Contains(filepath.Clean(path), "..") {
+			return nil, fmt.Errorf("resource must be written under package %s: %s",
+				r.PackagePath, filepath.Clean(path))
+		}
+	}
+	return outputFiles, nil
+}

kyaml/kio/pkgio_writer_test.go

@@ -0,0 +1,329 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package kio_test
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	. "sigs.k8s.io/kustomize/kyaml/kio"
+	"sigs.k8s.io/kustomize/kyaml/yaml"
+)
+
+// TestLocalPackageWriter_Write tests:
+// - ReaderAnnotations are cleared when writing the Resources
+func TestLocalPackageWriter_Write(t *testing.T) {
+	d, node1, node2, node3 := getWriterInputs(t)
+	defer os.RemoveAll(d)
+
+	w := LocalPackageWriter{PackagePath: d}
+	err := w.Write([]*yaml.RNode{node2, node1, node3})
+	if !assert.NoError(t, err) {
+		assert.FailNow(t, err.Error())
+	}
+
+	b, err := ioutil.ReadFile(filepath.Join(d, "a", "b", "a_test.yaml"))
+	if !assert.NoError(t, err) {
+		assert.FailNow(t, err.Error())
+	}
+	assert.Equal(t, `a: b #first
+---
+c: d # second
+`, string(b))
+
+	b, err = ioutil.ReadFile(filepath.Join(d, "a", "b", "b_test.yaml"))
+	if !assert.NoError(t, err) {
+		assert.FailNow(t, err.Error())
+	}
+	assert.Equal(t, `e: f
+g:
+  h:
+  - i # has a list
+  - j
+`, string(b))
+}
+
+// TestLocalPackageWriter_Write_keepReaderAnnotations tests:
+// - ReaderAnnotations are kept when writing the Resources
+func TestLocalPackageWriter_Write_keepReaderAnnotations(t *testing.T) {
+	d, node1, node2, node3 := getWriterInputs(t)
+	defer os.RemoveAll(d)
+
+	w := LocalPackageWriter{PackagePath: d, KeepReaderAnnotations: true}
+	err := w.Write([]*yaml.RNode{node2, node1, node3})
+	if !assert.NoError(t, err) {
+		return
+	}
+
+	b, err := ioutil.ReadFile(filepath.Join(d, "a", "b", "a_test.yaml"))
+	if !assert.NoError(t, err) {
+		return
+	}
+	assert.Equal(t, `a: b #first
+metadata:
+  annotations:
+    kyaml.kustomize.dev/kio/index: 0
+    kyaml.kustomize.dev/kio/path: a/b/a_test.yaml
+---
+c: d # second
+metadata:
+  annotations:
+    kyaml.kustomize.dev/kio/index: 1
+    kyaml.kustomize.dev/kio/path: a/b/a_test.yaml
+`, string(b))
+
+	b, err = ioutil.ReadFile(filepath.Join(d, "a", "b", "b_test.yaml"))
+	if !assert.NoError(t, err) {
+		return
+	}
+	assert.Equal(t, `e: f
+g:
+  h:
+  - i # has a list
+  - j
+metadata:
+  annotations:
+    kyaml.kustomize.dev/kio/index: 0
+    kyaml.kustomize.dev/kio/path: a/b/b_test.yaml
+`, string(b))
+}
+
+// TestLocalPackageWriter_Write_clearAnnotations tests:
+// - ClearAnnotations are removed from Resources
+func TestLocalPackageWriter_Write_clearAnnotations(t *testing.T) {
+	d, node1, node2, node3 := getWriterInputs(t)
+	defer os.RemoveAll(d)
+
+	w := LocalPackageWriter{PackagePath: d, ClearAnnotations: []string{"kyaml.kustomize.dev/kio/mode"}}
+	err := w.Write([]*yaml.RNode{node2, node1, node3})
+	if !assert.NoError(t, err) {
+		assert.FailNow(t, err.Error())
+	}
+
+	b, err := ioutil.ReadFile(filepath.Join(d, "a", "b", "a_test.yaml"))
+	if !assert.NoError(t, err) {
+		assert.FailNow(t, err.Error())
+	}
+	assert.Equal(t, `a: b #first
+---
+c: d # second
+`, string(b))
+
+	b, err = ioutil.ReadFile(filepath.Join(d, "a", "b", "b_test.yaml"))
+	if !assert.NoError(t, err) {
+		assert.FailNow(t, err.Error())
+	}
+	assert.Equal(t, `e: f
+g:
+  h:
+  - i # has a list
+  - j
+`, string(b))
+}
+
+// TestLocalPackageWriter_Write_failRelativePath tests:
+// - If a relative path above the package is defined, write fails
+func TestLocalPackageWriter_Write_failRelativePath(t *testing.T) {
+	d, node1, node2, node3 := getWriterInputs(t)
+	defer os.RemoveAll(d)
+
+	node4, err := yaml.Parse(`e: f
+g:
+  h:
+  - i # has a list
+  - j
+metadata:
+  annotations:
+    kyaml.kustomize.dev/kio/index: 0
+    kyaml.kustomize.dev/kio/path: "a/b/../../../b_test.yaml"
+`)
+	if !assert.NoError(t, err) {
+		assert.FailNow(t, err.Error())
+	}
+
+	w := LocalPackageWriter{PackagePath: d}
+	err = w.Write([]*yaml.RNode{node2, node1, node3, node4})
+	if assert.Error(t, err) {
+		assert.Contains(t, err.Error(), "resource must be written under package")
+	}
+}
+
+// TestLocalPackageWriter_Write_invalidIndex tests:
+// - If a non-int index is given, fail
+func TestLocalPackageWriter_Write_invalidIndex(t *testing.T) {
+	d, node1, node2, node3 := getWriterInputs(t)
+	defer os.RemoveAll(d)
+
+	node4, err := yaml.Parse(`e: f
+g:
+  h:
+  - i # has a list
+  - j
+metadata:
+  annotations:
+    kyaml.kustomize.dev/kio/index: a
+    kyaml.kustomize.dev/kio/path: "a/b/b_test.yaml" # use a different path, should still collide
+`)
+	if !assert.NoError(t, err) {
+		assert.FailNow(t, err.Error())
+	}
+
+	w := LocalPackageWriter{PackagePath: d}
+	err = w.Write([]*yaml.RNode{node2, node1, node3, node4})
+	if assert.Error(t, err) {
+		assert.Contains(t, err.Error(), "unable to parse kyaml.kustomize.dev/kio/index")
+	}
+}
+
+// TestLocalPackageWriter_Write_absPath tests:
+// - If kyaml.kustomize.dev/kio/path is absolute, fail
+func TestLocalPackageWriter_Write_absPath(t *testing.T) {
+	d, node1, node2, node3 := getWriterInputs(t)
+	defer os.RemoveAll(d)
+
+	node4, err := yaml.Parse(fmt.Sprintf(`e: f
+g:
+  h:
+  - i # has a list
+  - j
+metadata:
+  annotations:
+    kyaml.kustomize.dev/kio/index: a
+    kyaml.kustomize.dev/kio/path: "%s/a/b/b_test.yaml" # use a different path, should still collide
+`, d))
+	if !assert.NoError(t, err) {
+		assert.FailNow(t, err.Error())
+	}
+
+	w := LocalPackageWriter{PackagePath: d}
+	err = w.Write([]*yaml.RNode{node2, node1, node3, node4})
+	if assert.Error(t, err) {
+		assert.Contains(t, err.Error(), "package paths may not be absolute paths")
+	}
+}
+
+// TestLocalPackageWriter_Write_missingIndex tests:
+// - If kyaml.kustomize.dev/kio/path is missing, fail
+func TestLocalPackageWriter_Write_missingPath(t *testing.T) {
+	d, node1, node2, node3 := getWriterInputs(t)
+	defer os.RemoveAll(d)
+
+	node4, err := yaml.Parse(`e: f
+g:
+  h:
+  - i # has a list
+  - j
+metadata:
+  annotations:
+    kyaml.kustomize.dev/kio/index: a
+`)
+	if !assert.NoError(t, err) {
+		assert.FailNow(t, err.Error())
+	}
+
+	w := LocalPackageWriter{PackagePath: d}
+	err = w.Write([]*yaml.RNode{node2, node1, node3, node4})
+	if assert.Error(t, err) {
+		assert.Contains(t, err.Error(), "kyaml.kustomize.dev/kio/path")
+	}
+}
+
+// TestLocalPackageWriter_Write_missingIndex tests:
+// - If kyaml.kustomize.dev/kio/index is missing, fail
+func TestLocalPackageWriter_Write_missingIndex(t *testing.T) {
+	d, node1, node2, node3 := getWriterInputs(t)
+	defer os.RemoveAll(d)
+
+	node4, err := yaml.Parse(`e: f
+g:
+  h:
+  - i # has a list
+  - j
+metadata:
+  annotations:
+    kyaml.kustomize.dev/kio/path: a/a.yaml
+`)
+	if !assert.NoError(t, err) {
+		assert.FailNow(t, err.Error())
+	}
+
+	w := LocalPackageWriter{PackagePath: d}
+	err = w.Write([]*yaml.RNode{node2, node1, node3, node4})
+	if assert.Error(t, err) {
+		assert.Contains(t, err.Error(), "kyaml.kustomize.dev/kio/index")
+	}
+}
+
+// TestLocalPackageWriter_Write_pathIsDir tests:
+// - If  kyaml.kustomize.dev/kio/path is a directory, fail
+func TestLocalPackageWriter_Write_pathIsDir(t *testing.T) {
+	d, node1, node2, node3 := getWriterInputs(t)
+	defer os.RemoveAll(d)
+
+	node4, err := yaml.Parse(`e: f
+g:
+  h:
+  - i # has a list
+  - j
+metadata:
+  annotations:
+    kyaml.kustomize.dev/kio/path: a/
+    kyaml.kustomize.dev/kio/index: 0
+`)
+	if !assert.NoError(t, err) {
+		assert.FailNow(t, err.Error())
+	}
+
+	w := LocalPackageWriter{PackagePath: d}
+	err = w.Write([]*yaml.RNode{node2, node1, node3, node4})
+	if assert.Error(t, err) {
+		assert.Contains(t, err.Error(), "kyaml.kustomize.dev/kio/path cannot be a directory")
+	}
+}
+
+func getWriterInputs(t *testing.T) (string, *yaml.RNode, *yaml.RNode, *yaml.RNode) {
+	node1, err := yaml.Parse(`a: b #first
+metadata:
+  annotations:
+    kyaml.kustomize.dev/kio/index: 0
+    kyaml.kustomize.dev/kio/path: "a/b/a_test.yaml"
+`)
+	if !assert.NoError(t, err) {
+		assert.FailNow(t, err.Error())
+	}
+	node2, err := yaml.Parse(`c: d # second
+metadata:
+  annotations:
+    kyaml.kustomize.dev/kio/index: 1
+    kyaml.kustomize.dev/kio/path: "a/b/a_test.yaml"
+`)
+	if !assert.NoError(t, err) {
+		assert.FailNow(t, err.Error())
+	}
+	node3, err := yaml.Parse(`e: f
+g:
+  h:
+  - i # has a list
+  - j
+metadata:
+  annotations:
+    kyaml.kustomize.dev/kio/index: 0
+    kyaml.kustomize.dev/kio/path: "a/b/b_test.yaml"
+`)
+	if !assert.NoError(t, err) {
+		assert.FailNow(t, err.Error())
+	}
+	d, err := ioutil.TempDir("", "kyaml-test")
+	if !assert.NoError(t, err) {
+		assert.FailNow(t, err.Error())
+	}
+	if !assert.NoError(t, os.MkdirAll(filepath.Join(d, "a"), 0700)) {
+		assert.FailNow(t, "")
+	}
+	return d, node1, node2, node3
+}

kyaml/kio/tree.go

@@ -0,0 +1,491 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package kio
+
+import (
+	"fmt"
+	"io"
+	"path/filepath"
+	"sort"
+	"strings"
+
+	"github.com/xlab/treeprint"
+	"sigs.k8s.io/kustomize/kyaml/kio/kioutil"
+	"sigs.k8s.io/kustomize/kyaml/yaml"
+)
+
+type TreeStructure string
+
+const (
+	// TreeStructurePackage configures TreeWriter to generate the tree structure off of the
+	// Resources packages.
+	TreeStructurePackage TreeStructure = "package"
+
+	// TreeStructureOwners configures TreeWriter to generate the tree structure off of the
+	// Resource owners.
+	TreeStructureGraph TreeStructure = "graph"
+)
+
+// TreeWriter prints the package structured as a tree
+// TODO(pwittrock): test this package better.  it is lower-risk since it is only
+// used for printing rather than updating or editing.
+type TreeWriter struct {
+	Writer    io.Writer
+	Root      string
+	Fields    []TreeWriterField
+	Structure TreeStructure
+}
+
+// TreeWriterField configures a Resource field to be included in the tree
+type TreeWriterField struct {
+	yaml.PathMatcher
+	Name    string
+	SubName string
+}
+
+func (p TreeWriter) packageStructure(nodes []*yaml.RNode) error {
+	indexByPackage := p.index(nodes)
+
+	// create the new tree
+	tree := treeprint.New()
+	tree.SetValue(p.Root)
+
+	// add each package to the tree
+	treeIndex := map[string]treeprint.Tree{}
+	keys := p.sort(indexByPackage)
+	for _, pkg := range keys {
+		// create a branch for this package -- search for the parent package and create
+		// the branch under it -- requires that the keys are sorted
+		branch := tree
+		for parent, subTree := range treeIndex {
+			if strings.HasPrefix(pkg, parent) {
+				// found a package whose path is a prefix to our own, use this
+				// package if a closer one isn't found
+				branch = subTree
+				// don't break, continue searching for more closely related ancestors
+			}
+		}
+
+		// create a new branch for the package
+		createOk := pkg != "." // special edge case logic for tree on current working dir
+		if createOk {
+			branch = branch.AddBranch(pkg)
+		}
+
+		// cache the branch for this package
+		treeIndex[pkg] = branch
+
+		// print each resource in the package
+		for i := range indexByPackage[pkg] {
+			var err error
+			if _, err = p.doResource(indexByPackage[pkg][i], "", branch); err != nil {
+				return err
+			}
+		}
+	}
+
+	_, err := io.WriteString(p.Writer, tree.String())
+	return err
+}
+
+// Write writes the ascii tree to p.Writer
+func (p TreeWriter) Write(nodes []*yaml.RNode) error {
+	switch p.Structure {
+	case TreeStructurePackage:
+		return p.packageStructure(nodes)
+	case TreeStructureGraph:
+		return p.graphStructure(nodes)
+	default:
+		return p.packageStructure(nodes)
+	}
+}
+
+// node wraps a tree node, and any children nodes
+type node struct {
+	p TreeWriter
+	*yaml.RNode
+	children []*node
+}
+
+func (a node) Len() int      { return len(a.children) }
+func (a node) Swap(i, j int) { a.children[i], a.children[j] = a.children[j], a.children[i] }
+func (a node) Less(i, j int) bool {
+	return compareNodes(a.children[i].RNode, a.children[j].RNode)
+}
+
+// Tree adds this node to the root
+func (a node) Tree(root treeprint.Tree) error {
+	sort.Sort(a)
+	branch := root
+	var err error
+
+	// generate a node for the Resource
+	if a.RNode != nil {
+		branch, err = a.p.doResource(a.RNode, "Resource", root)
+		if err != nil {
+			return err
+		}
+	}
+
+	// attach children to the branch
+	for _, n := range a.children {
+		if err := n.Tree(branch); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// graphStructure writes the tree using owners for structure
+func (p TreeWriter) graphStructure(nodes []*yaml.RNode) error {
+	resourceToOwner := map[string]*node{}
+	root := &node{}
+	// index each of the nodes by their owner
+	for _, n := range nodes {
+		ownerVal, err := ownerToString(n)
+		if err != nil {
+			return err
+		}
+		var owner *node
+		if ownerVal == "" {
+			// no owner -- attach to the root
+			owner = root
+		} else {
+			// owner found -- attach to the owner
+			var found bool
+			owner, found = resourceToOwner[ownerVal]
+			if !found {
+				// initialize the owner if not found
+				resourceToOwner[ownerVal] = &node{p: p}
+				owner = resourceToOwner[ownerVal]
+			}
+		}
+
+		nodeVal, err := nodeToString(n)
+		if err != nil {
+			return err
+		}
+		val, found := resourceToOwner[nodeVal]
+		if !found {
+			// initialize the node if not found -- may have already been initialized if it
+			// is the owner of another node
+			resourceToOwner[nodeVal] = &node{p: p}
+			val = resourceToOwner[nodeVal]
+		}
+		val.RNode = n
+		owner.children = append(owner.children, val)
+	}
+
+	for k, v := range resourceToOwner {
+		if v.RNode == nil {
+			return fmt.Errorf(
+				"owner '%s' not found in input, but found as an owner of input objects", k)
+		}
+	}
+
+	// print the tree
+	tree := treeprint.New()
+	if err := root.Tree(tree); err != nil {
+		return err
+	}
+
+	_, err := io.WriteString(p.Writer, tree.String())
+	return err
+}
+
+// nodeToString generates a string to identify the node -- matches ownerToString format
+func nodeToString(node *yaml.RNode) (string, error) {
+	meta, err := node.GetMeta()
+	if err != nil {
+		return "", err
+	}
+
+	return fmt.Sprintf("%s %s/%s", meta.Kind, meta.Namespace, meta.Name), nil
+}
+
+// ownerToString generate a string to identify the owner -- matches nodeToString format
+func ownerToString(node *yaml.RNode) (string, error) {
+	meta, err := node.GetMeta()
+	if err != nil {
+		return "", err
+	}
+	namespace := meta.Namespace
+
+	owners, err := node.Pipe(yaml.Lookup("metadata", "ownerReferences"))
+	if err != nil {
+		return "", err
+	}
+	if owners == nil {
+		return "", nil
+	}
+
+	elements, err := owners.Elements()
+	if err != nil {
+		return "", err
+	}
+	if len(elements) == 0 {
+		return "", err
+	}
+	owner := elements[0]
+	var kind, name string
+
+	if value := owner.Field("kind"); !yaml.IsFieldEmpty(value) {
+		kind = value.Value.YNode().Value
+	}
+	if value := owner.Field("name"); !yaml.IsFieldEmpty(value) {
+		name = value.Value.YNode().Value
+	}
+
+	return fmt.Sprintf("%s %s/%s", kind, namespace, name), nil
+}
+
+// index indexes the Resources by their package
+func (p TreeWriter) index(nodes []*yaml.RNode) map[string][]*yaml.RNode {
+	// index the ResourceNodes by package
+	indexByPackage := map[string][]*yaml.RNode{}
+	for i := range nodes {
+		meta, err := nodes[i].GetMeta()
+		if err != nil || meta.Kind == "" {
+			// not a resource
+			continue
+		}
+		pkg := meta.Annotations[kioutil.PackageAnnotation]
+		indexByPackage[pkg] = append(indexByPackage[pkg], nodes[i])
+	}
+	return indexByPackage
+}
+
+func compareNodes(i, j *yaml.RNode) bool {
+	metai, _ := i.GetMeta()
+	metaj, _ := j.GetMeta()
+	pi := metai.Annotations[kioutil.PathAnnotation]
+	pj := metaj.Annotations[kioutil.PathAnnotation]
+
+	// compare file names
+	if filepath.Base(pi) != filepath.Base(pj) {
+		return filepath.Base(pi) < filepath.Base(pj)
+	}
+
+	// compare namespace
+	if metai.Namespace != metaj.Namespace {
+		return metai.Namespace < metaj.Namespace
+	}
+
+	// compare name
+	if metai.Name != metaj.Name {
+		return metai.Name < metaj.Name
+	}
+
+	// compare kind
+	if metai.Kind != metaj.Kind {
+		return metai.Kind < metaj.Kind
+	}
+
+	// compare apiVersion
+	if metai.ApiVersion != metaj.ApiVersion {
+		return metai.ApiVersion < metaj.ApiVersion
+	}
+	return true
+}
+
+// sort sorts the Resources in the index in display order and returns the ordered
+// keys for the index
+//
+// Packages are sorted by package name
+// Resources within a package are sorted by: [filename, namespace, name, kind, apiVersion]
+func (p TreeWriter) sort(indexByPackage map[string][]*yaml.RNode) []string {
+	var keys []string
+	for k := range indexByPackage {
+		pkgNodes := indexByPackage[k]
+		sort.Slice(pkgNodes, func(i, j int) bool { return compareNodes(pkgNodes[i], pkgNodes[j]) })
+		keys = append(keys, k)
+	}
+
+	// return the package names sorted lexicographically
+	sort.Strings(keys)
+	return keys
+}
+
+func (p TreeWriter) doResource(leaf *yaml.RNode, metaString string, branch treeprint.Tree) (treeprint.Tree, error) {
+	meta, _ := leaf.GetMeta()
+	if metaString == "" {
+		path := meta.Annotations[kioutil.PathAnnotation]
+		path = filepath.Base(path)
+		metaString = path
+	}
+
+	value := fmt.Sprintf("%s %s", meta.Kind, meta.Name)
+	if len(meta.Namespace) > 0 {
+		value = fmt.Sprintf("%s %s/%s", meta.Kind, meta.Namespace, meta.Name)
+	}
+
+	fields, err := p.getFields(leaf)
+	if err != nil {
+		return nil, err
+	}
+
+	n := branch.AddMetaBranch(metaString, value)
+	for i := range fields {
+		field := fields[i]
+
+		// do leaf node
+		if len(field.matchingElementsAndFields) == 0 {
+			n.AddNode(fmt.Sprintf("%s: %s", field.name, field.value))
+			continue
+		}
+
+		// do nested nodes
+		b := n.AddBranch(field.name)
+		for j := range field.matchingElementsAndFields {
+			elem := field.matchingElementsAndFields[j]
+			b := b.AddBranch(elem.name)
+			for k := range elem.matchingElementsAndFields {
+				field := elem.matchingElementsAndFields[k]
+				b.AddNode(fmt.Sprintf("%s: %s", field.name, field.value))
+			}
+		}
+	}
+
+	return n, nil
+}
+
+// getFields looks up p.Fields from leaf and structures them into treeFields.
+// TODO(pwittrock): simplify this function
+func (p TreeWriter) getFields(leaf *yaml.RNode) (treeFields, error) {
+	fieldsByName := map[string]*treeField{}
+
+	// index nested and non-nested fields
+	for i := range p.Fields {
+		f := p.Fields[i]
+		seq, err := leaf.Pipe(&f)
+		if err != nil {
+			return nil, err
+		}
+		if seq == nil {
+			continue
+		}
+
+		if fieldsByName[f.Name] == nil {
+			fieldsByName[f.Name] = &treeField{name: f.Name}
+		}
+
+		// non-nested field -- add directly to the treeFields list
+		if f.SubName == "" {
+			// non-nested field -- only 1 element
+			val, err := yaml.String(seq.Content()[0], yaml.Trim, yaml.Flow)
+			if err != nil {
+				return nil, err
+			}
+			fieldsByName[f.Name].value = val
+			continue
+		}
+
+		// nested-field -- create a parent elem, and index by the 'match' value
+		if fieldsByName[f.Name].subFieldByMatch == nil {
+			fieldsByName[f.Name].subFieldByMatch = map[string]treeFields{}
+		}
+		index := fieldsByName[f.Name].subFieldByMatch
+		for j := range seq.Content() {
+			elem := seq.Content()[j]
+			matches := f.Matches[elem]
+			str, err := yaml.String(elem, yaml.Trim, yaml.Flow)
+			if err != nil {
+				return nil, err
+			}
+
+			// map the field by the name of the element
+			// index the subfields by the matching element so we can put all the fields for the
+			// same element under the same branch
+			matchKey := strings.Join(matches, "/")
+			index[matchKey] = append(index[matchKey], &treeField{name: f.SubName, value: str})
+		}
+	}
+
+	// iterate over collection of all queried fields in the Resource
+	for _, field := range fieldsByName {
+		// iterate over collection of elements under the field -- indexed by element name
+		for match, subFields := range field.subFieldByMatch {
+			// create a new element for this collection of fields
+			// note: we will convert name to an index later, but keep the match for sorting
+			elem := &treeField{name: match}
+			field.matchingElementsAndFields = append(field.matchingElementsAndFields, elem)
+
+			// iterate over collection of queried fields for the element
+			for i := range subFields {
+				// add to the list of fields for this element
+				elem.matchingElementsAndFields = append(elem.matchingElementsAndFields, subFields[i])
+			}
+		}
+		// clear this cached data
+		field.subFieldByMatch = nil
+	}
+
+	// put the fields in a list so they are ordered
+	fieldList := treeFields{}
+	for _, v := range fieldsByName {
+		fieldList = append(fieldList, v)
+	}
+
+	// sort the fields
+	sort.Sort(fieldList)
+	for i := range fieldList {
+		field := fieldList[i]
+		// sort the elements under this field
+		sort.Sort(field.matchingElementsAndFields)
+
+		for i := range field.matchingElementsAndFields {
+			element := field.matchingElementsAndFields[i]
+			// sort the elements under a list field by their name
+			sort.Sort(element.matchingElementsAndFields)
+			// set the name of the element to its index
+			element.name = fmt.Sprintf("%d", i)
+		}
+	}
+
+	return fieldList, nil
+}
+
+// treeField wraps a field node
+type treeField struct {
+	// name is the name of the node
+	name string
+
+	// value is the value of the node -- may be empty
+	value string
+
+	// matchingElementsAndFields is a slice of fields that go under this as a branch
+	matchingElementsAndFields treeFields
+
+	// subFieldByMatch caches matchingElementsAndFields indexed by the name of the matching elem
+	subFieldByMatch map[string]treeFields
+}
+
+// treeFields wraps a slice of treeField so they can be sorted
+type treeFields []*treeField
+
+func (nodes treeFields) Len() int { return len(nodes) }
+
+func (nodes treeFields) Less(i, j int) bool {
+	iIndex, iFound := yaml.FieldOrder[nodes[i].name]
+	jIndex, jFound := yaml.FieldOrder[nodes[j].name]
+	if iFound && jFound {
+		return iIndex < jIndex
+	}
+	if iFound {
+		return true
+	}
+	if jFound {
+		return false
+	}
+
+	if nodes[i].name != nodes[j].name {
+		return nodes[i].name < nodes[j].name
+	}
+	if nodes[i].value != nodes[j].value {
+		return nodes[i].value < nodes[j].value
+	}
+	return false
+}
+
+func (nodes treeFields) Swap(i, j int) { nodes[i], nodes[j] = nodes[j], nodes[i] }

kyaml/kio/tree_test.go

@@ -0,0 +1,386 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package kio_test
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	. "sigs.k8s.io/kustomize/kyaml/kio"
+	"sigs.k8s.io/kustomize/kyaml/yaml"
+)
+
+func TestPrinter_Write(t *testing.T) {
+	in := `kind: Deployment
+metadata:
+  labels:
+    app: nginx3
+  name: foo
+  namespace: default
+  annotations:
+    app: nginx3
+    kyaml.kustomize.dev/kio/package: foo-package/3
+    kyaml.kustomize.dev/kio/path: f3.yaml
+spec:
+  replicas: 1
+---
+kind: Deployment
+metadata:
+  labels:
+    app: nginx2
+  name: foo
+  namespace: default
+  annotations:
+    app: nginx2
+    kyaml.kustomize.dev/kio/package: foo-package
+    kyaml.kustomize.dev/kio/path: f1.yaml
+spec:
+  replicas: 1
+---
+kind: Deployment
+metadata:
+  labels:
+    app: nginx
+  annotations:
+    app: nginx
+    kyaml.kustomize.dev/kio/package: bar-package
+    kyaml.kustomize.dev/kio/path: f2.yaml
+  name: bar
+spec:
+  replicas: 3
+---
+kind: Service
+metadata:
+  name: foo
+  namespace: default
+  annotations:
+    app: nginx
+    kyaml.kustomize.dev/kio/package: foo-package
+    kyaml.kustomize.dev/kio/path: f1.yaml
+spec:
+  selector:
+    app: nginx
+`
+	out := &bytes.Buffer{}
+	err := Pipeline{
+		Inputs:  []Reader{&ByteReader{Reader: bytes.NewBufferString(in)}},
+		Outputs: []Writer{TreeWriter{Writer: out}},
+	}.Execute()
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
+
+	if !assert.Equal(t, `
+├── bar-package
+│   └── [f2.yaml]  Deployment bar
+└── foo-package
+    ├── [f1.yaml]  Deployment default/foo
+    ├── [f1.yaml]  Service default/foo
+    └── foo-package/3
+        └── [f3.yaml]  Deployment default/foo
+`, out.String()) {
+		t.FailNow()
+	}
+}
+
+func TestPrinter_Write_base(t *testing.T) {
+	in := `kind: Deployment
+metadata:
+  labels:
+    app: nginx3
+  name: foo
+  namespace: default
+  annotations:
+    app: nginx3
+    kyaml.kustomize.dev/kio/package: .
+    kyaml.kustomize.dev/kio/path: f3.yaml
+spec:
+  replicas: 1
+---
+kind: Deployment
+metadata:
+  labels:
+    app: nginx2
+  name: foo
+  namespace: default
+  annotations:
+    app: nginx2
+    kyaml.kustomize.dev/kio/package: foo-package
+    kyaml.kustomize.dev/kio/path: f1.yaml
+spec:
+  replicas: 1
+---
+kind: Deployment
+metadata:
+  labels:
+    app: nginx
+  annotations:
+    app: nginx
+    kyaml.kustomize.dev/kio/package: bar-package
+    kyaml.kustomize.dev/kio/path: f2.yaml
+  name: bar
+spec:
+  replicas: 3
+---
+kind: Service
+metadata:
+  name: foo
+  namespace: default
+  annotations:
+    app: nginx
+    kyaml.kustomize.dev/kio/package: .
+    kyaml.kustomize.dev/kio/path: f1.yaml
+spec:
+  selector:
+    app: nginx
+`
+	out := &bytes.Buffer{}
+	err := Pipeline{
+		Inputs:  []Reader{&ByteReader{Reader: bytes.NewBufferString(in)}},
+		Outputs: []Writer{TreeWriter{Writer: out}},
+	}.Execute()
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
+
+	if !assert.Equal(t, `
+├── [f1.yaml]  Service default/foo
+├── [f3.yaml]  Deployment default/foo
+├── bar-package
+│   └── [f2.yaml]  Deployment bar
+└── foo-package
+    └── [f1.yaml]  Deployment default/foo
+`, out.String()) {
+		t.FailNow()
+	}
+}
+
+func TestPrinter_Write_sort(t *testing.T) {
+	in := `apiVersion: extensions/v1
+kind: Deployment
+metadata:
+  labels:
+    app: nginx2
+  name: foo3
+  namespace: default
+  annotations:
+    app: nginx2
+    kyaml.kustomize.dev/kio/package: .
+    kyaml.kustomize.dev/kio/path: f1.yaml
+spec:
+  replicas: 1
+---
+apiVersion: extensions/v1
+kind: Deployment
+metadata:
+  labels:
+    app: nginx2
+  name: foo3
+  namespace: default
+  annotations:
+    app: nginx2
+    kyaml.kustomize.dev/kio/package: .
+    kyaml.kustomize.dev/kio/path: f1.yaml
+spec:
+  replicas: 1
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: nginx2
+  name: foo3
+  namespace: default
+  annotations:
+    app: nginx2
+    kyaml.kustomize.dev/kio/package: .
+    kyaml.kustomize.dev/kio/path: f1.yaml
+spec:
+  replicas: 1
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: nginx2
+  name: foo2
+  namespace: default2
+  annotations:
+    app: nginx2
+    kyaml.kustomize.dev/kio/package: .
+    kyaml.kustomize.dev/kio/path: f1.yaml
+spec:
+  replicas: 1
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: nginx3
+  name: foo
+  namespace: default
+  annotations:
+    app: nginx3
+    kyaml.kustomize.dev/kio/package: .
+    kyaml.kustomize.dev/kio/path: f1.yaml
+spec:
+  replicas: 1
+---
+kind: Deployment
+metadata:
+  labels:
+    app: nginx
+  annotations:
+    app: nginx
+    kyaml.kustomize.dev/kio/package: bar-package
+    kyaml.kustomize.dev/kio/path: f2.yaml
+  name: bar
+spec:
+  replicas: 3
+---
+kind: Service
+metadata:
+  name: foo
+  namespace: default
+  annotations:
+    app: nginx
+    kyaml.kustomize.dev/kio/package: .
+    kyaml.kustomize.dev/kio/path: f1.yaml
+spec:
+  selector:
+    app: nginx
+`
+	out := &bytes.Buffer{}
+	err := Pipeline{
+		Inputs:  []Reader{&ByteReader{Reader: bytes.NewBufferString(in)}},
+		Outputs: []Writer{TreeWriter{Writer: out}},
+	}.Execute()
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
+
+	if !assert.Equal(t, `
+├── [f1.yaml]  Deployment default/foo
+├── [f1.yaml]  Service default/foo
+├── [f1.yaml]  Deployment default/foo3
+├── [f1.yaml]  Deployment default/foo3
+├── [f1.yaml]  Deployment default/foo3
+├── [f1.yaml]  Deployment default2/foo2
+└── bar-package
+    └── [f2.yaml]  Deployment bar
+`, out.String()) {
+		t.FailNow()
+	}
+}
+
+func TestPrinter_metaError(t *testing.T) {
+	out := &bytes.Buffer{}
+	err := TreeWriter{Writer: out}.Write([]*yaml.RNode{{}})
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
+	if !assert.Equal(t, `
+`, out.String()) {
+		t.FailNow()
+	}
+}
+
+func TestPrinter_Write_owners(t *testing.T) {
+	in := `
+apiVersion: v1
+kind: Pod
+metadata:
+  name: cockroachdb-0
+  namespace: myapp-staging
+  ownerReferences:
+  - apiVersion: apps/v1
+    kind: StatefulSet
+    name: cockroachdb
+spec:
+  containers:
+  - name: cockroachdb
+    image: cockraochdb:1.1.1
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: cockroachdb-1
+  namespace: myapp-staging
+  ownerReferences:
+  - apiVersion: apps/v1
+    kind: StatefulSet
+    name: cockroachdb
+spec:
+  containers:
+  - name: cockroachdb
+    image: cockraochdb:1.1.1
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: cockroachdb-2
+  namespace: myapp-staging
+  ownerReferences:
+  - apiVersion: apps/v1
+    kind: StatefulSet
+    name: cockroachdb
+spec:
+  containers:
+  - name: cockroachdb
+    image: cockraochdb:1.1.0
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: cockroachdb
+  namespace: myapp-staging
+  ownerReferences:
+  - apiVersion: app.k8s.io/v1beta1
+    kind: Application
+    name: myapp
+spec:
+  replicas: 3
+  containers:
+  - name: cockroachdb
+    image: cockraochdb:1.1.1
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: cockroachdb
+  namespace: myapp-staging
+  ownerReferences:
+  - apiVersion: app.k8s.io/v1beta1
+    kind: Application
+    name: myapp
+---
+apiVersion: app.k8s.io/v1beta1
+kind: Application
+metadata:
+  labels:
+    app.kubernetes.io/name: myapp
+  name: myapp
+  namespace: myapp-staging
+`
+	out := &bytes.Buffer{}
+	err := Pipeline{
+		Inputs:  []Reader{&ByteReader{Reader: bytes.NewBufferString(in)}},
+		Outputs: []Writer{TreeWriter{Writer: out, Structure: TreeStructureGraph}},
+	}.Execute()
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
+
+	if !assert.Equal(t, `.
+└── [Resource]  Application myapp-staging/myapp
+    ├── [Resource]  Service myapp-staging/cockroachdb
+    └── [Resource]  StatefulSet myapp-staging/cockroachdb
+        ├── [Resource]  Pod myapp-staging/cockroachdb-0
+        ├── [Resource]  Pod myapp-staging/cockroachdb-1
+        └── [Resource]  Pod myapp-staging/cockroachdb-2
+`, out.String()) {
+		t.FailNow()
+	}
+}

kyaml/sets/sets.go

@@ -0,0 +1,64 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package sets
+
+type String map[string]interface{}
+
+func (s String) Len() int {
+	return len(s)
+}
+
+func (s String) List() []string {
+	var val []string
+	for k := range s {
+		val = append(val, k)
+	}
+	return val
+}
+
+func (s String) Has(val string) bool {
+	_, found := s[val]
+	return found
+}
+
+func (s String) Insert(vals ...string) {
+	for _, val := range vals {
+		s[val] = nil
+	}
+}
+
+func (s String) Difference(s2 String) String {
+	s3 := String{}
+	for k := range s {
+		if _, found := s2[k]; !found {
+			s3.Insert(k)
+		}
+	}
+	return s3
+}
+
+func (s String) SymmetricDifference(s2 String) String {
+	s3 := String{}
+	for k := range s {
+		if _, found := s2[k]; !found {
+			s3.Insert(k)
+		}
+	}
+	for k := range s2 {
+		if _, found := s[k]; !found {
+			s3.Insert(k)
+		}
+	}
+	return s3
+}
+
+func (s String) Intersection(s2 String) String {
+	s3 := String{}
+	for k := range s {
+		if _, found := s2[k]; found {
+			s3.Insert(k)
+		}
+	}
+	return s3
+}

kyaml/yaml/alias.go

@@ -0,0 +1,44 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package yaml
+
+import (
+	"io"
+
+	"gopkg.in/yaml.v3"
+)
+
+// Expose the yaml.v3 functions so this package can be used as a replacement
+
+type Decoder = yaml.Decoder
+type Encoder = yaml.Encoder
+type IsZeroer = yaml.IsZeroer
+type Kind = yaml.Kind
+type Marshaler = yaml.Marshaler
+type Node = yaml.Node
+type Style = yaml.Style
+type TypeError = yaml.TypeError
+type Unmarshaler = yaml.Unmarshaler
+
+var Marshal = yaml.Marshal
+var UnMarshal = yaml.Unmarshal
+var NewDecoder = yaml.NewDecoder
+var NewEncoder = func(w io.Writer) *yaml.Encoder {
+	e := yaml.NewEncoder(w)
+	e.SetIndent(2)
+	return e
+}
+
+var AliasNode yaml.Kind = yaml.AliasNode
+var DocumentNode yaml.Kind = yaml.DocumentNode
+var MappingNode yaml.Kind = yaml.MappingNode
+var ScalarNode yaml.Kind = yaml.ScalarNode
+var SequenceNode yaml.Kind = yaml.SequenceNode
+
+var DoubleQuotedStyle yaml.Style = yaml.DoubleQuotedStyle
+var FlowStyle yaml.Style = yaml.FlowStyle
+var FoldedStyle yaml.Style = yaml.FoldedStyle
+var LiteralStyle yaml.Style = yaml.LiteralStyle
+var SingleQuotedStyle yaml.Style = yaml.SingleQuotedStyle
+var TaggedStyle yaml.Style = yaml.TaggedStyle

kyaml/yaml/doc.go

@@ -0,0 +1,12 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+// Package yaml contains low-level libraries for manipulating individual Kubernetes Resource
+// Configuration yaml.
+//
+// It exports the public pieces of "gopkg.in/yaml.v3", so can be used as a drop in replacement.
+//
+// This package should be used over sigs.k8s.io/yaml:
+// - If retaining or modifying yaml comments, structure, formatting
+// - If Resources should be round tripped without dropping fields
+package yaml

kyaml/yaml/example_test.go

@@ -0,0 +1,841 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package yaml_test
+
+import (
+	"fmt"
+	"log"
+
+	. "sigs.k8s.io/kustomize/kyaml/yaml"
+)
+
+func Example() {
+	obj, err := Parse(`apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: app
+  labels:
+    app: java
+  annotations:
+    a.b.c: d.e.f
+`)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	containers, err := Parse(`
+- name: nginx # first container
+  image: nginx
+- name: nginx2 # second container
+  image: nginx2
+`)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	node, err := obj.Pipe(
+		LookupCreate(SequenceNode, "spec", "template", "spec", "containers"),
+		Append(containers.YNode().Content...))
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	fmt.Println(node.String())
+	fmt.Println(obj.String())
+	// Output:
+	//  <nil>
+	// apiVersion: apps/v1
+	// kind: Deployment
+	// metadata:
+	//   name: app
+	//   labels:
+	//     app: java
+	//   annotations:
+	//     a.b.c: d.e.f
+	// spec:
+	//   template:
+	//     spec:
+	//       containers:
+	//       - name: nginx # first container
+	//         image: nginx
+	//       - name: nginx2 # second container
+	//         image: nginx2
+	//  <nil>
+}
+
+func ExampleAppend_appendScalars() {
+	obj, err := Parse(`
+- a
+- b
+`)
+	if err != nil {
+		log.Fatal(err)
+	}
+	_, err = obj.Pipe(Append(&Node{Value: "c", Kind: ScalarNode}))
+	if err != nil {
+		log.Fatal(err)
+	}
+	node, err := obj.Pipe(Append(
+		&Node{Value: "c", Kind: ScalarNode},
+		&Node{Value: "d", Kind: ScalarNode},
+	))
+	if err != nil {
+		log.Fatal(err)
+	}
+	fmt.Println(node.String())
+	fmt.Println(obj.String())
+	// Output:
+	//  <nil>
+	// - a
+	// - b
+	// - c
+	// - c
+	// - d
+	//  <nil>
+}
+
+func ExampleAppend_appendMap() {
+	obj, err := Parse(`
+- name: foo
+- name: bar
+`)
+	if err != nil {
+		log.Fatal(err)
+	}
+	elem, err := Parse("name: baz")
+	if err != nil {
+		log.Fatal(err)
+	}
+	node, err := obj.Pipe(Append(elem.YNode()))
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	// Expect the node to contain the appended element because only
+	// 1 element was appended
+	fmt.Println(node.String())
+	fmt.Println(obj.String())
+	// Output:
+	// name: baz
+	//  <nil>
+	// - name: foo
+	// - name: bar
+	// - name: baz
+	//  <nil>
+}
+
+func ExampleClear() {
+	obj, err := Parse(`
+kind: Deployment
+metadata:
+  name: app
+  annotations:
+    a.b.c: d.e.f
+    g: h
+spec:
+  template: {}
+`)
+	if err != nil {
+		log.Fatal(err)
+	}
+	node, err := obj.Pipe(Clear("metadata"))
+	if err != nil {
+		log.Fatal(err)
+	}
+	fmt.Println(node.String())
+	fmt.Println(obj.String())
+	// Output:
+	// name: app
+	// annotations:
+	//   a.b.c: d.e.f
+	//   g: h
+	//  <nil>
+	// kind: Deployment
+	// spec:
+	//   template: {}
+	//  <nil>
+}
+
+func ExampleGet() {
+	obj, err := Parse(`apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: app
+  annotations:
+    a.b.c: d.e.f
+    g: h
+spec:
+  template: {}
+`)
+	if err != nil {
+		log.Fatal(err)
+	}
+	node, err := obj.Pipe(Get("metadata"))
+	if err != nil {
+		log.Fatal(err)
+	}
+	fmt.Println(node.String())
+	fmt.Println(obj.String())
+	// Output:
+	// name: app
+	// annotations:
+	//   a.b.c: d.e.f
+	//   g: h
+	//  <nil>
+	// apiVersion: apps/v1
+	// kind: Deployment
+	// metadata:
+	//   name: app
+	//   annotations:
+	//     a.b.c: d.e.f
+	//     g: h
+	// spec:
+	//   template: {}
+	//  <nil>
+}
+
+func ExampleGet_notFound() {
+	obj, err := Parse(`apiVersion: apps/v1
+kind: Deployment
+spec:
+  template: {}
+`)
+	if err != nil {
+		log.Fatal(err)
+	}
+	node, err := obj.Pipe(FieldMatcher{Name: "metadata"})
+	if err != nil {
+		log.Fatal(err)
+	}
+	fmt.Println(node.String())
+	fmt.Println(obj.String())
+	// Output:
+	//  <nil>
+	// apiVersion: apps/v1
+	// kind: Deployment
+	// spec:
+	//   template: {}
+	//  <nil>
+}
+
+func ExampleElementMatcher_Filter() {
+	obj, err := Parse(`
+- a
+- b
+`)
+	if err != nil {
+		log.Fatal(err)
+	}
+	elem, err := obj.Pipe(ElementMatcher{
+		FieldValue: "c", Create: NewScalarRNode("c"),
+	})
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	fmt.Println(elem.String())
+	fmt.Println(obj.String())
+	// Output:
+	// c
+	//  <nil>
+	// - a
+	// - b
+	// - c
+	//  <nil>
+}
+
+func ExampleElementMatcher_Filter_primitiveFound() {
+	obj, err := Parse(`
+- a
+- b
+- c
+`)
+	if err != nil {
+		log.Fatal(err)
+	}
+	elem, err := obj.Pipe(ElementMatcher{
+		FieldValue: "c", Create: NewScalarRNode("c"),
+	})
+	if err != nil {
+		log.Fatal(err)
+	}
+	fmt.Println(elem.String())
+	fmt.Println(obj.String())
+	// Output:
+	// c
+	//  <nil>
+	// - a
+	// - b
+	// - c
+	//  <nil>
+}
+
+func ExampleElementMatcher_Filter_objectNotFound() {
+	obj, err := Parse(`
+- name: foo
+- name: bar
+`)
+	if err != nil {
+		log.Fatal(err)
+	}
+	append, err := Parse(`
+name: baz
+image: nginx
+`)
+	if err != nil {
+		log.Fatal(err)
+	}
+	elem, err := obj.Pipe(ElementMatcher{
+		FieldName: "name", FieldValue: "baz", Create: append})
+	if err != nil {
+		log.Fatal(err)
+	}
+	fmt.Println(elem.String())
+	fmt.Println(obj.String())
+	// Output:
+	// name: baz
+	// image: nginx
+	//  <nil>
+	// - name: foo
+	// - name: bar
+	// - name: baz
+	//   image: nginx
+	//  <nil>
+}
+
+func ExampleElementMatcher_Filter_objectFound() {
+	obj, err := Parse(`
+- name: foo
+- name: bar
+- name: baz
+`)
+	if err != nil {
+		log.Fatal(err)
+	}
+	append, err := Parse(`
+name: baz
+image: nginx
+`)
+	if err != nil {
+		log.Fatal(err)
+	}
+	elem, err := obj.Pipe(ElementMatcher{
+		FieldName: "name", FieldValue: "baz", Create: append})
+	if err != nil {
+		log.Fatal(err)
+	}
+	fmt.Println(elem.String())
+	fmt.Println(obj.String())
+	// Output:
+	// name: baz
+	//  <nil>
+	// - name: foo
+	// - name: bar
+	// - name: baz
+	//  <nil>
+}
+
+func ExampleFieldMatcher_Filter() {
+	obj, err := Parse(`
+kind: Deployment
+spec:
+  template: {}
+`)
+	if err != nil {
+		log.Fatal(err)
+	}
+	value, err := Parse(`
+name: app
+annotations:
+  a.b.c: d.e.f
+  g: h
+`)
+	if err != nil {
+		log.Fatal(err)
+	}
+	elem, err := obj.Pipe(FieldMatcher{
+		Name: "metadata", Value: value, Create: value})
+	if err != nil {
+		log.Fatal(err)
+	}
+	fmt.Println(elem.String())
+	fmt.Println(obj.String())
+	// Output:
+	// name: app
+	// annotations:
+	//   a.b.c: d.e.f
+	//   g: h
+	//  <nil>
+	// kind: Deployment
+	// spec:
+	//   template: {}
+	// metadata:
+	//   name: app
+	//   annotations:
+	//     a.b.c: d.e.f
+	//     g: h
+	//  <nil>
+}
+
+func ExampleLookup_element() {
+	obj, err := Parse(`apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: java
+  annotations:
+    a.b.c: d.e.f
+  name: app
+spec:
+  templates:
+    spec:
+      containers:
+      - name: nginx
+        image: nginx:latest
+`)
+	if err != nil {
+		log.Fatal(err)
+	}
+	value, err := obj.Pipe(Lookup(
+		"spec", "templates", "spec", "containers", "[name=nginx]"))
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	fmt.Println(value.String())
+	// Output:
+	// name: nginx
+	// image: nginx:latest
+	//  <nil>
+}
+
+func ExampleLookup_sequence() {
+	obj, err := Parse(`apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+     app: java
+  annotations:
+    a.b.c: d.e.f
+  name: app
+spec:
+  templates:
+    spec:
+      containers:
+      - name: nginx
+        image: nginx:latest
+`)
+	if err != nil {
+		log.Fatal(err)
+	}
+	value, err := obj.Pipe(Lookup(
+		"spec", "templates", "spec", "containers"))
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	fmt.Println(value.String())
+	// Output:
+	// - name: nginx
+	//   image: nginx:latest
+	//  <nil>
+}
+
+func ExampleLookup_scalar() {
+	obj, err := Parse(`apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: java
+  annotations:
+    a.b.c: d.e.f
+  name: app
+spec:
+  templates:
+    spec:
+      containers:
+      - name: nginx
+        image: nginx:latest
+`)
+	if err != nil {
+		log.Fatal(err)
+	}
+	value, err := obj.Pipe(Lookup(
+		"spec", "templates", "spec", "containers", "[name=nginx]", "image"))
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	fmt.Println(value.String())
+	// Output:
+	// nginx:latest
+	//  <nil>
+}
+
+func ExampleLookupCreate_element() {
+	obj, err := Parse(`
+kind: Deployment
+metadata:
+  labels:
+    app: java
+  annotations:
+    a.b.c: d.e.f
+  name: app
+`)
+	if err != nil {
+		log.Fatal(err)
+	}
+	rs, err := obj.Pipe(LookupCreate(
+		MappingNode, "spec", "templates", "spec", "containers", "[name=nginx]"))
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	fmt.Println(rs.String())
+	fmt.Println("---")
+	fmt.Println(obj.String())
+	// Output:
+	// name: nginx
+	//  <nil>
+	// ---
+	// kind: Deployment
+	// metadata:
+	//   labels:
+	//     app: java
+	//   annotations:
+	//     a.b.c: d.e.f
+	//   name: app
+	// spec:
+	//   templates:
+	//     spec:
+	//       containers:
+	//       - name: nginx
+	//  <nil>
+}
+
+func ExampleLookupCreate_sequence() {
+	obj, err := Parse(`
+kind: Deployment
+metadata:
+  labels:
+    app: java
+  annotations:
+    a.b.c: d.e.f
+  name: app
+`)
+	if err != nil {
+		log.Fatal(err)
+	}
+	rs, err := obj.Pipe(LookupCreate(
+		SequenceNode, "spec", "templates", "spec", "containers"))
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	fmt.Println(rs.String())
+	fmt.Println("---")
+	fmt.Println(obj.String())
+	// Output:
+	// []
+	//  <nil>
+	// ---
+	// kind: Deployment
+	// metadata:
+	//   labels:
+	//     app: java
+	//   annotations:
+	//     a.b.c: d.e.f
+	//   name: app
+	// spec:
+	//   templates:
+	//     spec:
+	//       containers: []
+	//  <nil>
+}
+
+func ExamplePathGetter_Filter() {
+	obj, err := Parse(`
+kind: Deployment
+metadata:
+  labels:
+    app: java
+  annotations:
+    a.b.c: d.e.f
+  name: app
+`)
+	if err != nil {
+		log.Fatal(err)
+	}
+	rs, err := obj.Pipe(PathGetter{
+		Path:   []string{"spec", "templates", "spec", "containers", "[name=nginx]", "image"},
+		Create: ScalarNode,
+	})
+	if err != nil {
+		log.Fatal(err)
+	}
+	rs.Document().Style = SingleQuotedStyle
+
+	fmt.Println(rs.String())
+	fmt.Println("---")
+	fmt.Println(obj.String())
+	// Output:
+	// ''
+	//  <nil>
+	// ---
+	// kind: Deployment
+	// metadata:
+	//   labels:
+	//     app: java
+	//   annotations:
+	//     a.b.c: d.e.f
+	//   name: app
+	// spec:
+	//   templates:
+	//     spec:
+	//       containers:
+	//       - name: nginx
+	//         image: ''
+	//  <nil>
+}
+
+func ExampleLookupCreate_object() {
+	obj, err := Parse(`
+kind: Deployment
+metadata:
+  labels:
+    app: java
+  annotations:
+    a.b.c: d.e.f
+  name: app
+`)
+	if err != nil {
+		log.Fatal(err)
+	}
+	rs, err := obj.Pipe(LookupCreate(
+		MappingNode, "spec", "templates", "spec"))
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	fmt.Println(rs.String())
+	fmt.Println("---")
+	fmt.Println(obj.String())
+	// Output:
+	// {}
+	//  <nil>
+	// ---
+	// kind: Deployment
+	// metadata:
+	//   labels:
+	//     app: java
+	//   annotations:
+	//     a.b.c: d.e.f
+	//   name: app
+	// spec:
+	//   templates:
+	//     spec: {}
+	//  <nil>
+}
+
+func ExampleLookup_notFound() {
+	obj, err := Parse(`
+kind: Deployment
+metadata:
+  labels:
+    app: java
+  annotations:
+    a.b.c: d.e.f
+  name: app
+`)
+	if err != nil {
+		log.Fatal(err)
+	}
+	rs, err := obj.Pipe(Lookup("spec", "templates", "spec"))
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	fmt.Println(rs)
+	fmt.Println("---")
+	fmt.Println(obj.String())
+	// Output:
+	//  <nil>
+	// ---
+	// kind: Deployment
+	// metadata:
+	//   labels:
+	//     app: java
+	//   annotations:
+	//     a.b.c: d.e.f
+	//   name: app
+	//  <nil>
+}
+
+func ExampleSetField_stringValue() {
+	obj, err := Parse(`apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: app
+`)
+	if err != nil {
+		log.Fatal(err)
+	}
+	_, err = obj.Pipe(SetField("foo", NewScalarRNode("bar")))
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	fmt.Println(obj.String())
+	// Output:
+	// apiVersion: apps/v1
+	// kind: Deployment
+	// metadata:
+	//   name: app
+	// foo: bar
+	//  <nil>
+}
+
+func ExampleSetField_stringValueOverwrite() {
+	obj, err := Parse(`apiVersion: apps/v1
+kind: Deployment
+metadata:
+   name: app
+foo: baz
+`)
+	if err != nil {
+		// handle error
+	}
+	// set metadata.annotations.foo = bar
+	_, err = obj.Pipe(SetField("foo", NewScalarRNode("bar")))
+	if err != nil {
+		// handle error
+	}
+
+	fmt.Println(obj.String())
+	// Output:
+	// apiVersion: apps/v1
+	// kind: Deployment
+	// metadata:
+	//   name: app
+	// foo: bar
+	//  <nil>
+}
+
+func ExampleSetField() {
+	obj, err := Parse(`apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: app
+  labels:
+    app: java
+  annotations:
+    a.b.c: d.e.f
+`)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	containers, err := Parse(`
+- name: nginx # first container
+  image: nginx
+- name: nginx2 # second container
+  image: nginx2
+`)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	_, err = obj.Pipe(
+		LookupCreate(MappingNode, "spec", "template", "spec"),
+		SetField("containers", containers))
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	fmt.Println(obj.String())
+	// Output:
+	// apiVersion: apps/v1
+	// kind: Deployment
+	// metadata:
+	//   name: app
+	//   labels:
+	//     app: java
+	//   annotations:
+	//     a.b.c: d.e.f
+	// spec:
+	//   template:
+	//     spec:
+	//       containers:
+	//       - name: nginx # first container
+	//         image: nginx
+	//       - name: nginx2 # second container
+	//         image: nginx2
+	//  <nil>
+}
+
+func ExampleTee() {
+	obj, err := Parse(`apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: nginx-deployment
+  labels:
+    app: nginx
+spec:
+  replicas: 3
+  selector:
+    matchLabels:
+      app: nginx
+  template:
+    metadata:
+      labels:
+        app: nginx
+    spec:
+      containers:
+      - name: nginx
+        image: nginx:1.7.9
+        ports:
+        - containerPort: 80
+`)
+	if err != nil {
+		// handle error
+	}
+	// set metadata.annotations.foo = bar
+	_, err = obj.Pipe(
+		Lookup("spec", "template", "spec", "containers", "[name=nginx]"),
+		Tee(SetField("filter", NewListRNode("foo"))),
+		SetField("args", NewListRNode("baz", "bar")))
+	if err != nil {
+		// handle error
+	}
+
+	fmt.Println(obj.String())
+	// Output:
+	// apiVersion: apps/v1
+	// kind: Deployment
+	// metadata:
+	//   name: nginx-deployment
+	//   labels:
+	//     app: nginx
+	// spec:
+	//   replicas: 3
+	//   selector:
+	//     matchLabels:
+	//       app: nginx
+	//   template:
+	//     metadata:
+	//       labels:
+	//         app: nginx
+	//     spec:
+	//       containers:
+	//       - name: nginx
+	//         image: nginx:1.7.9
+	//         ports:
+	//         - containerPort: 80
+	//         filter:
+	//         - foo
+	//         args:
+	//         - baz
+	//         - bar
+	//  <nil>
+}

kyaml/yaml/filters.go

@@ -0,0 +1,140 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package yaml
+
+import (
+	"fmt"
+	"regexp"
+	"sort"
+	"strings"
+)
+
+var Filters = map[string]func() Filter{
+	"AnnotationClearer": func() Filter { return &AnnotationClearer{} },
+	"AnnotationGetter":  func() Filter { return &AnnotationGetter{} },
+	"AnnotationSetter":  func() Filter { return &AnnotationSetter{} },
+	"ElementAppender":   func() Filter { return &ElementAppender{} },
+	"ElementMatcher":    func() Filter { return &ElementMatcher{} },
+	"FieldClearer":      func() Filter { return &FieldClearer{} },
+	"FilterMatcher":     func() Filter { return &FilterMatcher{} },
+	"FieldMatcher":      func() Filter { return &FieldMatcher{} },
+	"FieldSetter":       func() Filter { return &FieldSetter{} },
+	"PathGetter":        func() Filter { return &PathGetter{} },
+	"PathMatcher":       func() Filter { return &PathMatcher{} },
+	"Parser":            func() Filter { return &Parser{} },
+	"PrefixSetter":      func() Filter { return &PrefixSetter{} },
+	"ValueReplacer":     func() Filter { return &ValueReplacer{} },
+	"SuffixSetter":      func() Filter { return &SuffixSetter{} },
+	"TeePiper":          func() Filter { return &TeePiper{} },
+}
+
+// YFilter wraps the GrepFilter interface so it can be unmarshalled into a struct.
+type YFilter struct {
+	Filter
+}
+
+func (y YFilter) MarshalYAML() (interface{}, error) {
+	return y.Filter, nil
+}
+
+func (y *YFilter) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	meta := &ResourceMeta{}
+	if err := unmarshal(meta); err != nil {
+		return err
+	}
+	if filter, found := Filters[meta.Kind]; !found {
+		var knownFilters []string
+		for k := range Filters {
+			knownFilters = append(knownFilters, k)
+		}
+		sort.Strings(knownFilters)
+		return fmt.Errorf("unsupported GrepFilter Kind %s:  may be one of: [%s]",
+			meta.Kind, strings.Join(knownFilters, ","))
+	} else {
+		y.Filter = filter()
+	}
+	if err := unmarshal(y.Filter); err != nil {
+		return err
+	}
+	return nil
+}
+
+type YFilters []YFilter
+
+func (y YFilters) Filters() []Filter {
+	var f []Filter
+	for i := range y {
+		f = append(f, y[i].Filter)
+	}
+	return f
+}
+
+type FilterMatcher struct {
+	Kind string `yaml:"kind"`
+
+	// Filters are the set of Filters run by TeePiper.
+	Filters YFilters `yaml:"pipeline,omitempty"`
+}
+
+func (t FilterMatcher) Filter(rn *RNode) (*RNode, error) {
+	v, err := rn.Pipe(t.Filters.Filters()...)
+	if v == nil || err != nil {
+		return nil, err
+	}
+	// return the original input if the pipeline resolves to true
+	return rn, err
+}
+
+type ValueReplacer struct {
+	Kind string `yaml:"kind"`
+
+	StringMatch string `yaml:"stringMatch"`
+	RegexMatch  string `yaml:"regexMatch"`
+	Replace     string `yaml:"replace"`
+	Count       int    `yaml:"count"`
+}
+
+func (s ValueReplacer) Filter(object *RNode) (*RNode, error) {
+	if s.Count == 0 {
+		s.Count = -1
+	}
+	if s.StringMatch != "" {
+		object.value.Value = strings.Replace(object.value.Value, s.StringMatch, s.Replace, s.Count)
+	} else if s.RegexMatch != "" {
+		r, err := regexp.Compile(s.RegexMatch)
+		if err != nil {
+			return nil, fmt.Errorf("ValueReplacer RegexMatch does not compile: %v", err)
+		}
+		object.value.Value = r.ReplaceAllString(object.value.Value, s.Replace)
+	} else {
+		return nil, fmt.Errorf("ValueReplacer missing StringMatch and RegexMatch")
+	}
+	return object, nil
+}
+
+type PrefixSetter struct {
+	Kind string `yaml:"kind"`
+
+	Value string `yaml:"value"`
+}
+
+func (s PrefixSetter) Filter(object *RNode) (*RNode, error) {
+	if !strings.HasPrefix(object.value.Value, s.Value) {
+		object.value.Value = s.Value + object.value.Value
+	}
+	return object, nil
+}
+
+type SuffixSetter struct {
+	Kind string `yaml:"kind"`
+
+	Value string `yaml:"value"`
+}
+
+func (s SuffixSetter) Filter(object *RNode) (*RNode, error) {
+	if !strings.HasSuffix(object.value.Value, s.Value) {
+		object.value.Value = object.value.Value + s.Value
+	}
+	return object, nil
+}

kyaml/yaml/fns.go

@@ -0,0 +1,585 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package yaml
+
+import (
+	"fmt"
+	"regexp"
+	"strings"
+
+	"github.com/davecgh/go-spew/spew"
+	"github.com/go-errors/errors"
+	"gopkg.in/yaml.v3"
+)
+
+// Append creates an ElementAppender
+func Append(elements ...*yaml.Node) ElementAppender {
+	return ElementAppender{Elements: elements}
+}
+
+// ElementAppender adds all element to a SequenceNode's Content.
+// Returns Elements[0] if len(Elements) == 1, otherwise returns nil.
+type ElementAppender struct {
+	Kind string `yaml:"kind,omitempty"`
+
+	// Elem is the value to append.
+	Elements []*yaml.Node `yaml:"elements,omitempty"`
+}
+
+func (a ElementAppender) Filter(rn *RNode) (*RNode, error) {
+	if err := ErrorIfInvalid(rn, yaml.SequenceNode); err != nil {
+		return nil, err
+	}
+	for i := range a.Elements {
+		rn.YNode().Content = append(rn.Content(), a.Elements[i])
+	}
+	if len(a.Elements) == 1 {
+		return NewRNode(a.Elements[0]), nil
+	}
+	return nil, nil
+}
+
+// ElementSetter sets the value for an Element in an associative list.  ElementSetter
+// will remove any elements which are empty.
+type ElementSetter struct {
+	Kind string `yaml:"kind,omitempty"`
+
+	// Element is the new value to set -- remove the existing element if nil
+	Element *Node
+
+	// Key is a field on the elements.  It is used to find the matching element to
+	// update / delete.
+	Key string `yaml:"key,omitempty"`
+
+	// Value is a field value on the elements.  It is used to find matching elements to
+	// update / delete.
+	Value string `yaml:"value,omitempty"`
+}
+
+func (e ElementSetter) Filter(rn *RNode) (*RNode, error) {
+	if err := ErrorIfInvalid(rn, SequenceNode); err != nil {
+		return nil, err
+	}
+
+	// build the new Content slice
+	var newContent []*yaml.Node
+	matchingElementFound := false
+	for i := range rn.YNode().Content {
+		elem := rn.Content()[i]
+
+		// empty elements are not valid -- they at least need an associative key
+		if IsEmpty(NewRNode(elem)) {
+			continue
+		}
+
+		// check if this is the element we are matching
+		val, err := NewRNode(elem).Pipe(FieldMatcher{Name: e.Key, StringValue: e.Value})
+		if err != nil {
+			return nil, err
+		}
+		if val == nil {
+			// not the element we are looking for, keep it in the Content
+			newContent = append(newContent, elem)
+			continue
+		}
+		matchingElementFound = true
+
+		// deletion operation -- remove the element from the new Content
+		if e.Element == nil {
+			continue
+		}
+		// replace operation -- replace the element in the Content
+		newContent = append(newContent, e.Element)
+	}
+	rn.YNode().Content = newContent
+
+	// deletion operation -- return nil
+	if IsMissingOrNull(NewRNode(e.Element)) {
+		return nil, nil
+	}
+
+	// append operation -- add the element to the Content
+	if !matchingElementFound {
+		rn.YNode().Content = append(rn.YNode().Content, e.Element)
+	}
+
+	return NewRNode(e.Element), nil
+}
+
+// Clear returns a FieldClearer
+func Clear(name string) FieldClearer {
+	return FieldClearer{Name: name}
+}
+
+// FieldClearer removes the field or map key.
+// Returns a RNode with the removed field or map entry.
+type FieldClearer struct {
+	Kind string `yaml:"kind,omitempty"`
+
+	// Name is the name of the field or key in the map.
+	Name string `yaml:"name,omitempty"`
+
+	IfEmpty bool `yaml:"ifEmpty,omitempty"`
+}
+
+func (c FieldClearer) Filter(rn *RNode) (*RNode, error) {
+	if err := ErrorIfInvalid(rn, yaml.MappingNode); err != nil {
+		return nil, err
+	}
+
+	for i := 0; i < len(rn.Content()); i += 2 {
+
+		// if name matches, remove these 2 elements from the list because
+		// they are treated as a fieldName/fieldValue pair.
+		if rn.Content()[i].Value == c.Name {
+			if c.IfEmpty {
+				if len(rn.Content()[i+1].Content) > 0 {
+					continue
+				}
+			}
+
+			// save the item we are about to remove
+			removed := NewRNode(rn.Content()[i+1])
+			if len(rn.YNode().Content) > i+2 {
+				// remove from the middle of the list
+				rn.YNode().Content = append(
+					rn.Content()[:i],
+					rn.Content()[i+2:len(rn.YNode().Content)]...)
+			} else {
+				// remove from the end of the list
+				rn.YNode().Content = rn.Content()[:i]
+			}
+
+			// return the removed field name and value
+			return removed, nil
+		}
+	}
+	// nothing removed
+	return nil, nil
+}
+
+func MatchElement(field, value string) ElementMatcher {
+	return ElementMatcher{FieldName: field, FieldValue: value}
+}
+
+// ElementMatcher returns the first element from a Sequence matching the
+// specified field's value.
+type ElementMatcher struct {
+	Kind string `yaml:"kind,omitempty"`
+
+	// FieldName will attempt to match this field in each list element.
+	// Optional.  Leave empty for lists of primitives (ScalarNode).
+	FieldName string `yaml:"name,omitempty"`
+
+	// FieldValue will attempt to match each element field to this value.
+	// For lists of primitives, this will be used to match the primitive value.
+	FieldValue string `yaml:"value,omitempty"`
+
+	// Create will create the Element if it is not found
+	Create *RNode `yaml:"create,omitempty"`
+}
+
+func (e ElementMatcher) Filter(rn *RNode) (*RNode, error) {
+	if err := ErrorIfInvalid(rn, yaml.SequenceNode); err != nil {
+		return nil, err
+	}
+
+	// SequenceNode Content is a slice of ScalarNodes.  Each ScalarNode has a
+	// YNode containing the primitive data.
+	if len(e.FieldName) == 0 {
+		for i := range rn.Content() {
+			if rn.Content()[i].Value == e.FieldValue {
+				return &RNode{value: rn.Content()[i]}, nil
+			}
+		}
+		if e.Create != nil {
+			return rn.Pipe(Append(e.Create.YNode()))
+		}
+		return nil, nil
+	}
+
+	// SequenceNode Content is a slice of MappingNodes.  Each MappingNode has Content
+	// with a slice of key-value pairs containing the fields.
+	for i := range rn.Content() {
+		// cast the entry to a RNode so we can operate on it
+		elem := NewRNode(rn.Content()[i])
+
+		field, err := elem.Pipe(MatchField(e.FieldName, e.FieldValue))
+		if IsFoundOrError(field, err) {
+			return elem, err
+		}
+	}
+
+	// create the element
+	if e.Create != nil {
+		return rn.Pipe(Append(e.Create.YNode()))
+	}
+
+	return nil, nil
+}
+
+func Get(name string) FieldMatcher {
+	return FieldMatcher{Name: name}
+}
+
+func MatchField(name, value string) FieldMatcher {
+	return FieldMatcher{Name: name, Value: NewScalarRNode(value)}
+}
+
+func Match(value string) FieldMatcher {
+	return FieldMatcher{Value: NewScalarRNode(value)}
+}
+
+// FieldMatcher returns the value of a named field or map entry.
+type FieldMatcher struct {
+	Kind string `yaml:"kind,omitempty"`
+
+	// Name of the field to return
+	Name string `yaml:"name,omitempty"`
+
+	// YNode of the field to return.
+	// Optional.  Will only need to match field name if unset.
+	Value *RNode `yaml:"value,omitempty"`
+
+	StringValue string `yaml:"stringValue,omitempty"`
+
+	StringRegexValue string `yaml:"stringRegexValue,omitempty"`
+
+	// Create will cause the field to be created with this value
+	// if it is set.
+	Create *RNode `yaml:"create,omitempty"`
+}
+
+func (f FieldMatcher) Filter(rn *RNode) (*RNode, error) {
+	if f.StringValue != "" && f.Value == nil {
+		f.Value = NewScalarRNode(f.StringValue)
+	}
+
+	// never match nil or null fields
+	if IsMissingOrNull(rn) {
+		return nil, nil
+	}
+
+	if f.Name == "" {
+		if err := ErrorIfInvalid(rn, yaml.ScalarNode); err != nil {
+			return nil, err
+		}
+		if f.StringRegexValue != "" {
+			// TODO(pwittrock): pre-compile this when unmarshalling and cache to a field
+			rg, err := regexp.Compile(f.StringRegexValue)
+			if err != nil {
+				return nil, err
+			}
+			if match := rg.MatchString(rn.value.Value); match {
+				return rn, nil
+			}
+			return nil, nil
+		} else if rn.value.Value == f.Value.YNode().Value {
+			return rn, nil
+		} else {
+			return nil, nil
+		}
+	}
+
+	if err := ErrorIfInvalid(rn, yaml.MappingNode); err != nil {
+		return nil, err
+	}
+
+	for i := 0; i < len(rn.Content()); IncrementFieldIndex(&i) {
+		isMatchingField := rn.Content()[i].Value == f.Name
+		if isMatchingField {
+			requireMatchFieldValue := f.Value != nil
+			if !requireMatchFieldValue || rn.Content()[i+1].Value == f.Value.YNode().Value {
+				return NewRNode(rn.Content()[i+1]), nil
+			}
+		}
+	}
+
+	if f.Create != nil {
+		return rn.Pipe(SetField(f.Name, f.Create))
+	}
+
+	return nil, nil
+}
+
+func Lookup(path ...string) PathGetter {
+	return PathGetter{Path: path}
+}
+
+func LookupCreate(kind yaml.Kind, path ...string) PathGetter {
+	return PathGetter{Path: path, Create: kind}
+}
+
+// PathGetter returns the RNode under Path.
+type PathGetter struct {
+	Kind string `yaml:"kind,omitempty"`
+
+	// Path is a slice of parts leading to the RNode to lookup.
+	// Each path part may be one of:
+	// * FieldMatcher -- e.g. "spec"
+	// * Map Key -- e.g. "app.k8s.io/version"
+	// * List Entry -- e.g. "[name=nginx]" or "[=-jar]"
+	//
+	// Map Keys and Fields are equivalent.
+	// See FieldMatcher for more on Fields and Map Keys.
+	//
+	// List Entries are specified as map entry to match [fieldName=fieldValue].
+	// See Elem for more on List Entries.
+	//
+	// Examples:
+	// * spec.template.spec.container with matching name: [name=nginx]
+	// * spec.template.spec.container.argument matching a value: [=-jar]
+	Path []string `yaml:"path,omitempty"`
+
+	// Create will cause missing path parts to be created as they are walked.
+	//
+	// * The leaf Node (final path) will be created with a Kind matching Create
+	// * Intermediary Nodes will be created as either a MappingNodes or
+	//   SequenceNodes as appropriate for each's Path location.
+	Create yaml.Kind `yaml:"create,omitempty"`
+
+	// Style is the style to apply to created value Nodes.
+	// Created key Nodes keep an unspecified Style.
+	Style yaml.Style `yaml:"style,omitempty"`
+}
+
+func (l PathGetter) Filter(rn *RNode) (*RNode, error) {
+	var err error
+	fieldPath := append([]string{}, rn.FieldPath()...)
+	match := rn
+
+	// iterate over path until encountering an error or missing value
+	l.Path = cleanPath(l.Path)
+	for i := range l.Path {
+		var part, nextPart string
+		part = l.Path[i]
+		if len(l.Path) > i+1 {
+			nextPart = l.Path[i+1]
+		}
+		if IsListIndex(part) {
+			match, err = l.doElem(match, part)
+		} else {
+			fieldPath = append(fieldPath, part)
+			match, err = l.doField(match, part, l.getKind(nextPart))
+		}
+		if IsMissingOrError(match, err) {
+			return nil, err
+		}
+		match.AppendToFieldPath(fieldPath...)
+	}
+	return match, nil
+}
+
+func (l PathGetter) doElem(rn *RNode, part string) (*RNode, error) {
+	var match *RNode
+	name, value, err := SplitIndexNameValue(part)
+	if err != nil {
+		return nil, err
+	}
+	if !IsCreate(l.Create) {
+		return rn.Pipe(MatchElement(name, value))
+	}
+
+	var elem *RNode
+	primitiveElement := len(name) == 0
+	if primitiveElement {
+		// append a ScalarNode
+		elem = NewScalarRNode(value)
+		elem.YNode().Style = l.Style
+		match = elem
+	} else {
+		// append a MappingNode
+		match = NewRNode(&yaml.Node{Kind: yaml.ScalarNode, Value: value, Style: l.Style})
+		elem = NewRNode(&yaml.Node{
+			Kind:    yaml.MappingNode,
+			Content: []*yaml.Node{{Kind: yaml.ScalarNode, Value: name}, match.YNode()},
+			Style:   l.Style,
+		})
+	}
+	// Append the Node
+	return rn.Pipe(ElementMatcher{FieldName: name, FieldValue: value, Create: elem})
+}
+
+func (l PathGetter) doField(
+	rn *RNode, name string, kind yaml.Kind) (*RNode, error) {
+	if !IsCreate(l.Create) {
+		return rn.Pipe(Get(name))
+	}
+	return rn.Pipe(FieldMatcher{Name: name, Create: &RNode{value: &yaml.Node{Kind: kind, Style: l.Style}}})
+}
+
+func (l PathGetter) getKind(nextPart string) yaml.Kind {
+	if IsListIndex(nextPart) {
+		// if nextPart is of the form [a=b], then it is an index into a Sequence
+		// so the current part must be a SequenceNode
+		return yaml.SequenceNode
+	}
+	if nextPart == "" {
+		// final name in the path, use the l.Create defined Kind
+		return l.Create
+	}
+
+	// non-sequence intermediate Node
+	return yaml.MappingNode
+}
+
+func SetField(name string, value *RNode) FieldSetter {
+	return FieldSetter{Name: name, Value: value}
+}
+
+func Set(value *RNode) FieldSetter {
+	return FieldSetter{Value: value}
+}
+
+// FieldSetter sets a field or map entry to a value.
+type FieldSetter struct {
+	Kind string `yaml:"kind,omitempty"`
+
+	// Name is the name of the field or key to lookup in a MappingNode.
+	// If Name is unspecified, and the input is a ScalarNode, FieldSetter will set the
+	// value on the ScalarNode.
+	Name string `yaml:"name,omitempty"`
+
+	// YNode is the value to set.
+	// Optional if Kind is set.
+	Value *RNode `yaml:"value,omitempty"`
+
+	StringValue string `yaml:"stringValue,omitempty"`
+}
+
+// FieldSetter returns an GrepFilter that sets the named field to the given value.
+func (s FieldSetter) Filter(rn *RNode) (*RNode, error) {
+	if s.StringValue != "" && s.Value == nil {
+		s.Value = NewScalarRNode(s.StringValue)
+	}
+
+	if s.Name == "" {
+		if err := ErrorIfInvalid(rn, yaml.ScalarNode); err != nil {
+			return rn, err
+		}
+		rn.SetYNode(s.Value.YNode())
+		return rn, nil
+	}
+
+	// Clear the field if it is empty, or explicitly null
+	if s.Value == nil || IsNull(s.Value) {
+		return rn.Pipe(Clear(s.Name))
+	}
+
+	field, err := rn.Pipe(FieldMatcher{Name: s.Name})
+	if err != nil {
+		return nil, err
+	}
+	if field != nil {
+		// need to def ref the Node since field is ephemeral
+		field.SetYNode(s.Value.YNode())
+		return field, nil
+	}
+
+	// create the field
+	rn.YNode().Content = append(rn.YNode().Content,
+		&yaml.Node{Kind: yaml.ScalarNode, Value: s.Name},
+		s.Value.YNode())
+	return s.Value, nil
+}
+
+// Tee calls the provided Filters, and returns its argument rather than the result
+// of the filters.
+// May be used to fork sub-filters from a call.
+// e.g. locate field, set value; locate another field, set another value
+func Tee(filters ...Filter) Filter {
+	return TeePiper{Filters: filters}
+}
+
+// TeePiper Calls a slice of Filters and returns its input.
+// May be used to fork sub-filters from a call.
+// e.g. locate field, set value; locate another field, set another value
+type TeePiper struct {
+	Kind string `yaml:"kind,omitempty"`
+
+	// Filters are the set of Filters run by TeePiper.
+	Filters []Filter `yaml:"filters,omitempty"`
+}
+
+func (t TeePiper) Filter(rn *RNode) (*RNode, error) {
+	_, err := rn.Pipe(t.Filters...)
+	return rn, err
+}
+
+// IsCreate returns true if kind is specified
+func IsCreate(kind yaml.Kind) bool {
+	return kind != 0
+}
+
+// IsMissingOrError returns true if rn is NOT found or err is non-nil
+func IsMissingOrError(rn *RNode, err error) bool {
+	return rn == nil || err != nil
+}
+
+// IsFoundOrError returns true if rn is found or err is non-nil
+func IsFoundOrError(rn *RNode, err error) bool {
+	return rn != nil || err != nil
+}
+
+func ErrorIfAnyInvalidAndNonNull(kind yaml.Kind, rn ...*RNode) error {
+	for i := range rn {
+		if IsEmpty(rn[i]) {
+			continue
+		}
+		if err := ErrorIfInvalid(rn[i], kind); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func ErrorIfInvalid(rn *RNode, kind yaml.Kind) error {
+	if rn == nil || rn.YNode() == nil || IsNull(rn) {
+		// node has no type, pass validation
+		return nil
+	}
+
+	if rn.YNode().Kind != kind {
+		s, _ := rn.String()
+		return errors.Errorf(
+			"wrong Node Kind for %s expected: %v was %v: value: {%s}",
+			strings.Join(rn.FieldPath(), "."),
+			kind, rn.YNode().Kind, strings.TrimSpace(s))
+	}
+
+	if kind == yaml.MappingNode {
+		if len(rn.YNode().Content)%2 != 0 {
+			return fmt.Errorf("yaml MappingNodes must have even length contents: %v", spew.Sdump(rn))
+		}
+	}
+
+	return nil
+}
+
+// IsListIndex returns true if p is an index into a Val.
+// e.g. [fieldName=fieldValue]
+// e.g. [=primitiveValue]
+func IsListIndex(p string) bool {
+	return strings.HasPrefix(p, "[") && strings.HasSuffix(p, "]")
+}
+
+// SplitIndexNameValue splits a lookup part Val index into the field name
+// and field value to match.
+// e.g. splits [name=nginx] into (name, nginx)
+// e.g. splits [=-jar] into ("", jar)
+func SplitIndexNameValue(p string) (string, string, error) {
+	elem := strings.TrimSuffix(p, "]")
+	elem = strings.TrimPrefix(elem, "[")
+	parts := strings.SplitN(elem, "=", 2)
+	if len(parts) == 1 {
+		return "", "", fmt.Errorf("list path element must contain fieldName=fieldValue for element to match")
+	}
+	return parts[0], parts[1], nil
+}
+
+// IncrementFieldIndex increments i to point to the next field name element in
+// a slice of Contents.
+func IncrementFieldIndex(i *int) {
+	*i = *i + 2
+}

kyaml/yaml/fns_test.go

@@ -0,0 +1,704 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package yaml_test
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"gopkg.in/yaml.v3"
+	. "sigs.k8s.io/kustomize/kyaml/yaml"
+)
+
+func TestResourceNode_SetValue(t *testing.T) {
+	instance := *NewScalarRNode("foo")
+	copy := instance
+	instance.SetYNode(&yaml.Node{Kind: yaml.ScalarNode, Value: "bar"})
+	assert.Equal(t, `bar
+`, assertNoErrorString(t)(copy.String()))
+	assert.Equal(t, `bar
+`, assertNoErrorString(t)(instance.String()))
+
+	instance = *NewScalarRNode("foo")
+	copy = instance
+	instance.SetYNode(nil)
+	instance.SetYNode(&yaml.Node{Kind: yaml.ScalarNode, Value: "bar"})
+	assert.Equal(t, `foo
+`, assertNoErrorString(t)(copy.String()))
+	assert.Equal(t, `bar
+`, assertNoErrorString(t)(instance.String()))
+}
+
+func TestAppend(t *testing.T) {
+	s := `n: o
+a: b
+c: d
+`
+	node, err := Parse(s)
+	assert.NoError(t, err)
+	rn, err := node.Pipe(Append(NewScalarRNode("").YNode()))
+	if assert.Error(t, err) {
+		assert.Contains(t, err.Error(), "wrong Node Kind")
+	}
+	assert.Nil(t, rn)
+
+	s = `- a
+- b
+`
+	node, err = Parse(s)
+	assert.NoError(t, err)
+	rn, err = node.Pipe(Append())
+	assert.NoError(t, err)
+	assert.Nil(t, rn)
+}
+
+func TestClearField_Fn(t *testing.T) {
+	s := `n: o
+a: b
+c: d
+`
+	node, err := Parse(s)
+	assert.NoError(t, err)
+	rn, err := node.Pipe(FieldClearer{Name: "a"})
+	assert.NoError(t, err)
+	assert.Equal(t, "n: o\nc: d\n", assertNoErrorString(t)(node.String()))
+	assert.Equal(t, "b\n", assertNoErrorString(t)(rn.String()))
+
+	s = `n: o
+a: b
+c: d
+`
+	node, err = Parse(s)
+	assert.NoError(t, err)
+	rn, err = node.Pipe(FieldClearer{Name: "n"})
+	assert.NoError(t, err)
+	assert.Equal(t, "a: b\nc: d\n", assertNoErrorString(t)(node.String()))
+	assert.Equal(t, "o\n", assertNoErrorString(t)(rn.String()))
+
+	node, err = Parse(s)
+	assert.NoError(t, err)
+	rn, err = node.Pipe(FieldClearer{Name: "c"})
+	assert.NoError(t, err)
+	assert.Equal(t, "n: o\na: b\n", assertNoErrorString(t)(node.String()))
+	assert.Equal(t, "d\n", assertNoErrorString(t)(rn.String()))
+
+	s = `n: o
+a: b
+`
+	node, err = Parse(s)
+	assert.NoError(t, err)
+	rn, err = node.Pipe(FieldClearer{Name: "o"})
+	assert.NoError(t, err)
+	assert.Nil(t, rn)
+	assert.Equal(t, "n: o\na: b\n", assertNoErrorString(t)(node.String()))
+
+	s = `- a
+- b
+`
+	node, err = Parse(s)
+	assert.NoError(t, err)
+	rn, err = node.Pipe(FieldClearer{Name: "a"})
+	if assert.Error(t, err) {
+		assert.Contains(t, err.Error(), "wrong Node Kind")
+	}
+	assert.Nil(t, rn)
+	assert.Equal(t, s, assertNoErrorString(t)(node.String()))
+
+	// should not clear n because it is not empty
+	s = `n:
+  k: v
+a: b
+c: d
+`
+	node, err = Parse(s)
+	assert.NoError(t, err)
+	rn, err = node.Pipe(FieldClearer{Name: "n", IfEmpty: true})
+	assert.NoError(t, err)
+	assert.Equal(t, "n:\n  k: v\na: b\nc: d\n", assertNoErrorString(t)(node.String()))
+	assert.Equal(t, "", assertNoErrorString(t)(rn.String()))
+
+	// should clear n because it is empty
+	s = `n: {}
+a: b
+c: d
+`
+	node, err = Parse(s)
+	assert.NoError(t, err)
+	rn, err = node.Pipe(FieldClearer{Name: "n", IfEmpty: true})
+	assert.NoError(t, err)
+	assert.Equal(t, "a: b\nc: d\n", assertNoErrorString(t)(node.String()))
+	assert.Equal(t, "{}\n", assertNoErrorString(t)(rn.String()))
+}
+
+var s = `n: o
+a:
+  l: m
+  b:
+  - f: g
+  - c: e
+  - h: i
+r: s
+`
+
+func TestLookup_Fn_create(t *testing.T) {
+
+	// primitive
+	node, err := Parse(s)
+	assert.NoError(t, err)
+	rn, err := node.Pipe(PathGetter{
+		Path:   []string{"a", "b", "[c=d]", "t", "f", "[=h]"},
+		Create: yaml.ScalarNode,
+	})
+	assert.NoError(t, err)
+	assert.Equal(t, `n: o
+a:
+  l: m
+  b:
+  - f: g
+  - c: e
+  - h: i
+  - c: d
+    t:
+      f:
+      - h
+r: s
+`, assertNoErrorString(t)(node.String()))
+	assert.Equal(t, `h
+`, assertNoErrorString(t)(rn.String()))
+
+}
+
+func TestLookup_Fn_create2(t *testing.T) {
+	node, err := Parse(s)
+	assert.NoError(t, err)
+	rn, err := node.Pipe(PathGetter{
+		Path:   []string{"a", "b", "[c=d]", "t", "f"},
+		Create: yaml.SequenceNode,
+	})
+	assert.NoError(t, err)
+	assert.Equal(t, `n: o
+a:
+  l: m
+  b:
+  - f: g
+  - c: e
+  - h: i
+  - c: d
+    t:
+      f: []
+r: s
+`, assertNoErrorString(t)(node.String()))
+	assert.Equal(t, `[]
+`, assertNoErrorString(t)(rn.String()))
+}
+
+func TestLookup_Fn_create3(t *testing.T) {
+	node, err := Parse(s)
+	assert.NoError(t, err)
+	rn, err := node.Pipe(LookupCreate(yaml.MappingNode, "a", "b", "[c=d]", "t"))
+	assert.NoError(t, err)
+	assert.Equal(t, `n: o
+a:
+  l: m
+  b:
+  - f: g
+  - c: e
+  - h: i
+  - c: d
+    t: {}
+r: s
+`, assertNoErrorString(t)(node.String()))
+	assert.Equal(t, `{}
+`, assertNoErrorString(t)(rn.String()))
+}
+
+func TestLookupCreate_4(t *testing.T) {
+	node, err := Parse(`
+a: {}
+`)
+	assert.NoError(t, err)
+	rn, err := node.Pipe(
+		LookupCreate(yaml.MappingNode, "a", "b", "[c=d]", "t", "f", "[=h]"))
+
+	node.YNode().Style = yaml.FlowStyle
+	assert.NoError(t, err)
+	assert.Equal(t, "{a: {b: [{c: d, t: {f: [h]}}]}}\n", assertNoErrorString(t)(node.String()))
+	assert.Equal(t, "h\n", assertNoErrorString(t)(rn.String()))
+}
+
+func TestLookup(t *testing.T) {
+	s := `n: o
+a:
+  l: m
+  b:
+  - f: g
+  - c: e
+  - c: d
+    t:
+      u: v
+      f:
+      - g
+      - h
+      - i
+    j: k
+  - h: i
+    p: q
+r: s
+`
+	node, err := Parse(s)
+	assert.NoError(t, err)
+
+	// primitive
+	rn, err := node.Pipe(Lookup("a", "b", "[c=d]", "t", "f", "[=h]"))
+	assert.NoError(t, err)
+	assert.Equal(t, s, assertNoErrorString(t)(node.String()))
+	assert.Equal(t, `h
+`, assertNoErrorString(t)(rn.String()))
+
+	// seq
+	rn, err = node.Pipe(Lookup("a", "b", "[c=d]", "t", "f"))
+	assert.NoError(t, err)
+	assert.Equal(t, s, assertNoErrorString(t)(node.String()))
+	assert.Equal(t, `- g
+- h
+- i
+`, assertNoErrorString(t)(rn.String()))
+
+	rn, err = node.Pipe(Lookup("a", "b", "[c=d]", "t"))
+	assert.NoError(t, err)
+	assert.Equal(t, s, assertNoErrorString(t)(node.String()))
+	assert.Equal(t, `u: v
+f:
+- g
+- h
+- i
+`, assertNoErrorString(t)(rn.String()))
+
+	rn, err = node.Pipe(Lookup("a", "b", "[c=d]"))
+	assert.NoError(t, err)
+	assert.Equal(t, s, assertNoErrorString(t)(node.String()))
+	assert.Equal(t, `c: d
+t:
+  u: v
+  f:
+  - g
+  - h
+  - i
+j: k
+`, assertNoErrorString(t)(rn.String()))
+
+	rn, err = node.Pipe(Lookup("a", "b"))
+	assert.NoError(t, err)
+	assert.Equal(t, s, assertNoErrorString(t)(node.String()))
+	assert.Equal(t, `- f: g
+- c: e
+- c: d
+  t:
+    u: v
+    f:
+    - g
+    - h
+    - i
+  j: k
+- h: i
+  p: q
+`, assertNoErrorString(t)(rn.String()))
+
+	rn, err = node.Pipe(Lookup("l"))
+	assert.NoError(t, err)
+	assert.Equal(t, s, assertNoErrorString(t)(node.String()))
+	assert.Nil(t, rn)
+
+	rn, err = node.Pipe(Lookup("zzz"))
+	assert.NoError(t, err)
+	assert.Equal(t, s, assertNoErrorString(t)(node.String()))
+	assert.Nil(t, rn)
+
+	rn, err = node.Pipe(Lookup("[a=b]"))
+	assert.Error(t, err)
+	assert.Equal(t, s, assertNoErrorString(t)(node.String()))
+	assert.Nil(t, rn)
+
+	rn, err = node.Pipe(Lookup("a", "b", "f"))
+	assert.Error(t, err)
+	assert.Equal(t, s, assertNoErrorString(t)(node.String()))
+	assert.Nil(t, rn)
+
+	rn, err = node.Pipe(Lookup("a", "b", "c=zzz"))
+	assert.Error(t, err)
+	assert.Equal(t, s, assertNoErrorString(t)(node.String()))
+	assert.Nil(t, rn)
+
+	rn, err = node.Pipe(Lookup(" ", "a", "", "b", " ", "[c=d]", "\n", "t", "\t", "f", "  ", "[=h]", "  "))
+	assert.NoError(t, err)
+	assert.Equal(t, s, assertNoErrorString(t)(node.String()))
+	assert.Equal(t, `h
+`, assertNoErrorString(t)(rn.String()))
+
+	rn, err = node.Pipe(Lookup(" ", "a", "", "b", " ", "[]"))
+	assert.Error(t, err)
+	assert.Nil(t, rn)
+
+	rn, err = node.Pipe(Lookup("a", "b", "[c=d]", "t", "f", "[=c]"))
+	assert.NoError(t, err)
+	assert.Equal(t, s, assertNoErrorString(t)(node.String()))
+	assert.Nil(t, rn)
+
+	rn, err = node.Pipe(Lookup("a", "b", "[z=z]"))
+	assert.NoError(t, err)
+	assert.Equal(t, s, assertNoErrorString(t)(node.String()))
+	assert.Nil(t, rn)
+}
+
+func TestSetField_Fn(t *testing.T) {
+	// Change field
+	node, err := Parse(`
+foo: baz
+`)
+	assert.NoError(t, err)
+	instance := FieldSetter{
+		Name:  "foo",
+		Value: NewScalarRNode("bar"),
+	}
+	k, err := instance.Filter(node)
+	assert.NoError(t, err)
+	assert.Equal(t, `foo: bar
+`, assertNoErrorString(t)(node.String()))
+	assert.Equal(t, `bar
+`, assertNoErrorString(t)(k.String()))
+
+	// Add field
+	node, err = Parse(`
+foo: baz
+`)
+	assert.NoError(t, err)
+	instance = FieldSetter{
+		Name:  "bar",
+		Value: NewScalarRNode("buz"),
+	}
+	k, err = instance.Filter(node)
+	assert.NoError(t, err)
+	assert.Equal(t, `foo: baz
+bar: buz
+`, assertNoErrorString(t)(node.String()))
+	assert.Equal(t, `buz
+`, assertNoErrorString(t)(k.String()))
+
+	// Clear field
+	node, err = Parse(`
+foo: baz
+bar: buz
+`)
+	assert.NoError(t, err)
+	instance = FieldSetter{
+		Name: "foo",
+	}
+	k, err = instance.Filter(node)
+	assert.NoError(t, err)
+	assert.Equal(t, `bar: buz
+`, assertNoErrorString(t)(node.String()))
+	assert.Equal(t, `baz
+`, assertNoErrorString(t)(k.String()))
+
+	// Encounter error
+	node, err = Parse(`
+-a
+-b
+`)
+	assert.NoError(t, err)
+	instance = FieldSetter{
+		Name:  "foo",
+		Value: NewScalarRNode("v"),
+	}
+	k, err = instance.Filter(node)
+	if assert.Error(t, err) {
+		assert.Contains(t, err.Error(), "wrong Node Kind")
+	}
+	assert.Nil(t, k)
+}
+
+func TestSet_Fn(t *testing.T) {
+	node, err := Parse(`
+foo: baz
+`)
+	assert.NoError(t, err)
+	k, err := node.Pipe(Get("foo"), Set(NewScalarRNode("bar")))
+	assert.NoError(t, err)
+	assert.Equal(t, `foo: bar
+`, assertNoErrorString(t)(node.String()))
+	assert.Equal(t, `bar
+`, assertNoErrorString(t)(k.String()))
+
+	node, err = Parse(`
+foo: baz
+`)
+	assert.NoError(t, err)
+	_, err = node.Pipe(Set(NewScalarRNode("bar")))
+	if !assert.Error(t, err) {
+		return
+	}
+	assert.Contains(t, err.Error(), "wrong Node Kind")
+	assert.Equal(t, `foo: baz
+`, assertNoErrorString(t)(node.String()))
+
+}
+
+func TestErrorIfInvalid(t *testing.T) {
+	err := ErrorIfInvalid(
+		NewRNode(&yaml.Node{Kind: yaml.SequenceNode}), yaml.SequenceNode)
+	assert.NoError(t, err)
+
+	// nil values should pass validation -- they were not specified
+	err = ErrorIfInvalid(&RNode{}, yaml.SequenceNode)
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
+
+	err = ErrorIfInvalid(NewRNode(&Node{Content: []*yaml.Node{{Value: "hello"}}}), yaml.SequenceNode)
+	if !assert.Error(t, err) {
+		t.FailNow()
+	}
+	assert.Contains(t, err.Error(), "wrong Node Kind")
+
+	err = ErrorIfInvalid(NewRNode(&yaml.Node{}), yaml.SequenceNode)
+	if assert.Error(t, err) {
+		assert.Contains(t, err.Error(), "wrong Node Kind")
+	}
+	err = ErrorIfInvalid(NewRNode(&yaml.Node{}), yaml.MappingNode)
+	if assert.Error(t, err) {
+		assert.Contains(t, err.Error(), "wrong Node Kind")
+	}
+
+	err = ErrorIfInvalid(NewRNode(&yaml.Node{
+		Kind:    yaml.MappingNode,
+		Content: []*yaml.Node{{}, {}},
+	}), yaml.MappingNode)
+	assert.NoError(t, err)
+
+	err = ErrorIfInvalid(NewRNode(&yaml.Node{}), yaml.SequenceNode)
+	if assert.Error(t, err) {
+		assert.Contains(t, err.Error(), "wrong Node Kind")
+	}
+
+	err = ErrorIfInvalid(NewRNode(&yaml.Node{
+		Kind:    yaml.MappingNode,
+		Content: []*yaml.Node{{}},
+	}), yaml.MappingNode)
+	if assert.Error(t, err) {
+		assert.Contains(t, err.Error(), "even length")
+	}
+}
+
+func TestSplitIndexNameValue(t *testing.T) {
+	k, v, err := SplitIndexNameValue("")
+	if assert.Error(t, err) {
+		assert.Contains(t, err.Error(), "fieldName=fieldValue")
+	}
+	assert.Equal(t, "", k)
+	assert.Equal(t, "", v)
+
+	k, v, err = SplitIndexNameValue("a=b")
+	assert.NoError(t, err)
+	assert.Equal(t, "a", k)
+	assert.Equal(t, "b", v)
+
+	k, v, err = SplitIndexNameValue("=b")
+	assert.NoError(t, err)
+	assert.Equal(t, "", k)
+	assert.Equal(t, "b", v)
+
+	k, v, err = SplitIndexNameValue("a=b=c")
+	assert.NoError(t, err)
+	assert.Equal(t, "a", k)
+	assert.Equal(t, "b=c", v)
+}
+
+type filter struct {
+	fn func(object *RNode) (*RNode, error)
+}
+
+func (c filter) Filter(object *RNode) (*RNode, error) {
+	return c.fn(object)
+}
+
+func TestResourceNode_Pipe(t *testing.T) {
+	r0, r1, r2, r3 := &RNode{}, &RNode{}, &RNode{}, &RNode{}
+	var called []string
+
+	// check the nil value case
+	r0 = nil
+	_, err := r0.Pipe(FieldMatcher{Name: "foo"})
+	assert.NoError(t, err)
+
+	r0 = &RNode{}
+	// all filters successful
+	v, err := r0.Pipe(
+		filter{fn: func(object *RNode) (*RNode, error) {
+			assert.True(t, r0 == object)
+			called = append(called, "a")
+			return r1, nil
+		}},
+		filter{fn: func(object *RNode) (*RNode, error) {
+			assert.True(t, object == r1, "function arg doesn't match last function output")
+			called = append(called, "b")
+			return r2, nil
+		}},
+		filter{fn: func(object *RNode) (*RNode, error) {
+			assert.True(t, object == r2, "function arg doesn't match last function output")
+			return r3, nil
+		}},
+	)
+	assert.True(t, v == r3, "expected r3")
+	assert.Nil(t, err)
+	assert.Equal(t, called, []string{"a", "b"})
+
+	// filter returns nil
+	called = []string{}
+	v, err = r0.Pipe(
+		filter{fn: func(object *RNode) (*RNode, error) {
+			assert.True(t, r0 == object)
+			called = append(called, "a")
+			return r1, nil
+		}},
+		filter{fn: func(object *RNode) (*RNode, error) {
+			assert.True(t, object == r1, "function arg doesn't match last function output")
+			called = append(called, "b")
+			return nil, nil
+		}},
+		filter{fn: func(object *RNode) (*RNode, error) {
+			assert.Fail(t, "function should be run after error")
+			return nil, nil
+		}},
+	)
+	assert.Nil(t, v)
+	assert.Nil(t, err)
+	assert.Equal(t, called, []string{"a", "b"})
+
+	// filter returns an error
+	called = []string{}
+	v, err = r0.Pipe(
+		filter{fn: func(object *RNode) (*RNode, error) {
+			assert.True(t, r0 == object)
+			called = append(called, "a")
+			return r1, nil
+		}},
+		filter{fn: func(object *RNode) (*RNode, error) {
+			assert.True(t, object == r1, "function arg doesn't match last function output")
+			called = append(called, "b")
+			return r1, fmt.Errorf("expected-error")
+		}},
+		filter{fn: func(object *RNode) (*RNode, error) {
+			assert.Fail(t, "function should be run after error")
+			return nil, nil
+		}},
+	)
+	assert.True(t, v == r1, "expected r1 as value")
+	assert.EqualError(t, err, "expected-error")
+	assert.Equal(t, called, []string{"a", "b"})
+}
+
+func TestClearAnnotation(t *testing.T) {
+	// create metadata.annotations field
+	r0 := assertNoError(t)(Parse(`apiVersion: apps/v1
+kind: Deployment
+metadata:
+ annotations:
+   z: y
+   a.b.c: d.e.f
+   s: t
+`))
+
+	rn := assertNoError(t)(r0.Pipe(ClearAnnotation("a.b.c")))
+	assert.Equal(t, "d.e.f\n", assertNoErrorString(t)(rn.String()))
+	assert.Equal(t, `apiVersion: apps/v1
+kind: Deployment
+metadata:
+  annotations:
+    z: y
+    s: t
+`, assertNoErrorString(t)(r0.String()))
+}
+
+func TestGetAnnotation(t *testing.T) {
+	r0 := assertNoError(t)(Parse(`apiVersion: apps/v1
+kind: Deployment
+metadata:
+ labels:
+   app: java
+ annotations:
+   a.b.c: d.e.f
+   g: h
+   i: j
+   k: l
+ name: app`))
+
+	rn := assertNoError(t)(
+		r0.Pipe(GetAnnotation("a.b.c")))
+	assert.Equal(t, "d.e.f\n", assertNoErrorString(t)(rn.String()))
+}
+
+func TestSetAnnotation_Fn(t *testing.T) {
+	// create metadata.annotations field
+	r0 := assertNoError(t)(Parse(`apiVersion: apps/v1
+kind: Deployment`))
+
+	rn := assertNoError(t)(r0.Pipe(SetAnnotation("a.b.c", "d.e.f")))
+	assert.Equal(t, "d.e.f\n", assertNoErrorString(t)(rn.String()))
+	assert.Equal(t, `apiVersion: apps/v1
+kind: Deployment
+metadata:
+  annotations:
+    a.b.c: d.e.f
+`, assertNoErrorString(t)(r0.String()))
+}
+
+func TestRNode_GetMeta(t *testing.T) {
+	s := `apiVersion: v1/apps
+kind: Deployment
+metadata:
+  name: foo
+  namespace: bar
+  labels:
+    kl: vl
+  annotations:
+    ka: va
+`
+	node, err := Parse(s)
+	if !assert.NoError(t, err) {
+		return
+	}
+	meta, err := node.GetMeta()
+	if !assert.NoError(t, err) {
+		return
+	}
+	assert.Equal(t, ResourceMeta{
+		Kind:       "Deployment",
+		ApiVersion: "v1/apps",
+		ObjectMeta: ObjectMeta{
+			Name:      "foo",
+			Namespace: "bar",
+			Annotations: map[string]string{
+				"ka": "va",
+			},
+			Labels: map[string]string{
+				"kl": "vl",
+			},
+		},
+	}, meta)
+}
+
+func assertNoError(t *testing.T) func(o *RNode, err error) *RNode {
+	return func(o *RNode, err error) *RNode {
+		assert.NoError(t, err)
+		return o
+	}
+}
+
+func assertNoErrorString(t *testing.T) func(string, error) string {
+	return func(s string, err error) string {
+		assert.NoError(t, err)
+		return s
+	}
+}

kyaml/yaml/kfns.go

@@ -0,0 +1,66 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package yaml
+
+import "gopkg.in/yaml.v3"
+
+// AnnotationClearer removes an annotation at metadata.annotations.
+// Returns nil if the annotation or field does not exist.
+type AnnotationClearer struct {
+	Kind string `yaml:"kind,omitempty"`
+	Key  string `yaml:"key,omitempty"`
+}
+
+func (c AnnotationClearer) Filter(rn *RNode) (*RNode, error) {
+	return rn.Pipe(
+		PathGetter{Path: []string{"metadata", "annotations"}},
+		FieldClearer{Name: c.Key})
+}
+
+func ClearAnnotation(key string) AnnotationClearer {
+	return AnnotationClearer{Key: key}
+}
+
+// AnnotationSetter sets an annotation at metadata.annotations.
+// Creates metadata.annotations if does not exist.
+type AnnotationSetter struct {
+	Kind  string `yaml:"kind,omitempty"`
+	Key   string `yaml:"key,omitempty"`
+	Value string `yaml:"value,omitempty"`
+}
+
+func (s AnnotationSetter) Filter(rn *RNode) (*RNode, error) {
+	return rn.Pipe(
+		PathGetter{Path: []string{"metadata", "annotations"}, Create: yaml.MappingNode},
+		FieldSetter{Name: s.Key, Value: NewScalarRNode(s.Value)})
+}
+
+func SetAnnotation(key, value string) AnnotationSetter {
+	return AnnotationSetter{Key: key, Value: value}
+}
+
+// AnnotationGetter gets an annotation at metadata.annotations.
+// Returns nil if metadata.annotations does not exist.
+type AnnotationGetter struct {
+	Kind  string `yaml:"kind,omitempty"`
+	Key   string `yaml:"key,omitempty"`
+	Value string `yaml:"value,omitempty"`
+}
+
+// AnnotationGetter returns the annotation value.
+// Returns "", nil if the annotation does not exist.
+func (g AnnotationGetter) Filter(rn *RNode) (*RNode, error) {
+	v, err := rn.Pipe(PathGetter{Path: []string{"metadata", "annotations", g.Key}})
+	if v == nil || err != nil {
+		return v, err
+	}
+	if g.Value == "" || v.value.Value == g.Value {
+		return v, err
+	}
+	return nil, err
+}
+
+func GetAnnotation(key string) AnnotationGetter {
+	return AnnotationGetter{Key: key}
+}

kyaml/yaml/match.go

@@ -0,0 +1,208 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package yaml
+
+import (
+	"regexp"
+	"strings"
+)
+
+// PathMatcher returns all RNodes matching the path wrapped in a SequenceNode.
+// Lists may have multiple elements matching the path, and each matching element
+// is added to the return result.
+// If Path points to a SequenceNode, the SequenceNode is wrapped in another SequenceNode
+// If Path does not contain any lists, the result is still wrapped in a SequenceNode of len == 1
+type PathMatcher struct {
+	Kind string `yaml:"kind,omitempty"`
+
+	// Path is a slice of parts leading to the RNode to lookup.
+	// Each path part may be one of:
+	// * FieldMatcher -- e.g. "spec"
+	// * Map Key -- e.g. "app.k8s.io/version"
+	// * List Entry -- e.g. "[name=nginx]" or "[=-jar]"
+	//
+	// Map Keys and Fields are equivalent.
+	// See FieldMatcher for more on Fields and Map Keys.
+	//
+	// List Entries are specified as map entry to match [fieldName=fieldValue].
+	// See Elem for more on List Entries.
+	//
+	// Examples:
+	// * spec.template.spec.container with matching name: [name=nginx] -- match 'name': 'nginx'
+	// * spec.template.spec.container.argument matching a value: [=-jar] -- match '-jar'
+	Path []string `yaml:"path,omitempty"`
+
+	// Matches is set by PathMatch to publish the matched element values for each node.
+	// After running  PathMatcher.Filter, each node from the SequenceNode result may be
+	// looked up in Matches to find the field values that were matched.
+	Matches map[*Node][]string
+
+	// StripComments may be set to remove the comments on the matching Nodes.
+	// This is useful for if the nodes are to be printed in FlowStyle.
+	StripComments bool
+
+	val        *RNode
+	field      string
+	matchRegex string
+}
+
+func (p *PathMatcher) stripComments(n *Node) {
+	if n == nil {
+		return
+	}
+	if p.StripComments {
+		n.LineComment = ""
+		n.HeadComment = ""
+		n.FootComment = ""
+		for i := range n.Content {
+			p.stripComments(n.Content[i])
+		}
+	}
+}
+
+func (p *PathMatcher) Filter(rn *RNode) (*RNode, error) {
+	val, err := p.filter(rn)
+	if err != nil {
+		return nil, err
+	}
+	p.stripComments(val.YNode())
+	return val, err
+}
+
+func (p *PathMatcher) filter(rn *RNode) (*RNode, error) {
+	p.Matches = map[*Node][]string{}
+
+	if len(p.Path) == 0 {
+		// return the element wrapped in a SequenceNode
+		p.appendRNode("", rn)
+		return p.val, nil
+	}
+
+	if IsListIndex(p.Path[0]) {
+		// match seq elements
+		return p.doSeq(rn)
+	} else {
+		// match a field
+		return p.doField(rn)
+	}
+}
+
+func (p *PathMatcher) doField(rn *RNode) (*RNode, error) {
+	// lookup the field
+	field, err := rn.Pipe(Get(p.Path[0]))
+	if err != nil || field == nil {
+		// if the field doesn't exist, return nil
+		return nil, err
+	}
+
+	// recurse on the field, removing the first element of the path
+	pm := &PathMatcher{Path: p.Path[1:]}
+	p.val, err = pm.filter(field)
+	p.Matches = pm.Matches
+	return p.val, err
+}
+
+// doSeq iterates over a sequence and appends elements matching the path regex to p.Val
+func (p *PathMatcher) doSeq(rn *RNode) (*RNode, error) {
+
+	// parse the field + match pair
+	var err error
+	p.field, p.matchRegex, err = SplitIndexNameValue(p.Path[0])
+	if err != nil {
+		return nil, err
+	}
+
+	if p.field == "" {
+		err = rn.VisitElements(p.visitPrimitiveElem)
+	} else {
+		err = rn.VisitElements(p.visitElem)
+	}
+	if err != nil || p.val == nil || len(p.val.YNode().Content) == 0 {
+		return nil, err
+	}
+
+	return p.val, nil
+}
+
+func (p *PathMatcher) visitPrimitiveElem(elem *RNode) error {
+	r, err := regexp.Compile(p.matchRegex)
+	if err != nil {
+		return err
+	}
+
+	str, err := elem.String()
+	if err != nil {
+		return err
+	}
+	str = strings.TrimSpace(str)
+	if !r.MatchString(str) {
+		return nil
+	}
+
+	p.appendRNode("", elem)
+	return nil
+}
+
+func (p *PathMatcher) visitElem(elem *RNode) error {
+	r, err := regexp.Compile(p.matchRegex)
+	if err != nil {
+		return err
+	}
+
+	// check if this elements field matches the regex
+	val := elem.Field(p.field)
+	if val == nil || val.Value == nil {
+		return nil
+	}
+	str, err := val.Value.String()
+	if err != nil {
+		return err
+	}
+	str = strings.TrimSpace(str)
+	if !r.MatchString(str) {
+		return nil
+	}
+
+	// recurse on the matching element
+	pm := &PathMatcher{Path: p.Path[1:]}
+	add, err := pm.filter(elem)
+	for k, v := range pm.Matches {
+		p.Matches[k] = v
+	}
+	if err != nil || add == nil {
+		return err
+	}
+	p.append(str, add.Content()...)
+	return nil
+}
+
+func (p *PathMatcher) appendRNode(path string, node *RNode) {
+	p.append(path, node.YNode())
+}
+
+func (p *PathMatcher) append(path string, nodes ...*Node) {
+	if p.val == nil {
+		p.val = NewRNode(&Node{Kind: SequenceNode})
+	}
+	for i := range nodes {
+		node := nodes[i]
+		p.val.YNode().Content = append(p.val.YNode().Content, node)
+		// record the path if specified
+		if path != "" {
+			p.Matches[node] = append(p.Matches[node], path)
+		}
+	}
+}
+
+func cleanPath(path []string) []string {
+	var p []string
+	for _, elem := range path {
+		elem = strings.TrimSpace(elem)
+		if len(elem) == 0 {
+			continue
+		}
+		p = append(p, elem)
+	}
+	return p
+}

kyaml/yaml/match_test.go

@@ -0,0 +1,88 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package yaml
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestPathMatcher_Filter(t *testing.T) {
+	node := MustParse(`apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: nginx-deployment
+  labels:
+    app: nginx
+spec:
+  replicas: 3
+  selector:
+    matchLabels:
+      app: nginx
+  template:
+    metadata:
+      labels:
+        app: nginx
+    spec:
+      containers:
+      - name: nginx
+        image: nginx:1.7.9
+        ports:
+        - containerPort: 80
+      - name: sidecar
+        image: sidecar:1.0.0
+        ports:
+        - containerPort: 8081
+        - containerPort: 9090
+`)
+
+	updates := []struct {
+		path  []string
+		value string
+	}{
+		{[]string{
+			"spec", "template", "spec", "containers", "[name=.*]"},
+			"- name: nginx\n  image: nginx:1.7.9\n  ports:\n  - containerPort: 80\n" +
+				"- name: sidecar\n  image: sidecar:1.0.0\n  ports:\n  - containerPort: 8081\n  - containerPort: 9090\n"},
+		{[]string{
+			"spec", "template", "spec", "containers", "[name=.*]", "image"},
+			"- nginx:1.7.9\n- sidecar:1.0.0\n"},
+		{[]string{
+			"spec", "template", "spec", "containers", "[name=n.*]", "image"},
+			"- nginx:1.7.9\n"},
+		{[]string{
+			"spec", "template", "spec", "containers", "[name=s.*]", "image"},
+			"- sidecar:1.0.0\n"},
+		{[]string{
+			"spec", "template", "spec", "containers", "[name=.*x]", "image"},
+			"- nginx:1.7.9\n"},
+		{[]string{
+			"spec", "template", "spec", "containers", "[name=.*]", "ports"},
+			"- - containerPort: 80\n- - containerPort: 8081\n  - containerPort: 9090\n"},
+		{[]string{
+			"spec", "template", "spec", "containers", "[name=.*]", "ports", "[containerPort=8.*]"},
+			"- containerPort: 80\n- containerPort: 8081\n"},
+		{[]string{
+			"spec", "template", "spec", "containers", "[name=.*]", "ports", "[containerPort=.*1]"},
+			"- containerPort: 8081\n"},
+		{[]string{
+			"spec", "template", "spec", "containers", "[name=.*]", "ports", "[containerPort=9.*]"},
+			"- containerPort: 9090\n"},
+		{[]string{
+			"spec", "template", "spec", "containers", "[name=s.*]", "ports", "[containerPort=8.*]"},
+			"- containerPort: 8081\n"},
+		{[]string{
+			"spec", "template", "spec", "containers", "[name=s.*]", "ports", "[containerPort=.*2]"},
+			""},
+	}
+	for i, u := range updates {
+		result, err := node.Pipe(&PathMatcher{Path: u.path})
+		if !assert.NoError(t, err) {
+			return
+		}
+		assert.Equal(t, u.value, result.MustString(), fmt.Sprintf("%d", i))
+	}
+}

kyaml/yaml/merge2/element_test.go

@@ -0,0 +1,265 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package merge2_test
+
+var elementTestCases = []testCase{
+	{`merge Element -- keep field in dest`,
+		`
+kind: Deployment
+items:
+- name: foo
+  image: foo:v1
+`,
+		`
+kind: Deployment
+items:
+- name: foo
+  image: foo:v0
+  command: ['run.sh']
+`,
+		`
+kind: Deployment
+items:
+- name: foo
+  image: foo:v1
+  command:
+  - run.sh
+`,
+	},
+
+	{`merge Element -- add field to dest`,
+		`
+kind: Deployment
+items:
+- name: foo
+  image: foo:v1
+  command: ['run.sh']
+`,
+		`
+kind: Deployment
+items:
+- name: foo
+  image: foo:v0
+`,
+		`
+kind: Deployment
+items:
+- name: foo
+  image: foo:v1
+  command:
+  - run.sh
+`,
+	},
+
+	{`merge Element -- add list, empty in dest`,
+		`
+kind: Deployment
+items:
+- name: foo
+  image: foo:v1
+  command: ['run.sh']
+`,
+		`
+kind: Deployment
+items: {}
+`,
+		`
+kind: Deployment
+items:
+- name: foo
+  image: foo:v1
+  command:
+  - run.sh
+`,
+	},
+
+	{`merge Element -- add list, missing from dest`,
+		`
+kind: Deployment
+items:
+- name: foo
+  image: foo:v1
+  command: ['run.sh']
+`,
+		`
+kind: Deployment
+`,
+		`
+kind: Deployment
+items:
+- name: foo
+  image: foo:v1
+  command:
+  - run.sh
+`,
+	},
+
+	{`merge Element -- add Element first`,
+		`
+kind: Deployment
+items:
+- name: bar
+  image: bar:v1
+  command: ['run2.sh']
+- name: foo
+  image: foo:v1
+`,
+		`
+kind: Deployment
+items:
+- name: foo
+  image: foo:v0
+`,
+		`
+kind: Deployment
+items:
+- name: foo
+  image: foo:v1
+- name: bar
+  image: bar:v1
+  command:
+  - run2.sh
+`,
+	},
+
+	{`merge Element -- add Element second`,
+		`
+kind: Deployment
+items:
+- name: foo
+  image: foo:v1
+- name: bar
+  image: bar:v1
+  command: ['run2.sh']
+`,
+		`
+kind: Deployment
+items:
+- name: foo
+  image: foo:v0
+`,
+		`
+kind: Deployment
+items:
+- name: foo
+  image: foo:v1
+- name: bar
+  image: bar:v1
+  command:
+  - run2.sh
+`,
+	},
+
+	//
+	// Test Case
+	//
+	{`keep list -- list missing from src`,
+		`
+kind: Deployment
+`,
+		`
+kind: Deployment
+items:
+- name: foo
+  image: foo:v1
+- name: bar
+  image: bar:v1
+  command: ['run2.sh']
+`,
+		`
+kind: Deployment
+items:
+- name: foo
+  image: foo:v1
+- name: bar
+  image: bar:v1
+  command:
+  - run2.sh
+`,
+	},
+
+	//
+	// Test Case
+	//
+	{`keep Element -- element missing in src`,
+		`
+kind: Deployment
+items:
+- name: foo
+  image: foo:v1
+`,
+		`
+kind: Deployment
+items:
+- name: foo
+  image: foo:v0
+- name: bar
+  image: bar:v1
+  command: ['run2.sh']
+`,
+		`
+kind: Deployment
+items:
+- name: foo
+  image: foo:v1
+- name: bar
+  image: bar:v1
+  command:
+  - run2.sh
+`,
+	},
+
+	//
+	// Test Case
+	//
+	{`keep element -- empty list in src`,
+		`
+kind: Deployment
+items: {}
+`,
+		`
+kind: Deployment
+items:
+- name: foo
+  image: foo:v1
+- name: bar
+  image: bar:v1
+  command:
+  - run2.sh
+`,
+		`
+kind: Deployment
+items:
+- name: foo
+  image: foo:v1
+- name: bar
+  image: bar:v1
+  command:
+  - run2.sh
+`,
+	},
+
+	//
+	// Test Case
+	//
+	{`remove Element -- null in src`,
+		`
+kind: Deployment
+items: null
+`,
+		`
+kind: Deployment
+items:
+- name: foo
+  image: foo:v1
+- name: bar
+  image: bar:v1
+  command:
+  - run2.sh
+`,
+		`
+kind: Deployment
+`,
+	},
+}

kyaml/yaml/merge2/list_test.go

@@ -0,0 +1,140 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package merge2_test
+
+var listTestCases = []testCase{
+	{`replace List -- different value in dest`,
+		`
+kind: Deployment
+items:
+- 1
+- 2
+- 3
+`,
+		`
+kind: Deployment
+items:
+- 0
+- 1
+`,
+		`
+kind: Deployment
+items:
+- 1
+- 2
+- 3
+`,
+	},
+
+	{`replace List -- missing from dest`,
+		`
+kind: Deployment
+items:
+- 1
+- 2
+- 3
+`,
+		`
+kind: Deployment
+`,
+		`
+kind: Deployment
+items:
+- 1
+- 2
+- 3
+`,
+	},
+
+	//
+	// Test Case
+	//
+	{`keep List -- same value in src and dest`,
+		`
+kind: Deployment
+items:
+- 1
+- 2
+- 3
+`,
+		`
+kind: Deployment
+items:
+- 1
+- 2
+- 3
+`,
+		`
+kind: Deployment
+items:
+- 1
+- 2
+- 3
+`,
+	},
+
+	//
+	// Test Case
+	//
+	{`keep List -- unspecified in src`,
+		`
+kind: Deployment
+`,
+		`
+kind: Deployment
+items:
+- 1
+- 2
+- 3
+`,
+		`
+kind: Deployment
+items:
+- 1
+- 2
+- 3
+`,
+	},
+
+	//
+	// Test Case
+	//
+	{`remove List -- null in src`,
+		`
+kind: Deployment
+items: null
+`,
+		`
+kind: Deployment
+items:
+- 1
+- 2
+- 3
+`,
+		`
+kind: Deployment
+`,
+	},
+
+	//
+	// Test Case
+	//
+	{`remove list -- empty in src`,
+		`
+kind: Deployment
+items: {}
+`,
+		`
+kind: Deployment
+items:
+- 1
+- 2
+- 3
+`,
+		`
+kind: Deployment
+items: {}
+`,
+	},
+}

kyaml/yaml/merge2/map_test.go

@@ -0,0 +1,186 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package merge2_test
+
+var mapTestCases = []testCase{
+	{`merge Map -- update field in dest`,
+		`
+kind: Deployment
+spec:
+  foo: bar1
+`,
+		`
+kind: Deployment
+spec:
+  foo: bar0
+  baz: buz
+`,
+		`
+kind: Deployment
+spec:
+  foo: bar1
+  baz: buz
+`,
+	},
+
+	{`merge Map -- add field to dest`,
+		`
+kind: Deployment
+spec:
+  foo: bar1
+  baz: buz
+`,
+		`
+kind: Deployment
+spec:
+  foo: bar0
+`,
+		`
+kind: Deployment
+spec:
+  foo: bar1
+  baz: buz
+`,
+	},
+
+	{`merge Map -- add list, empty in dest`,
+		`
+kind: Deployment
+spec:
+  foo: bar1
+  baz: buz
+`,
+		`
+kind: Deployment
+spec: {}
+`,
+		`
+kind: Deployment
+spec:
+  foo: bar1
+  baz: buz
+`,
+	},
+
+	{`merge Map -- add list, missing from dest`,
+		`
+kind: Deployment
+spec:
+  foo: bar1
+  baz: buz
+`,
+		`
+kind: Deployment
+`,
+		`
+kind: Deployment
+spec:
+  foo: bar1
+  baz: buz
+`,
+	},
+
+	{`merge Map -- add Map first`,
+		`
+kind: Deployment
+spec:
+  foo: bar1
+  baz: buz
+`,
+		`
+kind: Deployment
+spec:
+  foo: bar1
+`,
+		`
+kind: Deployment
+spec:
+  foo: bar1
+  baz: buz
+`,
+	},
+
+	{`merge Map -- add Map second`,
+		`
+kind: Deployment
+spec:
+  baz: buz
+  foo: bar1
+`,
+		`
+kind: Deployment
+spec:
+  foo: bar1
+`,
+		`
+kind: Deployment
+spec:
+  foo: bar1
+  baz: buz
+`,
+	},
+
+	//
+	// Test Case
+	//
+	{`keep map -- map missing from src`,
+		`
+kind: Deployment
+`,
+		`
+kind: Deployment
+spec:
+  foo: bar1
+  baz: buz
+`,
+		`
+kind: Deployment
+spec:
+  foo: bar1
+  baz: buz
+`,
+	},
+
+	//
+	// Test Case
+	//
+	{`keep map -- empty list in src`,
+		`
+kind: Deployment
+items: {}
+`,
+		`
+kind: Deployment
+spec:
+  foo: bar1
+  baz: buz
+`,
+		`
+kind: Deployment
+spec:
+  foo: bar1
+  baz: buz
+items: {}
+`,
+	},
+
+	//
+	// Test Case
+	//
+	{`remove Map -- null in src`,
+		`
+kind: Deployment
+spec: null
+`,
+		`
+kind: Deployment
+spec:
+  foo: bar1
+  baz: buz
+`,
+		`
+kind: Deployment
+`,
+	},
+}

kyaml/yaml/merge2/merge2.go

@@ -0,0 +1,201 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+// Package merge contains libraries for merging fields from one RNode to another
+// RNode
+package merge2
+
+import (
+	"sigs.k8s.io/kustomize/kyaml/yaml"
+	"sigs.k8s.io/kustomize/kyaml/yaml/walk"
+)
+
+const Help = `
+Description:
+
+  merge merges fields from a source to a destination, overriding the destination fields
+  where they differ.
+
+  ### Merge Rules
+
+  Fields are recursively merged using the following rules:
+
+  - scalars
+    - if present only in the dest, it keeps its value
+    - if present in the src and is non-null, take the src value -- if ` + "`null`" + `, clear it
+` + "    - example src: `5`, dest: `3` => result: `5`" + `
+
+  - non-associative lists -- lists without a merge key
+    - if present only in the dest, it keeps its value
+    - if present in the src and is non-null, take the src value -- if ` + "`null`" + `, clear it
+` + "    - example src: `[1, 2, 3]`, dest: `[a, b, c]` => result: `[1, 2, 3]`" + `
+
+  - map keys and fields -- paired by the map-key / field-name
+    - if present only in the dest, it keeps its value
+    - if present only in the src, it is added to the dest
+    - if the field is present in both the src and dest, and the src value is 'null', the field is removed from the dest
+    - if the field is present in both the src and dest, the value is recursively merged
+` + "    - example src: `{'key1': 'value1', 'key2': 'value2'}`, dest: `{'key2': 'value0', 'key3': 'value3'}` => result: `{'key1': 'value1', 'key2': 'value2', 'key3': 'value3'}`" + `
+
+  - associative list elements -- paired by the associative key
+    - if present only in the dest, it keeps its value in the list
+    - if present only in the src, it is added to the dest list
+    - if the field is present in both the src and dest, the value is recursively merged
+
+  ### Associative Keys
+
+  Associative keys are used to identify "same" elements within 2 different lists, and merge them.
+  The following fields are recognized as associative keys:
+
+` + "[`mountPath`, `devicePath`, `ip`, `type`, `topologyKey`, `name`, `containerPort`]" + `
+
+  Any lists where all of the elements contain associative keys will be merged as associative lists.
+
+  ### Example
+
+  > Source
+
+	apiVersion: apps/v1
+	kind: Deployment
+	spec:
+	  replicas: 3 # scalar
+	  template:
+	    spec:
+	      containers:  # associative list -- (name)
+	      - name: nginx
+	        image: nginx:1.7
+	        command: ['new_run.sh', 'arg1'] # non-associative list
+	      - name: sidecar2
+	        image: sidecar2:v1
+
+  > Destination
+
+	apiVersion: apps/v1
+	kind: Deployment
+	spec:
+	  replicas: 1
+	  template:
+	    spec:
+	      containers:
+	      - name: nginx
+	        image: nginx:1.6
+	        command: ['old_run.sh', 'arg0']
+	      - name: sidecar1
+	        image: sidecar1:v1
+
+  > Result
+
+	apiVersion: apps/v1
+	kind: Deployment
+	spec:
+	  replicas: 3 # scalar
+	  template:
+	    spec:
+	      containers:  # associative list -- (name)
+	      - name: nginx
+	        image: nginx:1.7
+	        command: ['new_run.sh', 'arg1'] # non-associative list
+	      - name: sidecar1
+	        image: sidecar1:v1
+	      - name: sidecar2
+	        image: sidecar2:v1
+`
+
+// Merge merges fields from src into dest.
+func Merge(src, dest *yaml.RNode) (*yaml.RNode, error) {
+	return walk.Walker{Sources: []*yaml.RNode{dest, src}, Visitor: Merger{}}.Walk()
+}
+
+// Merge parses the arguments, and merges fields from srcStr into destStr.
+func MergeStrings(srcStr, destStr string) (string, error) {
+	src, err := yaml.Parse(srcStr)
+	if err != nil {
+		return "", err
+	}
+	dest, err := yaml.Parse(destStr)
+	if err != nil {
+		return "", err
+	}
+
+	result, err := Merge(src, dest)
+	if err != nil {
+		return "", err
+	}
+	return result.String()
+}
+
+type Merger struct {
+	// for forwards compatibility when new functions are added to the interface
+}
+
+var _ walk.Visitor = Merger{}
+
+func (m Merger) VisitMap(nodes walk.Sources) (*yaml.RNode, error) {
+	if err := m.SetComments(nodes); err != nil {
+		return nil, err
+	}
+	if yaml.IsEmpty(nodes.Dest()) {
+		// Add
+		return nodes.Origin(), nil
+	}
+	if yaml.IsNull(nodes.Origin()) {
+		// clear the value
+		return walk.ClearNode, nil
+	}
+	// Recursively Merge dest
+	return nodes.Dest(), nil
+}
+
+func (m Merger) VisitScalar(nodes walk.Sources) (*yaml.RNode, error) {
+	if err := m.SetComments(nodes); err != nil {
+		return nil, err
+	}
+	// Override value
+	if nodes.Origin() != nil {
+		return nodes.Origin(), nil
+	}
+	// Keep
+	return nodes.Dest(), nil
+}
+
+func (m Merger) VisitList(nodes walk.Sources, kind walk.ListKind) (*yaml.RNode, error) {
+	if err := m.SetComments(nodes); err != nil {
+		return nil, err
+	}
+	if kind == walk.NonAssociateList {
+		// Override value
+		if nodes.Origin() != nil {
+			return nodes.Origin(), nil
+		}
+		// Keep
+		return nodes.Dest(), nil
+	}
+
+	// Add
+	if yaml.IsEmpty(nodes.Dest()) {
+		return nodes.Origin(), nil
+	}
+	// Clear
+	if yaml.IsNull(nodes.Origin()) {
+		return walk.ClearNode, nil
+	}
+	// Recursively Merge dest
+	return nodes.Dest(), nil
+}
+
+// SetComments copies the dest comments to the source comments if they are present
+// on the source.
+func (m Merger) SetComments(sources walk.Sources) error {
+	source := sources.Origin()
+	dest := sources.Dest()
+	if source != nil && source.YNode().FootComment != "" {
+		dest.YNode().FootComment = source.YNode().FootComment
+	}
+	if source != nil && source.YNode().HeadComment != "" {
+		dest.YNode().HeadComment = source.YNode().HeadComment
+	}
+	if source != nil && source.YNode().LineComment != "" {
+		dest.YNode().LineComment = source.YNode().LineComment
+	}
+	return nil
+}

kyaml/yaml/merge2/merge2_old_test.go

@@ -0,0 +1,510 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package merge2_test
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"sigs.k8s.io/kustomize/kyaml/kio/filters"
+	"sigs.k8s.io/kustomize/kyaml/yaml"
+	. "sigs.k8s.io/kustomize/kyaml/yaml/merge2"
+)
+
+const dest = `
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: app
+  labels:
+    app: java
+  annotations:
+    a.b.c: d.e.f
+    g: h1
+    i: j
+    m: n2
+spec:
+  template:
+    spec:
+      containers:
+      - name: nginx
+        image: nginx:1.7.9
+        args: ['c', 'a', 'b']
+        env:
+        - name: DEMO_GREETING
+          value: "Hello from the environment"
+        - name: DEMO_FAREWELL
+          value: "Such a sweet sorrow"
+`
+
+func TestMerge_map(t *testing.T) {
+	dest := yaml.MustParse(dest)
+	src := yaml.MustParse(`
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: app
+  labels:
+    app: java
+  annotations:
+    a.b.c: d.e.f
+    g: h2
+    k: l
+    m: n1
+`)
+
+	result, err := Merge(src, dest)
+	if !assert.NoError(t, err) {
+		return
+	}
+	actual, err := result.String()
+	if !assert.NoError(t, err) {
+		return
+	}
+
+	expected := `
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: app
+  labels:
+    app: java
+  annotations:
+    a.b.c: d.e.f
+    g: h2
+    i: j
+    k: l
+    m: n1
+spec:
+  template:
+    spec:
+      containers:
+      - name: nginx
+        image: nginx:1.7.9
+        args:
+        - c
+        - a
+        - b
+        env:
+        - name: DEMO_GREETING
+          value: "Hello from the environment"
+        - name: DEMO_FAREWELL
+          value: "Such a sweet sorrow"
+`
+	b, err := filters.FormatInput(bytes.NewBufferString(expected))
+	if !assert.NoError(t, err) {
+		return
+	}
+	expected = b.String()
+
+	b, err = filters.FormatInput(bytes.NewBufferString(actual))
+	if !assert.NoError(t, err) {
+		return
+	}
+	actual = b.String()
+
+	assert.Equal(t, expected, actual)
+}
+
+func TestMerge_clear(t *testing.T) {
+	dest := yaml.MustParse(dest)
+	src := yaml.MustParse(`
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  annotations: null
+`)
+
+	result, err := Merge(src, dest)
+	if !assert.NoError(t, err) {
+		return
+	}
+	actual, err := result.String()
+	if !assert.NoError(t, err) {
+		return
+	}
+
+	expected := `
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: app
+  labels:
+    app: java
+spec:
+  template:
+    spec:
+      containers:
+      - name: nginx
+        image: nginx:1.7.9
+        args:
+        - c
+        - a
+        - b
+        env:
+        - name: DEMO_GREETING
+          value: "Hello from the environment"
+        - name: DEMO_FAREWELL
+          value: "Such a sweet sorrow"
+`
+	b, err := filters.FormatInput(bytes.NewBufferString(expected))
+	if !assert.NoError(t, err) {
+		return
+	}
+	expected = b.String()
+
+	b, err = filters.FormatInput(bytes.NewBufferString(actual))
+	if !assert.NoError(t, err) {
+		return
+	}
+	actual = b.String()
+
+	assert.Equal(t, expected, actual)
+}
+
+func TestMerge_mapInverse(t *testing.T) {
+	dest := yaml.MustParse(dest)
+	src := yaml.MustParse(`
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: app
+  labels:
+    app: java
+  annotations:
+    a.b.c: d.e.f
+    g: h2
+    k: l
+    m: n1
+`)
+
+	result, err := Merge(dest, src)
+	if !assert.NoError(t, err) {
+		return
+	}
+	actual, err := result.String()
+	if !assert.NoError(t, err) {
+		return
+	}
+
+	expected := `
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: app
+  labels:
+    app: java
+  annotations:
+    a.b.c: d.e.f
+    g: h1
+    i: j
+    k: l
+    m: n2
+spec:
+  template:
+    spec:
+      containers:
+      - name: nginx
+        image: nginx:1.7.9
+        args:
+        - c
+        - a
+        - b
+        env:
+        - name: DEMO_GREETING
+          value: "Hello from the environment"
+        - name: DEMO_FAREWELL
+          value: "Such a sweet sorrow"
+`
+	b, err := filters.FormatInput(bytes.NewBufferString(expected))
+	if !assert.NoError(t, err) {
+		return
+	}
+	expected = b.String()
+
+	b, err = filters.FormatInput(bytes.NewBufferString(actual))
+	if !assert.NoError(t, err) {
+		return
+	}
+	actual = b.String()
+
+	assert.Equal(t, expected, actual)
+}
+
+func TestMerge_listElem(t *testing.T) {
+	dest := yaml.MustParse(dest)
+	src := yaml.MustParse(`
+apiVersion: apps/v1
+kind: Deployment
+spec:
+  template:
+    spec:
+      containers:
+      - name: nginx
+        env:
+        - name: DEMO_GREETING
+          value: "New Demo Greeting"
+        - name: NEW_DEMO_VALUE
+          value: "Another Env Not In The Dest"
+`)
+
+	result, err := Merge(src, dest)
+	if !assert.NoError(t, err) {
+		return
+	}
+	actual, err := result.String()
+	if !assert.NoError(t, err) {
+		return
+	}
+
+	expected := `apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: app
+  labels:
+    app: java
+  annotations:
+    a.b.c: d.e.f
+    g: h1
+    i: j
+    m: n2
+spec:
+  template:
+    spec:
+      containers:
+      - name: nginx
+        image: nginx:1.7.9
+        args:
+        - c
+        - a
+        - b
+        env:
+        - name: DEMO_GREETING
+          value: "New Demo Greeting"
+        - name: DEMO_FAREWELL
+          value: "Such a sweet sorrow"
+        - name: NEW_DEMO_VALUE
+          value: "Another Env Not In The Dest"
+`
+
+	b, err := filters.FormatInput(bytes.NewBufferString(expected))
+	if !assert.NoError(t, err) {
+		return
+	}
+	expected = b.String()
+
+	b, err = filters.FormatInput(bytes.NewBufferString(actual))
+	if !assert.NoError(t, err) {
+		return
+	}
+	actual = b.String()
+
+	assert.Equal(t, expected, actual)
+}
+
+func TestMerge_list(t *testing.T) {
+	dest := yaml.MustParse(dest)
+	src := yaml.MustParse(`
+apiVersion: apps/v1
+kind: Deployment
+spec:
+  template:
+    spec:
+      containers:
+      - name: nginx
+        args: ['e', 'd', 'f']
+`)
+
+	result, err := Merge(src, dest)
+	if !assert.NoError(t, err) {
+		return
+	}
+	actual, err := result.String()
+	if !assert.NoError(t, err) {
+		return
+	}
+
+	expected := `apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: app
+  labels:
+    app: java
+  annotations:
+    a.b.c: d.e.f
+    g: h1
+    i: j
+    m: n2
+spec:
+  template:
+    spec:
+      containers:
+      - name: nginx
+        image: nginx:1.7.9
+        args:
+        - e
+        - d
+        - f
+        env:
+        - name: DEMO_GREETING
+          value: "Hello from the environment"
+        - name: DEMO_FAREWELL
+          value: "Such a sweet sorrow"
+`
+
+	b, err := filters.FormatInput(bytes.NewBufferString(expected))
+	if !assert.NoError(t, err) {
+		return
+	}
+	expected = b.String()
+
+	b, err = filters.FormatInput(bytes.NewBufferString(actual))
+	if !assert.NoError(t, err) {
+		return
+	}
+	actual = b.String()
+
+	assert.Equal(t, expected, actual)
+}
+
+func TestMerge_commentsKept(t *testing.T) {
+	actual, err := MergeStrings(`
+a:
+  b:
+    c: e
+`,
+		`
+a:
+  b:
+    # header comment
+    c: d
+`)
+	if !assert.NoError(t, err) {
+		return
+	}
+	assert.Equal(t, `a:
+  b:
+    # header comment
+    c: e
+`, actual)
+
+	actual, err = MergeStrings(`
+a:
+  b:
+    c: e
+`,
+		`
+a:
+  b:
+    c: d
+    # footer comment
+`)
+	if !assert.NoError(t, err) {
+		return
+	}
+	assert.Equal(t, `a:
+  b:
+    c: e
+    # footer comment
+`, actual)
+
+	actual, err = MergeStrings(`
+a:
+  b:
+    c: e
+`,
+		`
+a:
+  b:
+    c: d # line comment
+`)
+	if !assert.NoError(t, err) {
+		return
+	}
+	assert.Equal(t, `a:
+  b:
+    c: e
+`, actual)
+}
+
+func TestMerge_commentsOverride(t *testing.T) {
+	actual, err := MergeStrings(`
+a:
+  b:
+    # header comment
+    c: e
+`,
+		`
+a:
+  b:
+    # replace comment
+    c: d
+`)
+	if !assert.NoError(t, err) {
+		return
+	}
+	assert.Equal(t, `a:
+  b:
+    # replace comment
+    c: e
+`, actual)
+
+	actual, err = MergeStrings(`
+a:
+  b:
+    c: e
+    # footer comment
+`,
+		`
+a:
+  b:
+    c: d
+    # replace comment
+`)
+	if !assert.NoError(t, err) {
+		return
+	}
+	assert.Equal(t, `a:
+  b:
+    c: e
+    # replace comment
+`, actual)
+
+	actual, err = MergeStrings(`
+a:
+  b:
+    c: e # line comment
+`,
+		`
+a:
+  b:
+    c: d # replace comment
+`)
+	if !assert.NoError(t, err) {
+		return
+	}
+	assert.Equal(t, `a:
+  b:
+    c: e # line comment
+`, actual)
+
+	actual, err = MergeStrings(`
+a:
+  b:
+    c: d # line comment
+`,
+		`
+a:
+  b:
+    c: d # replace comment
+`)
+	if !assert.NoError(t, err) {
+		return
+	}
+	assert.Equal(t, `a:
+  b:
+    c: d # line comment
+`, actual)
+}

kyaml/yaml/merge2/merge2_test.go

@@ -0,0 +1,47 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package merge2_test
+
+import (
+	"bytes"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"sigs.k8s.io/kustomize/kyaml/kio/filters"
+	. "sigs.k8s.io/kustomize/kyaml/yaml/merge2"
+)
+
+var testCases = [][]testCase{scalarTestCases, listTestCases, elementTestCases, mapTestCases}
+
+func TestMerge(t *testing.T) {
+	for i := range testCases {
+		for _, tc := range testCases[i] {
+			actual, err := MergeStrings(tc.source, tc.dest)
+			if !assert.NoError(t, err, tc.description) {
+				t.FailNow()
+			}
+			e, err := filters.FormatInput(bytes.NewBufferString(tc.expected))
+			if !assert.NoError(t, err) {
+				t.FailNow()
+			}
+			estr := strings.TrimSpace(e.String())
+			a, err := filters.FormatInput(bytes.NewBufferString(actual))
+			if !assert.NoError(t, err) {
+				t.FailNow()
+			}
+			astr := strings.TrimSpace(a.String())
+			if !assert.Equal(t, estr, astr, tc.description) {
+				t.FailNow()
+			}
+		}
+	}
+}
+
+type testCase struct {
+	description string
+	source      string
+	dest        string
+	expected    string
+}

kyaml/yaml/merge2/scalar_test.go

@@ -0,0 +1,138 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package merge2_test
+
+var scalarTestCases = []testCase{
+	{`replace scalar -- different value in dest`,
+		`
+kind: Deployment
+field: value1
+`,
+		`
+kind: Deployment
+field: value0
+`,
+		`
+kind: Deployment
+field: value1
+`,
+	},
+
+	{`replace scalar -- missing from dest`,
+		`
+kind: Deployment
+field: value1
+`,
+		`
+kind: Deployment
+`,
+		`
+kind: Deployment
+field: value1
+`,
+	},
+
+	//
+	// Test Case
+	//
+	{`keep scalar -- same value in src and dest`,
+		`
+kind: Deployment
+field: value1
+`,
+		`
+kind: Deployment
+field: value1
+`,
+		`
+kind: Deployment
+field: value1
+`,
+	},
+
+	//
+	// Test Case
+	//
+	{`keep scalar -- unspecified in src`,
+		`
+kind: Deployment
+`,
+		`
+kind: Deployment
+field: value1
+`,
+		`
+kind: Deployment
+field: value1
+`,
+	},
+
+	//
+	// Test Case
+	//
+	{`remove scalar -- null in src`,
+		`
+kind: Deployment
+field: null
+`,
+		`
+kind: Deployment
+field: value1
+`,
+		`
+kind: Deployment
+`,
+	},
+
+	//
+	// Test Case
+	//
+	{`remove scalar -- empty in src`,
+		`
+kind: Deployment
+field: {}
+`,
+		`
+kind: Deployment
+field: value1
+`,
+		`
+kind: Deployment
+field: {}
+`,
+	},
+
+	//
+	// Test Case
+	//
+	{`remove scalar -- null in src, missing in dest`,
+		`
+kind: Deployment
+field: null
+`,
+		`
+kind: Deployment
+`,
+		`
+kind: Deployment
+`,
+	},
+
+	//
+	// Test Case
+	//
+	{`merge an empty value`,
+		`
+kind: Deployment
+field: {}
+`,
+		`
+kind: Deployment
+`,
+		`
+kind: Deployment
+field: {}
+`,
+	},
+}

kyaml/yaml/merge3/element_test.go

@@ -0,0 +1,422 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package merge3_test
+
+var elementTestCases = []testCase{
+	//
+	// Test Case
+	//
+	{`Add an element to an existing list`,
+		`
+kind: Deployment
+containers:
+- name: foo
+  image: foo:1
+`,
+		`
+kind: Deployment
+containers:
+- name: foo
+  image: foo:1
+- name: baz
+  image: baz:2
+`,
+		`
+kind: Deployment
+containers:
+- name: foo
+  image: foo:1
+`,
+		`
+kind: Deployment
+containers:
+- name: foo
+  image: foo:1
+- image: baz:2
+  name: baz
+  
+`, nil},
+
+	//
+	// Test Case
+	//
+	{`Add an element to a non-existing list`,
+		`
+kind: Deployment`,
+		`
+kind: Deployment
+containers:
+- name: foo
+  image: foo:bar
+`,
+		`
+kind: Deployment
+`,
+		`
+kind: Deployment
+containers:
+- image: foo:bar
+  name: foo
+`, nil},
+
+	{`Add an element to a non-existing list, existing in dest`,
+		`
+kind: Deployment`,
+		`
+kind: Deployment
+containers:
+- name: foo
+  image: foo:bar
+`,
+		`
+kind: Deployment
+containers:
+- name: baz
+  image: baz:bar
+`,
+		`
+kind: Deployment
+containers:
+- name: baz
+  image: baz:bar
+- image: foo:bar
+  name: foo
+`, nil},
+
+	//
+	// Test Case
+	// TODO(pwittrock): Figure out if there is something better we can do here
+	// This element is missing from the destination -- only the new fields are added
+	{`Add a field to the element, element missing from dest`,
+		`
+kind: Deployment
+containers:
+- name: foo
+  image: foo:bar`,
+		`
+kind: Deployment
+containers:
+- name: foo
+  image: foo:bar
+  command:
+  - run.sh
+`,
+		`
+kind: Deployment
+`,
+		`
+kind: Deployment
+containers:
+- command:
+  - run.sh
+  name: foo
+`, nil},
+
+	//
+	// Test Case
+	//
+	{`Update a field on the elem, element missing from the dest`,
+		`
+kind: Deployment
+containers:
+- name: foo
+  image: foo:bar
+  command:
+  - run.sh
+`,
+		`
+kind: Deployment
+containers:
+- name: foo
+  image: foo:bar
+  command: 
+  - run2.sh
+`,
+		`
+kind: Deployment
+`,
+		`
+kind: Deployment
+containers:
+- command:
+  - run2.sh
+  name: foo
+`, nil},
+
+	//
+	// Test Case
+	//
+	{`Update a field on the elem, element present in the dest`,
+		`
+kind: Deployment
+containers:
+- name: foo
+  image: foo:bar
+  command: ['run.sh']
+`,
+		`
+kind: Deployment
+containers:
+- name: foo
+  image: foo:bar
+  command: ['run2.sh']
+`,
+		`
+kind: Deployment
+containers:
+- name: foo
+  image: foo:bar
+  command: ['run.sh']
+`,
+		`
+kind: Deployment
+containers:
+- name: foo
+  image: foo:bar
+  command:
+  - run2.sh
+`, nil},
+
+	//
+	// Test Case
+	//
+	{`Add a field on the elem, element present in the dest`,
+		`
+kind: Deployment
+containers:
+- name: foo
+  image: foo:bar
+`,
+		`
+kind: Deployment
+containers:
+- name: foo
+  image: foo:bar
+  command: ['run2.sh']
+`,
+		`
+kind: Deployment
+containers:
+- name: foo
+  image: foo:bar
+`,
+		`
+kind: Deployment
+containers:
+- name: foo
+  image: foo:bar
+  command:
+  - run2.sh
+`, nil},
+
+	//
+	// Test Case
+	//
+	{`Add a field on the elem, element and field present in the dest`,
+		`
+kind: Deployment
+containers:
+- name: foo
+  image: foo:bar
+`,
+		`
+kind: Deployment
+containers:
+- name: foo
+  image: foo:bar
+  command: ['run2.sh']
+`,
+		`
+kind: Deployment
+containers:
+- name: foo
+  image: foo:bar
+  command: ['run.sh']
+`,
+		`
+kind: Deployment
+containers:
+- name: foo
+  image: foo:bar
+  command:
+  - run2.sh
+`, nil},
+
+	//
+	// Test Case
+	//
+	{`Ignore an element`,
+		`
+kind: Deployment
+containers: {}
+`,
+		`
+kind: Deployment
+containers: {}
+`,
+		`
+kind: Deployment
+containers:
+- name: foo
+  image: foo:bar
+`,
+		`
+kind: Deployment
+containers:
+- name: foo
+  image: foo:bar
+`, nil},
+
+	//
+	// Test Case
+	//
+	{`Leave deleted`,
+		`
+kind: Deployment
+containers:
+- name: foo
+  image: foo:bar
+`,
+		`
+kind: Deployment
+`,
+		`
+kind: Deployment
+`,
+		`
+kind: Deployment
+`, nil},
+
+	//
+	// Test Case
+	//
+	{`Remove an element -- matching`,
+		`
+kind: Deployment
+containers:
+- name: foo
+  image: foo:bar
+`,
+		`
+kind: Deployment
+`,
+		`
+kind: Deployment
+containers:
+- name: foo
+  image: foo:bar
+`,
+		`
+kind: Deployment
+`, nil},
+
+	//
+	// Test Case
+	//
+	{`Remove an element -- field missing from update`,
+		`
+kind: Deployment
+containers:
+- name: foo
+  image: foo:bar
+`,
+		`
+kind: Deployment
+`,
+		`
+kind: Deployment
+containers:
+- name: foo
+  image: foo:bar
+  command: ['run.sh']
+`,
+		`
+kind: Deployment
+`, nil},
+
+	//
+	// Test Case
+	//
+	{`Remove an element -- element missing`,
+		`
+kind: Deployment
+containers:
+- name: foo
+  image: foo:bar
+- name: baz
+  image: baz:bar
+`,
+		`
+kind: Deployment
+containers:
+- name: foo
+  image: foo:bar
+`,
+		`
+kind: Deployment
+containers:
+- name: foo
+  image: foo:bar
+  command: ['run.sh']
+- name: baz
+  image: baz:bar
+`,
+		`
+kind: Deployment
+containers:
+- name: foo
+  image: foo:bar
+  command:
+  - run.sh
+`, nil},
+
+	//
+	// Test Case
+	//
+	{`Remove an element -- empty containers`,
+		`
+kind: Deployment
+containers:
+- name: foo
+  image: foo:bar
+`,
+		`
+kind: Deployment
+containers: {}
+`,
+		`
+kind: Deployment
+containers:
+- name: foo
+  image: foo:bar
+  command: ['run.sh']
+`,
+		`
+kind: Deployment
+`, nil},
+
+	//
+	// Test Case
+	//
+	{`Remove an element -- missing list field`,
+		`
+kind: Deployment
+containers:
+- name: foo
+  image: foo:bar
+`,
+		`
+kind: Deployment
+`,
+		`
+kind: Deployment
+containers:
+- name: foo
+  image: foo:bar
+  command: ['run.sh']
+`,
+		`
+kind: Deployment
+`, nil},
+}

kyaml/yaml/merge3/list_test.go

@@ -0,0 +1,232 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package merge3_test
+
+var listTestCases = []testCase{
+	// List Field Test Cases
+
+	//
+	// Test Case
+	//
+	{`Replace list`,
+		`
+list:
+- 1
+- 2
+- 3`,
+		`
+list:
+- 2
+- 3
+- 4`,
+		`
+list:
+- 1
+- 2
+- 3`,
+		`
+list:
+- 2
+- 3
+- 4`, nil},
+
+	//
+	// Test Case
+	//
+	{`Add an updated list`,
+		`
+apiVersion: apps/v1
+list: # old value
+- 1
+- 2
+- 3
+`,
+		`
+apiVersion: apps/v1
+list: # new value
+- 2
+- 3
+- 4
+`,
+		`
+apiVersion: apps/v1`,
+		`
+apiVersion: apps/v1
+list:
+- 2
+- 3
+- 4
+`, nil},
+
+	//
+	// Test Case
+	//
+	{`Add keep an omitted field`,
+		`
+apiVersion: apps/v1
+kind: Deployment`,
+		`
+apiVersion: apps/v1
+kind: StatefulSet`,
+		`
+apiVersion: apps/v1
+list: # not present in sources
+- 2
+- 3
+- 4
+`,
+		`
+apiVersion: apps/v1
+list: # not present in sources
+- 2
+- 3
+- 4
+kind: StatefulSet
+`, nil},
+
+	//
+	// Test Case
+	//
+	// TODO(#36): consider making this an error
+	{`Change an updated field`,
+		`
+apiVersion: apps/v1
+list: # old value
+- 1
+- 2
+- 3`,
+		`
+apiVersion: apps/v1
+list: # new value
+- 2
+- 3
+- 4`,
+		`
+apiVersion: apps/v1
+list: # conflicting value
+- a
+- b
+- c`,
+		`
+apiVersion: apps/v1
+list: # conflicting value
+- 2
+- 3
+- 4
+`, nil},
+
+	//
+	// Test Case
+	//
+	{`Ignore a field -- set`,
+		`
+apiVersion: apps/v1
+list: # ignore value
+- 1
+- 2
+- 3
+`,
+		`
+apiVersion: apps/v1
+list: # ignore value
+- 1
+- 2
+- 3`, `
+apiVersion: apps/v1
+list:
+- 2
+- 3
+- 4
+`, `
+apiVersion: apps/v1
+list:
+- 2
+- 3
+- 4
+`, nil},
+
+	//
+	// Test Case
+	//
+	{`Ignore a field -- empty`,
+		`
+apiVersion: apps/v1
+list: # ignore value
+- 1
+- 2
+- 3`,
+		`
+apiVersion: apps/v1
+list: # ignore value
+- 1
+- 2
+- 3`,
+		`
+apiVersion: apps/v1
+`,
+		`
+apiVersion: apps/v1
+`, nil},
+
+	//
+	// Test Case
+	//
+	{`Explicitly clear a field`,
+		`
+apiVersion: apps/v1`,
+		`
+apiVersion: apps/v1
+list: null # clear`,
+		`
+apiVersion: apps/v1
+list: # value to clear
+- 1
+- 2
+- 3`,
+		`
+apiVersion: apps/v1`, nil},
+
+	//
+	// Test Case
+	//
+	{`Implicitly clear a field`,
+		`
+apiVersion: apps/v1
+list: # clear value
+- 1
+- 2
+- 3`,
+		`
+apiVersion: apps/v1`,
+		`
+apiVersion: apps/v1
+list: # old value
+- 1
+- 2
+- 3`,
+		`
+apiVersion: apps/v1`, nil},
+
+	//
+	// Test Case
+	//
+	// TODO(#36): consider making this an error
+	{`Implicitly clear a changed field`,
+		`
+apiVersion: apps/v1
+list: # old value
+- 1
+- 2
+- 3`,
+		`
+apiVersion: apps/v1`,
+		`
+apiVersion: apps/v1
+list: # old value
+- a
+- b
+- c`,
+		`
+apiVersion: apps/v1`, nil},
+}

kyaml/yaml/merge3/map_test.go

@@ -0,0 +1,298 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package merge3_test
+
+var mapTestCases = []testCase{
+	//
+	// Test Case
+	//
+	{`Add the annotations map field`,
+		`
+kind: Deployment`,
+		`
+kind: Deployment
+metadata:
+  annotations:
+    d: e # add these annotations
+`,
+		`
+kind: Deployment`,
+		`
+kind: Deployment
+metadata:
+  annotations:
+    d: e # add these annotations`, nil},
+
+	//
+	// Test Case
+	//
+	{`Add an annotation to the field`,
+		`
+kind: Deployment
+metadata:
+  annotations:
+    a: b`,
+		`
+kind: Deployment
+metadata:
+  annotations:
+    a: b
+    d: e  # add these annotations`,
+		`
+kind: Deployment
+metadata:
+  annotations:
+    g: h  # keep these annotations`,
+		`
+kind: Deployment
+metadata:
+  annotations:
+    g: h # keep these annotations
+    d: e # add these annotations`, nil},
+
+	//
+	// Test Case
+	//
+	{`Add an annotation to the field, field missing from dest`,
+		`
+kind: Deployment
+metadata:
+  annotations:
+    a: b # ignored because unchanged`,
+		`
+kind: Deployment
+metadata:
+  annotations:
+    a: b # ignore because unchanged
+    d: e`,
+		`
+kind: Deployment`,
+		`
+kind: Deployment
+metadata:
+  annotations:
+    d: e`, nil},
+
+	//
+	// Test Case
+	//
+	{`Update an annotation on the field, field messing rom the dest`,
+		`
+kind: Deployment
+metadata:
+  annotations:
+    a: b
+    d: c`,
+		`
+kind: Deployment
+metadata:
+  annotations:
+    a: b
+    d: e  # set these annotations`,
+		`
+kind: Deployment
+metadata:
+  annotations:
+    g: h  # keep these annotations`,
+		`
+kind: Deployment
+metadata:
+  annotations:
+    g: h # keep these annotations
+    d: e # set these annotations`, nil},
+
+	//
+	// Test Case
+	//
+	{`Add an annotation to the field, field missing from dest`,
+		`
+kind: Deployment
+metadata:
+  annotations:
+    a: b # ignored because unchanged`,
+		`
+kind: Deployment
+metadata:
+  annotations:
+    a: b # ignore because unchanged
+    d: e`,
+		`
+kind: Deployment`,
+		`
+kind: Deployment
+metadata:
+  annotations:
+    d: e`, nil},
+
+	//
+	// Test Case
+	//
+	{`Remove an annotation`,
+		`
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  annotations:
+    a: b`,
+		`
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  annotations: {}`,
+		`
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  annotations:
+    c: d
+    a: b`,
+		`
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  annotations:
+    c: d`, nil},
+
+	//
+	// Test Case
+	//
+	// TODO(#36) support ~annotations~: {} deletion
+	{`Specify a field as empty that isn't present in the source`,
+		`
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: foo`,
+		`
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: foo
+  annotations: null`,
+		`
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: foo
+  annotations:
+    a: b`,
+		`
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: foo`, nil},
+
+	//
+	// Test Case
+	//
+	{`Remove an annotation`,
+		`
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  annotations:
+    a: b`,
+		`
+apiVersion: apps/v1
+kind: Deployment`,
+		`
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  annotations:
+    c: d
+    a: b`,
+		`
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  annotations:
+    c: d`, nil},
+
+	//
+	// Test Case
+	//
+	{`Remove annotations field`,
+		`
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  annotations:
+    a: b`,
+		`
+apiVersion: apps/v1
+kind: Deployment`,
+		`
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: foo`,
+		`
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: foo
+`, nil},
+
+	//
+	// Test Case
+	//
+	{`Remove annotations field, but keep in dest`,
+		`
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  annotations:
+    a: b`,
+		`
+apiVersion: apps/v1
+kind: Deployment`,
+		`
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: foo
+  annotations:
+    foo: bar # keep this annotation even though the parent field was removed`,
+		`
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: foo
+  annotations:
+    foo: bar # keep this annotation even though the parent field was removed`, nil},
+
+	//
+	// Test Case
+	//
+	{`Remove annotations, but they are already empty`,
+		`
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: foo
+  annotations:
+    a: b
+`,
+		`
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: foo
+`,
+		`
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: foo
+  annotations: {}
+`,
+		`
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: foo
+  annotations: {}
+`, nil},
+}

kyaml/yaml/merge3/merge3.go

@@ -0,0 +1,88 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+// Package merge contains libraries for merging fields from one RNode to another
+// RNode
+package merge3
+
+import (
+	"sigs.k8s.io/kustomize/kyaml/yaml"
+	"sigs.k8s.io/kustomize/kyaml/yaml/walk"
+)
+
+const Help = `
+Description:
+
+  merge3 identifies changes between an original source + updated source and merges the result
+  into a destination, overriding the destination fields where they have changed between
+  original and updated.
+
+  ### Resource MergeRules
+
+  - Resources present in the original and deleted from the update are deleted.
+  - Resources missing from the original and added in the update are added.
+  - Resources present only in the dest are kept without changes.
+  - Resources present in both the update and the dest are merged *original + update + dest => dest*.
+
+  ### Field Merge Rules
+
+  Fields are recursively merged using the following rules:
+
+  - scalars
+    - if present in either dest or updated and 'null', clear the value
+    - if unchanged between original and updated, keep dest value
+    - if changed between original and updated (added, deleted, changed), take the updated value
+
+  - non-associative lists -- lists without a merge key
+    - if present in either dest or updated and 'null', clear the value
+    - if unchanged between original and updated, keep dest value
+    - if changed between original and updated (added, deleted, changed), take the updated value
+
+  - map keys and fields -- paired by the map-key / field-name
+    - if present in either dest or updated and 'null', clear the value
+    - if present only in the dest, it keeps its value
+    - if not-present in the dest, add the delta between original-updated as a field
+    - otherwise recursively merge the value between original, updated, dest
+
+  - associative list elements -- paired by the associative key
+    - if present only in the dest, it keeps its value
+    - if not-present in the dest, add the delta between original-updated as a field
+    - otherwise recursively merge the value between original, updated, dest
+
+  ### Associative Keys
+
+  Associative keys are used to identify "same" elements within 2 different lists, and merge them.
+  The following fields are recognized as associative keys:
+
+` + "[`mountPath`, `devicePath`, `ip`, `type`, `topologyKey`, `name`, `containerPort`]" + `
+
+  Any lists where all of the elements contain associative keys will be merged as associative lists.
+`
+
+func Merge(dest, original, update *yaml.RNode) (*yaml.RNode, error) {
+	// if update == nil && original != nil => declarative deletion
+
+	return walk.Walker{Visitor: Visitor{},
+		Sources: []*yaml.RNode{dest, original, update}}.Walk()
+}
+
+func MergeStrings(dest, original, update string) (string, error) {
+	srcOriginal, err := yaml.Parse(original)
+	if err != nil {
+		return "", err
+	}
+	srcUpdated, err := yaml.Parse(update)
+	if err != nil {
+		return "", err
+	}
+	d, err := yaml.Parse(dest)
+	if err != nil {
+		return "", err
+	}
+
+	result, err := Merge(d, srcOriginal, srcUpdated)
+	if err != nil {
+		return "", err
+	}
+	return result.String()
+}

kyaml/yaml/merge3/merge3_test.go

@@ -0,0 +1,47 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package merge3_test
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	. "sigs.k8s.io/kustomize/kyaml/yaml/merge3"
+)
+
+var testCases = [][]testCase{scalarTestCases, listTestCases, mapTestCases, elementTestCases}
+
+func TestMerge(t *testing.T) {
+	for i := range testCases {
+		for _, tc := range testCases[i] {
+			actual, err := MergeStrings(tc.local, tc.origin, tc.update)
+			if tc.err == nil {
+				if !assert.NoError(t, err, tc.description) {
+					t.FailNow()
+				}
+				if !assert.Equal(t,
+					strings.TrimSpace(tc.expected), strings.TrimSpace(actual), tc.description) {
+					t.FailNow()
+				}
+			} else {
+				if !assert.Errorf(t, err, tc.description) {
+					t.FailNow()
+				}
+				if !assert.Contains(t, tc.err.Error(), err.Error()) {
+					t.FailNow()
+				}
+			}
+		}
+	}
+}
+
+type testCase struct {
+	description string
+	origin      string
+	update      string
+	local       string
+	expected    string
+	err         error
+}

kyaml/yaml/merge3/scalar_test.go

@@ -0,0 +1,135 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package merge3_test
+
+var scalarTestCases = []testCase{
+	// Scalar Field Test Cases
+	//
+	// Test Case
+	//
+	{`Set and updated a field`,
+		`kind: Deployment`,
+		`kind: StatefulSet`,
+		`kind: Deployment`,
+		`kind: StatefulSet`, nil},
+
+	{`Add an updated field`,
+		`
+apiVersion: apps/v1
+kind: Deployment # old value`,
+		`
+apiVersion: apps/v1
+kind: StatefulSet # new value`,
+		`
+apiVersion: apps/v1`,
+		`
+apiVersion: apps/v1
+kind: StatefulSet # new value`, nil},
+
+	{`Add keep an omitted field`,
+		`
+apiVersion: apps/v1
+kind: Deployment`,
+		`
+apiVersion: apps/v1
+kind: StatefulSet`,
+		`
+apiVersion: apps/v1
+spec: foo # field not present in source
+`,
+		`
+apiVersion: apps/v1
+spec: foo # field not present in source
+kind: StatefulSet
+`, nil},
+
+	//
+	// Test Case
+	//
+	// TODO(#36): consider making this an error
+	{`Change an updated field`,
+		`
+apiVersion: apps/v1
+kind: Deployment # old value`,
+		`
+apiVersion: apps/v1
+kind: StatefulSet # new value`,
+		`
+apiVersion: apps/v1
+kind: Service # conflicting value`,
+		`
+apiVersion: apps/v1
+kind: StatefulSet # new value`, nil},
+
+	{`Ignore a field`,
+		`
+apiVersion: apps/v1
+kind: Deployment # ignore this field`,
+		`
+apiVersion: apps/v1
+kind: Deployment # ignore this field`,
+		`
+apiVersion: apps/v1`,
+		`
+apiVersion: apps/v1`, nil},
+
+	{`Explicitly clear a field`,
+		`
+apiVersion: apps/v1`,
+		`
+apiVersion: apps/v1
+kind: null # clear this value`,
+		`
+apiVersion: apps/v1
+kind: Deployment # value to be cleared`,
+		`
+apiVersion: apps/v1`, nil},
+
+	{`Implicitly clear a field`,
+		`
+apiVersion: apps/v1
+kind: Deployment # clear this field`,
+		`
+apiVersion: apps/v1`,
+		`
+apiVersion: apps/v1
+kind: Deployment # clear this field`,
+		`
+apiVersion: apps/v1`, nil},
+
+	//
+	// Test Case
+	//
+	// TODO(#36): consider making this an error
+	{`Implicitly clear a changed field`,
+		`
+apiVersion: apps/v1
+kind: Deployment`,
+		`
+apiVersion: apps/v1`,
+		`
+apiVersion: apps/v1
+kind: StatefulSet`,
+		`
+apiVersion: apps/v1`, nil},
+
+	//
+	// Test Case
+	//
+	{`Merge an empty scalar value`,
+		`
+apiVersion: apps/v1
+`,
+		`
+apiVersion: apps/v1
+kind: {}
+`,
+		`
+apiVersion: apps/v1
+`,
+		`
+apiVersion: apps/v1
+kind: {}
+`, nil},
+}

kyaml/yaml/merge3/visitor.go

@@ -0,0 +1,161 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package merge3
+
+import (
+	"sigs.k8s.io/kustomize/kyaml/yaml"
+	"sigs.k8s.io/kustomize/kyaml/yaml/walk"
+)
+
+type ConflictStrategy uint
+
+const (
+	// TODO: Support more strategies
+	TakeUpdate ConflictStrategy = 1 + iota
+)
+
+type Visitor struct{}
+
+func (m Visitor) VisitMap(nodes walk.Sources) (*yaml.RNode, error) {
+	if yaml.IsNull(nodes.Updated()) || yaml.IsNull(nodes.Dest()) {
+		// explicitly cleared from either dest or update
+		return walk.ClearNode, nil
+	}
+	if nodes.Dest() == nil && nodes.Updated() == nil {
+		// implicitly cleared missing from both dest and update
+		return walk.ClearNode, nil
+	}
+
+	if nodes.Dest() == nil {
+		// not cleared, but missing from the dest
+		// initialize a new value that can be recursively merged
+		return yaml.NewRNode(&yaml.Node{Kind: yaml.MappingNode}), nil
+	}
+	// recursively merge the dest with the original and updated
+	return nodes.Dest(), nil
+}
+
+func (m Visitor) visitAList(nodes walk.Sources) (*yaml.RNode, error) {
+	if yaml.IsEmpty(nodes.Updated()) && !yaml.IsEmpty(nodes.Origin()) {
+		// implicitly cleared from update -- element was deleted
+		return walk.ClearNode, nil
+	}
+	if yaml.IsEmpty(nodes.Dest()) {
+		// not cleared, but missing from the dest
+		// initialize a new value that can be recursively merged
+		return yaml.NewRNode(&yaml.Node{Kind: yaml.SequenceNode}), nil
+	}
+
+	// recursively merge the dest with the original and updated
+	return nodes.Dest(), nil
+}
+
+func (m Visitor) VisitScalar(nodes walk.Sources) (*yaml.RNode, error) {
+	if yaml.IsNull(nodes.Updated()) || yaml.IsNull(nodes.Dest()) {
+		// explicitly cleared from either dest or update
+		return nil, nil
+	}
+	if yaml.IsEmpty(nodes.Updated()) != yaml.IsEmpty(nodes.Origin()) {
+		// value added or removed in update
+		return nodes.Updated(), nil
+	}
+	if yaml.IsEmpty(nodes.Updated()) && yaml.IsEmpty(nodes.Origin()) {
+		// value added or removed in update
+		return nodes.Dest(), nil
+	}
+
+	if nodes.Updated().YNode().Value != nodes.Origin().YNode().Value {
+		// value changed in update
+		return nodes.Updated(), nil
+	}
+
+	// unchanged between origin and update, keep the dest
+	return nodes.Dest(), nil
+}
+
+func (m Visitor) visitNAList(nodes walk.Sources) (*yaml.RNode, error) {
+	if yaml.IsNull(nodes.Updated()) || yaml.IsNull(nodes.Dest()) {
+		// explicitly cleared from either dest or update
+		return walk.ClearNode, nil
+	}
+
+	if yaml.IsEmpty(nodes.Updated()) != yaml.IsEmpty(nodes.Origin()) {
+		// value added or removed in update
+		return nodes.Updated(), nil
+	}
+	if yaml.IsEmpty(nodes.Updated()) && yaml.IsEmpty(nodes.Origin()) {
+		// value not present in source or dest
+		return nodes.Dest(), nil
+	}
+
+	// compare origin and update values to see if they have changed
+	values, err := m.getStrValues(nodes)
+	if err != nil {
+		return nil, err
+	}
+	if values.Update != values.Origin {
+		// value changed in update
+		return nodes.Updated(), nil
+	}
+
+	// unchanged between origin and update, keep the dest
+	return nodes.Dest(), nil
+}
+
+func (m Visitor) VisitList(nodes walk.Sources, kind walk.ListKind) (*yaml.RNode, error) {
+	if kind == walk.AssociativeList {
+		return m.visitAList(nodes)
+	}
+	// non-associative list
+	return m.visitNAList(nodes)
+}
+
+func (m Visitor) getStrValues(nodes walk.Sources) (strValues, error) {
+	var uStr, oStr, dStr string
+	var err error
+	if nodes.Updated() != nil && nodes.Updated().YNode() != nil {
+		s := nodes.Updated().YNode().Style
+		defer func() {
+			nodes.Updated().YNode().Style = s
+		}()
+		nodes.Updated().YNode().Style = yaml.FlowStyle | yaml.SingleQuotedStyle
+		uStr, err = nodes.Updated().String()
+		if err != nil {
+			return strValues{}, err
+		}
+	}
+	if nodes.Origin() != nil && nodes.Origin().YNode() != nil {
+		s := nodes.Origin().YNode().Style
+		defer func() {
+			nodes.Origin().YNode().Style = s
+		}()
+		nodes.Origin().YNode().Style = yaml.FlowStyle | yaml.SingleQuotedStyle
+		oStr, err = nodes.Origin().String()
+		if err != nil {
+			return strValues{}, err
+		}
+
+	}
+	if nodes.Dest() != nil && nodes.Dest().YNode() != nil {
+		s := nodes.Dest().YNode().Style
+		defer func() {
+			nodes.Dest().YNode().Style = s
+		}()
+		nodes.Dest().YNode().Style = yaml.FlowStyle | yaml.SingleQuotedStyle
+		dStr, err = nodes.Dest().String()
+		if err != nil {
+			return strValues{}, err
+		}
+	}
+
+	return strValues{Origin: oStr, Update: uStr, Dest: dStr}, nil
+}
+
+type strValues struct {
+	Origin string
+	Update string
+	Dest   string
+}
+
+var _ walk.Visitor = Visitor{}

kyaml/yaml/order.go

@@ -0,0 +1,108 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package yaml
+
+// fieldSortOrder contains the relative ordering of fields when formatting an
+// object.
+var fieldSortOrder = []string{
+	// top-level metadata
+	"name", "generateName", "namespace", "clusterName",
+	"apiVersion", "kind", "metadata", "type",
+	"labels", "annotations",
+	"spec", "status",
+
+	// secret and configmap
+	"stringData", "data", "binaryData",
+
+	//cronjobspec,  daemonsetspec, deploymentspec, statefulsetspec,
+	// jobspec fields
+	"parallelism", "completions", "activeDeadlineSeconds", "backoffLimit",
+	"replicas", "selector", "manualSelector", "template",
+	"ttlSecondsAfterFinished", "volumeClaimTemplates", "service", "serviceName",
+	"podManagementPolicy", "updateStrategy", "strategy", "minReadySeconds",
+	"revision", "revisionHistoryLimit", "paused", "progressDeadlineSeconds",
+
+	// podspec
+	// podspec scalars
+	"restartPolicy", "terminationGracePeriodSeconds",
+	"activeDeadlineSeconds", "dnsPolicy", "serviceAccountName",
+	"serviceAccount", "automountServiceAccountToken", "nodeName",
+	"hostNetwork", "hostPID", "hostIPC", "shareProcessNamespace", "hostname",
+	"subdomain", "schedulerName", "priorityClassName", "priority",
+	"runtimeClassName", "enableServiceLinks",
+
+	// podspec lists and maps
+	"nodeSelector", "hostAliases",
+
+	// podspec objects
+	"initContainers", "containers", "volumes", "securityContext",
+	"imagePullSecrets", "affinity", "tolerations", "dnsConfig",
+	"readinessGates",
+
+	// containers
+	"image", "command", "args", "workingDir", "ports", "envFrom", "env",
+	"resources", "volumeMounts", "volumeDevices", "livenessProbe",
+	"readinessProbe", "lifecycle", "terminationMessagePath",
+	"terminationMessagePolicy", "imagePullPolicy", "securityContext",
+	"stdin", "stdinOnce", "tty",
+
+	// service
+	"clusterIP", "externalIPs", "loadBalancerIP", "loadBalancerSourceRanges",
+	"externalName", "externalTrafficPolicy", "sessionAffinity",
+
+	// ports
+	"protocol", "port", "targetPort", "hostPort", "containerPort", "hostIP",
+
+	// volumemount
+	"readOnly", "mountPath", "subPath", "subPathExpr", "mountPropagation",
+
+	// envvar + envvarsource
+	"value", "valueFrom", "fieldRef", "resourceFieldRef", "configMapKeyRef",
+	"secretKeyRef", "prefix", "configMapRef", "secretRef",
+}
+
+type set map[string]interface{}
+
+func newSet(values ...string) set {
+	m := map[string]interface{}{}
+	for _, value := range values {
+		m[value] = nil
+	}
+	return m
+}
+
+func (s set) Has(key string) bool {
+	_, found := s[key]
+	return found
+}
+
+// whitelistedListSortKinds contains the set of kinds that are whitelisted
+// for sorting list field elements
+var WhitelistedListSortKinds = newSet(
+	"CronJob", "DaemonSet", "Deployment", "Job", "ReplicaSet", "StatefulSet",
+	"ValidatingWebhookConfiguration")
+
+// whitelistedListSortApis contains the set of apis that are whitelisted for
+// sorting list field elements
+var WhitelistedListSortApis = newSet(
+	"apps/v1", "apps/v1beta1", "apps/v1beta2", "batch/v1", "batch/v1beta1",
+	"extensions/v1beta1", "v1", "admissionregistration.k8s.io/v1beta1")
+
+// whitelistedListSortFields contains json paths to list fields that should
+// be sorted, and the field they should be sorted by
+var WhitelistedListSortFields = map[string]string{
+	".spec.template.spec.containers": "name",
+	".webhooks.rules.operations":     "",
+}
+
+// FieldOrder indexes fields and maps them to relative precedence
+var FieldOrder map[string]int
+
+func init() {
+	// create an index of field orderings
+	FieldOrder = map[string]int{}
+	for i, f := range fieldSortOrder {
+		FieldOrder[f] = i + 1
+	}
+}

kyaml/yaml/types.go

@@ -0,0 +1,534 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package yaml
+
+import (
+	"bytes"
+	"errors"
+	"reflect"
+	"strings"
+
+	"gopkg.in/yaml.v3"
+	"sigs.k8s.io/kustomize/kyaml/sets"
+)
+
+const (
+	// NullNodeTag is the tag set for a yaml.Document that contains no data -- e.g. it isn't a
+	// Map, Slice, Document, etc
+	NullNodeTag = "!!null"
+)
+
+func NullNode() *RNode {
+	return NewRNode(&Node{Tag: NullNodeTag})
+}
+
+func IsMissingOrNull(node *RNode) bool {
+	if node == nil || node.YNode() == nil || node.YNode().Tag == NullNodeTag {
+		return true
+	}
+	return false
+}
+
+func IsEmpty(node *RNode) bool {
+	if node == nil || node.YNode() == nil || node.YNode().Tag == NullNodeTag {
+		return true
+	}
+
+	if node.YNode().Kind == yaml.MappingNode && len(node.YNode().Content) == 0 {
+		return true
+	}
+	if node.YNode().Kind == yaml.SequenceNode && len(node.YNode().Content) == 0 {
+		return true
+	}
+
+	return false
+}
+
+func IsNull(node *RNode) bool {
+	return node != nil && node.YNode() != nil && node.YNode().Tag == NullNodeTag
+}
+
+func IsFieldEmpty(node *MapNode) bool {
+	if node == nil || node.Value == nil || node.Value.YNode() == nil ||
+		node.Value.YNode().Tag == NullNodeTag {
+		return true
+	}
+
+	if node.Value.YNode().Kind == yaml.MappingNode && len(node.Value.YNode().Content) == 0 {
+		return true
+	}
+	if node.Value.YNode().Kind == yaml.SequenceNode && len(node.Value.YNode().Content) == 0 {
+		return true
+	}
+
+	return false
+}
+
+func IsFieldNull(node *MapNode) bool {
+	return node != nil && node.Value != nil && node.Value.YNode() != nil &&
+		node.Value.YNode().Tag == NullNodeTag
+}
+
+// Parser parses values into configuration.
+type Parser struct {
+	Kind  string `yaml:"kind,omitempty"`
+	Value string `yaml:"value,omitempty"`
+}
+
+func (p Parser) Filter(_ *RNode) (*RNode, error) {
+	d := yaml.NewDecoder(bytes.NewBuffer([]byte(p.Value)))
+	o := &RNode{value: &yaml.Node{}}
+	return o, d.Decode(o.value)
+}
+
+// Parse parses a yaml string into an *RNode
+func Parse(value string) (*RNode, error) {
+	return Parser{Value: value}.Filter(nil)
+}
+
+// TODO(pwittrock): test this
+func GetStyle(styles ...string) Style {
+	var style Style
+	for _, s := range styles {
+		switch s {
+		case "TaggedStyle":
+			style |= TaggedStyle
+		case "DoubleQuotedStyle":
+			style |= DoubleQuotedStyle
+		case "SingleQuotedStyle":
+			style |= SingleQuotedStyle
+		case "LiteralStyle":
+			style |= LiteralStyle
+		case "FoldedStyle":
+			style |= FoldedStyle
+		case "FlowStyle":
+			style |= FlowStyle
+		}
+	}
+	return style
+}
+
+// MustParse parses a yaml string into an *RNode and panics if there is an error
+func MustParse(value string) *RNode {
+	v, err := Parser{Value: value}.Filter(nil)
+	if err != nil {
+		panic(err)
+	}
+	return v
+}
+
+// NewScalarRNode returns a new Scalar *RNode containing the provided value.
+func NewScalarRNode(value string) *RNode {
+	return &RNode{
+		value: &yaml.Node{
+			Kind:  yaml.ScalarNode,
+			Value: value,
+		}}
+}
+
+// NewListRNode returns a new List *RNode containing the provided value.
+func NewListRNode(values ...string) *RNode {
+	seq := &RNode{value: &yaml.Node{Kind: yaml.SequenceNode}}
+	for _, v := range values {
+		seq.value.Content = append(seq.value.Content, &yaml.Node{
+			Kind:  yaml.ScalarNode,
+			Value: v,
+		})
+	}
+	return seq
+}
+
+// NewRNode returns a new *RNode containing the provided value.
+func NewRNode(value *yaml.Node) *RNode {
+	if value != nil {
+		value.Style = 0
+	}
+	return &RNode{value: value}
+}
+
+// GrepFilter may modify or walk the RNode.
+// When possible, Filters should be serializable to yaml so that they can be described
+// declaratively as data.
+//
+// Analogous to http://www.linfo.org/filters.html
+type Filter interface {
+	Filter(object *RNode) (*RNode, error)
+}
+
+type FilterFunc func(object *RNode) (*RNode, error)
+
+func (f FilterFunc) Filter(object *RNode) (*RNode, error) {
+	return f(object)
+}
+
+// RNode provides functions for manipulating Kubernetes Resources
+// Objects unmarshalled into *yaml.Nodes
+type RNode struct {
+	// fieldPath contains the path from the root of the KubernetesObject to
+	// this field.
+	// Only field names are captured in the path.
+	// e.g. a image field in a Deployment would be
+	// 'spec.template.spec.containers.image'
+	fieldPath []string
+
+	// FieldValue contains the value.
+	// FieldValue is always set:
+	// field: field value
+	// list entry: list entry value
+	// object root: object root
+	value *yaml.Node
+
+	Match []string
+}
+
+// MapNode wraps a field key and value.
+type MapNode struct {
+	Key   *RNode
+	Value *RNode
+}
+
+type MapNodeSlice []*MapNode
+
+func (m MapNodeSlice) Keys() []*RNode {
+	var keys []*RNode
+	for i := range m {
+		if m[i] != nil {
+			keys = append(keys, m[i].Key)
+		}
+	}
+	return keys
+}
+
+func (m MapNodeSlice) Values() []*RNode {
+	var values []*RNode
+	for i := range m {
+		if m[i] != nil {
+			values = append(values, m[i].Value)
+		} else {
+			values = append(values, nil)
+		}
+	}
+	return values
+}
+
+// ResourceMeta contains the metadata for a Resource.
+type ResourceMeta struct {
+	ApiVersion string `yaml:"apiVersion,omitempty"`
+	Kind       string `yaml:"kind,omitempty"`
+	ObjectMeta `yaml:"metadata,omitempty"`
+}
+
+func NewResourceMeta(name string, typeMeta ResourceMeta) ResourceMeta {
+	return ResourceMeta{
+		Kind:       typeMeta.Kind,
+		ApiVersion: typeMeta.ApiVersion,
+		ObjectMeta: ObjectMeta{Name: name},
+	}
+}
+
+type ObjectMeta struct {
+	Name        string            `yaml:"name,omitempty"`
+	Namespace   string            `yaml:"namespace,omitempty"`
+	Labels      map[string]string `yaml:"labels,omitempty"`
+	Annotations map[string]string `yaml:"annotations,omitempty"`
+}
+
+var MissingMetaError = errors.New("missing Resource metadata")
+
+func (rn *RNode) GetMeta() (ResourceMeta, error) {
+	m := ResourceMeta{}
+	b := &bytes.Buffer{}
+	e := NewEncoder(b)
+	if err := e.Encode(rn.YNode()); err != nil {
+		return m, err
+	}
+	if err := e.Close(); err != nil {
+		return m, err
+	}
+	d := yaml.NewDecoder(b)
+	d.KnownFields(false) // only want to parse the metadata
+	if err := d.Decode(&m); err != nil {
+		return m, err
+	}
+	if reflect.DeepEqual(m, ResourceMeta{}) {
+		return m, MissingMetaError
+	}
+	return m, nil
+}
+
+// Pipe sequentially invokes each GrepFilter, and passes the result to the next
+// GrepFilter.
+//
+// Analogous to http://www.linfo.org/pipes.html
+//
+// * rn is provided as input to the first GrepFilter.
+// * if any GrepFilter returns an error, immediately return the error
+// * if any GrepFilter returns a nil RNode, immediately return nil, nil
+// * if all Filters succeed with non-empty results, return the final result
+func (rn *RNode) Pipe(functions ...Filter) (*RNode, error) {
+	// check if rn is nil to make chaining Pipe calls easier
+	if rn == nil {
+		return nil, nil
+	}
+
+	var err error
+	var v *RNode
+	if rn.value != nil && rn.value.Kind == yaml.DocumentNode {
+		// the first node may be a DocumentNode containing a single MappingNode
+		v = &RNode{value: rn.value.Content[0]}
+	} else {
+		v = rn
+	}
+
+	// return each fn in sequence until encountering an error or missing value
+	for _, c := range functions {
+		v, err = c.Filter(v)
+		if err != nil || v == nil {
+			return v, err
+		}
+	}
+	return v, err
+}
+
+// Document returns the Node RNode for the value.  Does not unwrap the node if it is a
+// DocumentNodes
+func (rn *RNode) Document() *yaml.Node {
+	return rn.value
+}
+
+// YNode returns the yaml.Node value.  If the yaml.Node value is a DocumentNode,
+// YNode will return the DocumentNode Content entry instead of the DocumentNode.
+func (rn *RNode) YNode() *yaml.Node {
+	if rn == nil || rn.value == nil {
+		return nil
+	}
+	if rn.value.Kind == yaml.DocumentNode {
+		return rn.value.Content[0]
+	}
+	return rn.value
+}
+
+// SetYNode sets the yaml.Node value.
+func (rn *RNode) SetYNode(node *yaml.Node) {
+	if rn.value == nil || node == nil {
+		rn.value = node
+		return
+	}
+	*rn.value = *node
+}
+
+// SetYNode sets the value on a Document.
+func (rn *RNode) AppendToFieldPath(parts ...string) {
+	rn.fieldPath = append(rn.fieldPath, parts...)
+}
+
+// FieldPath returns the field path from the object root to rn.  Does not include list indexes.
+func (rn *RNode) FieldPath() []string {
+	return rn.fieldPath
+}
+
+const (
+	Trim = "Trim"
+	Flow = "Flow"
+)
+
+func String(node *yaml.Node, opts ...string) (string, error) {
+	if node == nil {
+		return "", nil
+	}
+	optsSet := sets.String{}
+	optsSet.Insert(opts...)
+	if optsSet.Has(Flow) {
+		oldStyle := node.Style
+		defer func() {
+			node.Style = oldStyle
+		}()
+		node.Style = yaml.FlowStyle
+	}
+
+	b := &bytes.Buffer{}
+	e := NewEncoder(b)
+	err := e.Encode(node)
+	e.Close()
+	val := b.String()
+	if optsSet.Has(Trim) {
+		val = strings.TrimSpace(val)
+	}
+	return val, err
+}
+
+// NewScalarRNode returns the yaml NewScalarRNode representation of the RNode value.
+func (rn *RNode) String() (string, error) {
+	if rn == nil {
+		return "", nil
+	}
+	return String(rn.value)
+}
+
+func (rn *RNode) MustString() string {
+	s, err := rn.String()
+	if err != nil {
+		panic(err)
+	}
+	return s
+}
+
+// Content returns the value node's Content field.
+func (rn *RNode) Content() []*yaml.Node {
+	if rn == nil {
+		return nil
+	}
+	return rn.YNode().Content
+}
+
+// Fields returns the list of fields for a ResourceNode containing a MappingNode
+// value.
+func (rn *RNode) Fields() ([]string, error) {
+	if err := ErrorIfInvalid(rn, yaml.MappingNode); err != nil {
+		return nil, err
+	}
+	var fields []string
+	for i := 0; i < len(rn.Content()); i += 2 {
+		fields = append(fields, rn.Content()[i].Value)
+	}
+	return fields, nil
+}
+
+// Field returns the fieldName, fieldValue pair for MappingNodes.  Returns nil for non-MappingNodes.
+func (rn *RNode) Field(field string) *MapNode {
+	if rn.YNode().Kind != yaml.MappingNode {
+		return nil
+	}
+	for i := 0; i < len(rn.Content()); IncrementFieldIndex(&i) {
+		isMatchingField := rn.Content()[i].Value == field
+		if isMatchingField {
+			return &MapNode{Key: NewRNode(rn.Content()[i]), Value: NewRNode(rn.Content()[i+1])}
+		}
+	}
+	return nil
+}
+
+// VisitFields calls fn for each field in rn.
+func (rn *RNode) VisitFields(fn func(node *MapNode) error) error {
+	// get the list of srcFieldNames
+	srcFieldNames, err := rn.Fields()
+	if err != nil {
+		return err
+	}
+
+	// visit each field
+	for _, fieldName := range srcFieldNames {
+		if err := fn(rn.Field(fieldName)); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// Elements returns a list of elements for a ResourceNode containing a
+// SequenceNode value.
+func (rn *RNode) Elements() ([]*RNode, error) {
+	if err := ErrorIfInvalid(rn, yaml.SequenceNode); err != nil {
+		return nil, err
+	}
+	var elements []*RNode
+	for i := 0; i < len(rn.Content()); i += 1 {
+		elements = append(elements, NewRNode(rn.Content()[i]))
+	}
+	return elements, nil
+}
+
+func (rn *RNode) ElementValues(key string) ([]string, error) {
+	if err := ErrorIfInvalid(rn, yaml.SequenceNode); err != nil {
+		return nil, err
+	}
+	var elements []string
+	for i := 0; i < len(rn.Content()); i += 1 {
+		field := NewRNode(rn.Content()[i]).Field(key)
+		if !IsFieldEmpty(field) {
+			elements = append(elements, field.Value.YNode().Value)
+		}
+	}
+	return elements, nil
+}
+
+// Element returns the element in the list which contains the field matching the value.
+// Returns nil for non-SequenceNodes
+func (rn *RNode) Element(key, value string) *RNode {
+	if rn.YNode().Kind != yaml.SequenceNode {
+		return nil
+	}
+	elem, err := rn.Pipe(MatchElement(key, value))
+	if err != nil {
+		return nil
+	}
+	return elem
+}
+
+// VisitElements calls fn for each element in the list.
+func (rn *RNode) VisitElements(fn func(node *RNode) error) error {
+	elements, err := rn.Elements()
+	if err != nil {
+		return err
+	}
+
+	for i := range elements {
+		if err := fn(elements[i]); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// AssociativeSequencePaths is a map of paths to sequences that have associative keys.
+// The order sets the precedence of the merge keys -- if multiple keys are present
+// in the list, then the FIRST key which ALL elements have is used as the
+// associative key.
+var AssociativeSequenceKeys = []string{
+	"mountPath", "devicePath", "ip", "type", "topologyKey", "name", "containerPort",
+}
+
+func IsAssociative(nodes []*RNode) bool {
+	for i := range nodes {
+		node := nodes[i]
+		if IsEmpty(node) {
+			continue
+		}
+		if node.IsAssociative() {
+			return true
+		}
+	}
+	return false
+}
+
+// IsAssociative returns true if the RNode is for an associative list.
+func (rn *RNode) IsAssociative() bool {
+	return rn.GetAssociativeKey() != ""
+}
+
+// GetAssociativeKey returns the associative key used to merge the list, or "" if the
+// list is not associative.
+func (rn *RNode) GetAssociativeKey() string {
+	// look for any associative keys in the first element
+	for _, key := range AssociativeSequenceKeys {
+		if checkKey(key, rn.Content()) {
+			return key
+		}
+	}
+
+	// element doesn't have an associative keys
+	return ""
+}
+
+// checkKey returns true if all elems have the key
+func checkKey(key string, elems []*Node) bool {
+	count := 0
+	for i := range elems {
+		elem := NewRNode(elems[i])
+		if elem.Field(key) != nil {
+			count++
+		}
+	}
+	return count == len(elems)
+}

kyaml/yaml/walk/associative_sequence.go

@@ -0,0 +1,128 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package walk
+
+import (
+	"strings"
+
+	"github.com/go-errors/errors"
+	"sigs.k8s.io/kustomize/kyaml/sets"
+	"sigs.k8s.io/kustomize/kyaml/yaml"
+)
+
+func (l *Walker) walkAssociativeSequence() (*yaml.RNode, error) {
+
+	// may require initializing the dest node
+	dest, err := l.Sources.setDestNode(l.VisitList(l.Sources, AssociativeList))
+	if dest == nil || err != nil {
+		return nil, err
+	}
+
+	// find the list of elements we need to recursively walk
+	key, err := l.elementKey()
+	if err != nil {
+		return nil, err
+	}
+	values, err := l.elementValues(key)
+	if err != nil {
+		return nil, err
+	}
+
+	// recursively set the elements in the list
+	for _, value := range values {
+		val, err := Walker{Visitor: l,
+			Sources: l.elementValue(key, value)}.Walk()
+		if err != nil {
+			return nil, err
+		}
+		if yaml.IsEmpty(val) {
+			_, err = dest.Pipe(yaml.ElementSetter{Key: key, Value: value})
+			if err != nil {
+				return nil, err
+			}
+			continue
+		}
+
+		if val.Field(key) == nil {
+			// make sure the key is set on the field
+			_, err = val.Pipe(yaml.SetField(key, yaml.NewScalarRNode(value)))
+			if err != nil {
+				return nil, err
+			}
+		}
+
+		// this handles empty and non-empty values
+		_, err = dest.Pipe(yaml.ElementSetter{Element: val.YNode(), Key: key, Value: value})
+		if err != nil {
+			return nil, err
+		}
+	}
+	// field is empty
+	if yaml.IsEmpty(dest) {
+		return nil, nil
+	}
+	return dest, nil
+}
+
+// elementKey returns the merge key to use for the associative list
+func (l Walker) elementKey() (string, error) {
+	var key string
+	for i := range l.Sources {
+		if l.Sources[i] != nil && len(l.Sources[i].Content()) > 0 {
+			newKey := l.Sources[i].GetAssociativeKey()
+			if key != "" && key != newKey {
+				return "", errors.Errorf(
+					"conflicting merge keys [%s,%s] for field %s",
+					key, newKey, strings.Join(l.Path, "."))
+			}
+			key = newKey
+		}
+	}
+	if key == "" {
+		return "", errors.Errorf("no merge key found for field %s",
+			strings.Join(l.Path, "."))
+	}
+	return key, nil
+}
+
+// elementValues returns a slice containing all values for the field across all elements
+// from all sources.
+// Return value slice is ordered using the original ordering from the elements, where
+// elements missing from earlier sources appear later.
+func (l Walker) elementValues(key string) ([]string, error) {
+	// use slice to to keep elements in the original order
+	// dest node must be first
+	var returnValues []string
+	seen := sets.String{}
+	for i := range l.Sources {
+		if l.Sources[i] == nil {
+			continue
+		}
+
+		// add the value of the field for each element
+		// don't check error, we know this is a list node
+		values, _ := l.Sources[i].ElementValues(key)
+		for _, s := range values {
+			if seen.Has(s) {
+				continue
+			}
+			returnValues = append(returnValues, s)
+			seen.Insert(s)
+		}
+	}
+	return returnValues, nil
+}
+
+// fieldValue returns a slice containing each source's value for fieldName
+func (l Walker) elementValue(key, value string) []*yaml.RNode {
+	var fields []*yaml.RNode
+	for i := range l.Sources {
+		if l.Sources[i] == nil {
+			fields = append(fields, nil)
+			continue
+		}
+		fields = append(fields, l.Sources[i].Element(key, value))
+	}
+	return fields
+}

kyaml/yaml/walk/map.go

@@ -0,0 +1,81 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package walk
+
+import (
+	"sort"
+
+	"sigs.k8s.io/kustomize/kyaml/sets"
+	"sigs.k8s.io/kustomize/kyaml/yaml"
+)
+
+// walkMap returns the value of VisitMap
+//
+// - call VisitMap
+// - set the return value on l.Dest
+// - walk each source field
+// - set each source field value on l.Dest
+func (l Walker) walkMap() (*yaml.RNode, error) {
+	// get the new map value
+	dest, err := l.Sources.setDestNode(l.VisitMap(l.Sources))
+	if dest == nil || err != nil {
+		return nil, err
+	}
+
+	// recursively set the field values on the map
+	for _, key := range l.fieldNames() {
+		val, err := Walker{Visitor: l,
+			Sources: l.fieldValue(key), Path: append(l.Path, key)}.Walk()
+		if err != nil {
+			return nil, err
+		}
+
+		// this handles empty and non-empty values
+		_, err = dest.Pipe(yaml.FieldSetter{Name: key, Value: val})
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return dest, nil
+}
+
+// valueIfPresent returns node.Value if node is non-nil, otherwise returns nil
+func (l Walker) valueIfPresent(node *yaml.MapNode) *yaml.RNode {
+	if node == nil {
+		return nil
+	}
+	return node.Value
+}
+
+// fieldNames returns a sorted slice containing the names of all fields that appear in any of
+// the sources
+func (l Walker) fieldNames() []string {
+	fields := sets.String{}
+	for _, s := range l.Sources {
+		if s == nil {
+			continue
+		}
+		// don't check error, we know this is a mapping node
+		sFields, _ := s.Fields()
+		fields.Insert(sFields...)
+	}
+	result := fields.List()
+	sort.Strings(result)
+	return result
+}
+
+// fieldValue returns a slice containing each source's value for fieldName
+func (l Walker) fieldValue(fieldName string) []*yaml.RNode {
+	var fields []*yaml.RNode
+	for i := range l.Sources {
+		if l.Sources[i] == nil {
+			fields = append(fields, nil)
+			continue
+		}
+		field := l.Sources[i].Field(fieldName)
+		fields = append(fields, l.valueIfPresent(field))
+	}
+	return fields
+}

kyaml/yaml/walk/nonassociative_sequence.go

@@ -0,0 +1,13 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package walk
+
+import (
+	"sigs.k8s.io/kustomize/kyaml/yaml"
+)
+
+// walkNonAssociativeSequence returns the value of VisitList
+func (l Walker) walkNonAssociativeSequence() (*yaml.RNode, error) {
+	return l.VisitList(l.Sources, NonAssociateList)
+}

kyaml/yaml/walk/scalar.go

@@ -0,0 +1,11 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package walk
+
+import "sigs.k8s.io/kustomize/kyaml/yaml"
+
+// walkScalar returns the value of VisitScalar
+func (l Walker) walkScalar() (*yaml.RNode, error) {
+	return l.VisitScalar(l.Sources)
+}

kyaml/yaml/walk/visitor.go

@@ -0,0 +1,38 @@
+// Copyright 2019 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package walk
+
+import (
+	"sigs.k8s.io/kustomize/kyaml/yaml"
+)
+
+type ListKind int32
+
+const (
+	AssociativeList ListKind = 1 + iota
+	NonAssociateList
+)
+
+// Visitor is invoked by walk with source and destination node pairs
+type Visitor interface {
+	VisitMap(nodes Sources) (*yaml.RNode, error)
+
+	VisitScalar(nodes Sources) (*yaml.RNode, error)
+
+	VisitList(nodes Sources, kind ListKind) (*yaml.RNode, error)
+}
+
+// NoOp is returned if GrepFilter should do nothing after calling Set
+var ClearNode *yaml.RNode = nil

kyaml/yaml/walk/walk.go

@@ -0,0 +1,144 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package walk
+
+import (
+	"fmt"
+	"os"
+	"strings"
+
+	"sigs.k8s.io/kustomize/kyaml/yaml"
+)
+
+// Filter walks the Source RNode and modifies the RNode provided to GrepFilter.
+type Walker struct {
+	// Visitor is invoked by GrepFilter
+	Visitor
+
+	// Source is the RNode to walk.  All Source fields and associative list elements
+	// will be visited.
+	Sources Sources
+
+	// Path is the field path to the current Source Node.
+	Path []string
+}
+
+func (l Walker) Kind() yaml.Kind {
+	for _, s := range l.Sources {
+		if !yaml.IsEmpty(s) {
+			return s.YNode().Kind
+		}
+	}
+	return 0
+}
+
+// GrepFilter implements yaml.GrepFilter
+func (l Walker) Walk() (*yaml.RNode, error) {
+	// invoke the handler for the corresponding node type
+	switch l.Kind() {
+	case yaml.MappingNode:
+		if err := yaml.ErrorIfAnyInvalidAndNonNull(yaml.MappingNode, l.Sources...); err != nil {
+			return nil, err
+		}
+		return l.walkMap()
+	case yaml.SequenceNode:
+		if err := yaml.ErrorIfAnyInvalidAndNonNull(yaml.SequenceNode, l.Sources...); err != nil {
+			return nil, err
+		}
+		if yaml.IsAssociative(l.Sources) {
+			return l.walkAssociativeSequence()
+		} else {
+			return l.walkNonAssociativeSequence()
+		}
+	case yaml.ScalarNode:
+		if err := yaml.ErrorIfAnyInvalidAndNonNull(yaml.ScalarNode, l.Sources...); err != nil {
+			return nil, err
+		}
+		return l.walkScalar()
+	case 0:
+		// walk empty nodes as maps
+		return l.walkMap()
+	default:
+		return nil, nil
+	}
+}
+
+const (
+	DestIndex = iota
+	OriginIndex
+	UpdatedIndex
+)
+
+type Sources []*yaml.RNode
+
+// Dest returns the destination node
+func (s Sources) Dest() *yaml.RNode {
+	if len(s) <= DestIndex {
+		return nil
+	}
+	return s[DestIndex]
+}
+
+// Origin returns the origin node
+func (s Sources) Origin() *yaml.RNode {
+	if len(s) <= OriginIndex {
+		return nil
+	}
+	return s[OriginIndex]
+}
+
+// Updated returns the updated node
+func (s Sources) Updated() *yaml.RNode {
+	if len(s) <= UpdatedIndex {
+		return nil
+	}
+	return s[UpdatedIndex]
+}
+
+func (s Sources) String() string {
+	var values []string
+	for i := range s {
+		str, err := s[i].String()
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "%v\n", err)
+		}
+		values = append(values, str)
+	}
+	return strings.Join(values, "\n")
+}
+
+// setDestNode sets the destination source node
+func (s Sources) setDestNode(node *yaml.RNode, err error) (*yaml.RNode, error) {
+	if err != nil {
+		return nil, err
+	}
+	s[0] = node
+	return node, nil
+}
+
+type FieldSources []*yaml.MapNode
+
+// Dest returns the destination node
+func (s FieldSources) Dest() *yaml.MapNode {
+	if len(s) <= DestIndex {
+		return nil
+	}
+	return s[DestIndex]
+}
+
+// Origin returns the origin node
+func (s FieldSources) Origin() *yaml.MapNode {
+	if len(s) <= OriginIndex {
+		return nil
+	}
+	return s[OriginIndex]
+}
+
+// Updated returns the updated node
+func (s FieldSources) Updated() *yaml.MapNode {
+	if len(s) <= UpdatedIndex {
+		return nil
+	}
+	return s[UpdatedIndex]
+}

travis/kyaml-pre-commit.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+set -e
+
+cd kyaml
+make all