kyaml: initial support for yaml and resource manipulation

kyaml/kio/byteio_reader.go

@@ -0,0 +1,198 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package kio
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"sort"
+	"strings"
+
+	"sigs.k8s.io/kustomize/kyaml/kio/kioutil"
+	"sigs.k8s.io/kustomize/kyaml/yaml"
+)
+
+const (
+	ResourceListKind       = "ResourceList"
+	ResourceListApiVersion = "kyaml.kustomize.dev/v1alpha1"
+)
+
+// ByteReadWriter reads from an input and writes to an output
+type ByteReadWriter struct {
+	// Reader is where ResourceNodes are decoded from.
+	Reader io.Reader
+
+	// Writer is where ResourceNodes are encoded.
+	Writer io.Writer
+
+	// OmitReaderAnnotations will configures Read to skip setting the kyaml.kustomize.dev/kio/index
+	// annotation on Resources as they are Read.
+	OmitReaderAnnotations bool
+
+	// KeepReaderAnnotations if set will keep the Reader specific annotations when writing
+	// the Resources, otherwise they will be cleared.
+	KeepReaderAnnotations bool
+
+	// Style is a style that is set on the Resource Node Document.
+	Style yaml.Style
+
+	FunctionConfig *yaml.RNode
+
+	WrappingApiVersion string
+	WrappingKind       string
+}
+
+func (rw *ByteReadWriter) Read() ([]*yaml.RNode, error) {
+	b := &ByteReader{
+		Reader:                rw.Reader,
+		OmitReaderAnnotations: rw.OmitReaderAnnotations,
+	}
+	val, err := b.Read()
+	rw.FunctionConfig = b.FunctionConfig
+	rw.WrappingApiVersion = b.WrappingApiVersion
+	rw.WrappingKind = b.WrappingKind
+	return val, err
+}
+
+func (rw *ByteReadWriter) Write(nodes []*yaml.RNode) error {
+	return ByteWriter{
+		Writer:                rw.Writer,
+		KeepReaderAnnotations: rw.KeepReaderAnnotations,
+		Style:                 rw.Style,
+		FunctionConfig:        rw.FunctionConfig,
+		WrappingApiVersion:    rw.WrappingApiVersion,
+		WrappingKind:          rw.WrappingKind,
+	}.Write(nodes)
+}
+
+// ByteReader decodes ResourceNodes from bytes.
+// By default, Read will set the kyaml.kustomize.dev/kio/index annotation on each RNode as it
+// is read so they can be written back in the same order.
+type ByteReader struct {
+	// Reader is where ResourceNodes are decoded from.
+	Reader io.Reader
+
+	// OmitReaderAnnotations will configures Read to skip setting the kyaml.kustomize.dev/kio/index
+	// annotation on Resources as they are Read.
+	OmitReaderAnnotations bool
+
+	// SetAnnotations is a map of caller specified annotations to set on resources as they are read
+	// These are independent of the annotations controlled by OmitReaderAnnotations
+	SetAnnotations map[string]string
+
+	FunctionConfig *yaml.RNode
+
+	WrappingApiVersion string
+	WrappingKind       string
+}
+
+var _ Reader = &ByteReader{}
+
+func (r *ByteReader) Read() ([]*yaml.RNode, error) {
+	output := ResourceNodeSlice{}
+
+	// by manually splitting resources -- otherwise the decoder will get the Resource
+	// boundaries wrong for header comments.
+	input := &bytes.Buffer{}
+	_, err := io.Copy(input, r.Reader)
+	if err != nil {
+		return nil, err
+	}
+	values := strings.Split(input.String(), "\n---\n")
+
+	index := 0
+	for i := range values {
+		decoder := yaml.NewDecoder(bytes.NewBufferString(values[i]))
+		node, err := r.decode(index, decoder)
+		if err == io.EOF {
+			continue
+		}
+		if err != nil {
+			return nil, err
+		}
+		if yaml.IsMissingOrNull(node) {
+			// empty value
+			continue
+		}
+
+		// ok if no metadata -- assume not an InputList
+		meta, _ := node.GetMeta()
+
+		// the elements are wrapped in an InputList, unwrap them
+		// don't check apiVersion, we haven't standardized on the domain
+		if (meta.Kind == ResourceListKind || meta.Kind == "List") &&
+			node.Field("items") != nil {
+			r.WrappingKind = meta.Kind
+			r.WrappingApiVersion = meta.ApiVersion
+
+			// unwrap the list
+			fc := node.Field("functionConfig")
+			if fc != nil {
+				r.FunctionConfig = fc.Value
+			}
+
+			items := node.Field("items")
+			if items != nil {
+				for i := range items.Value.Content() {
+					// add items
+					output = append(output, yaml.NewRNode(items.Value.Content()[i]))
+				}
+
+			}
+			continue
+		}
+
+		// add the node to the list
+		output = append(output, node)
+
+		// increment the index annotation value
+		index++
+	}
+	return output, nil
+}
+
+func isEmptyDocument(node *yaml.Node) bool {
+	// node is a Document with no content -- e.g. "---\n---"
+	return node.Kind == yaml.DocumentNode &&
+		node.Content[0].Tag == yaml.NullNodeTag
+}
+
+func (r *ByteReader) decode(index int, decoder *yaml.Decoder) (*yaml.RNode, error) {
+	node := &yaml.Node{}
+	err := decoder.Decode(node)
+	if err == io.EOF {
+		return nil, io.EOF
+	}
+	if err != nil {
+		return nil, err
+	}
+
+	if isEmptyDocument(node) {
+		return nil, nil
+	}
+
+	// set annotations on the read Resources
+	// sort the annotations by key so the output Resources is consistent (otherwise the
+	// annotations will be in a random order)
+	n := yaml.NewRNode(node)
+	if r.SetAnnotations == nil {
+		r.SetAnnotations = map[string]string{}
+	}
+	if !r.OmitReaderAnnotations {
+		r.SetAnnotations[kioutil.IndexAnnotation] = fmt.Sprintf("%d", index)
+	}
+	var keys []string
+	for k := range r.SetAnnotations {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+	for _, k := range keys {
+		_, err = n.Pipe(yaml.SetAnnotation(k, r.SetAnnotations[k]))
+		if err != nil {
+			return nil, err
+		}
+	}
+	return yaml.NewRNode(node), nil
+}

kyaml/kio/byteio_reader_test.go

@@ -0,0 +1,310 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package kio_test
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	. "sigs.k8s.io/kustomize/kyaml/kio"
+)
+
+// getByteReaderTestInput returns test input
+func getByteReaderTestInput(t *testing.T) *bytes.Buffer {
+	b := &bytes.Buffer{}
+	_, err := b.WriteString(`
+---
+a: b # first resource
+c: d
+---
+# second resource
+e: f
+g:
+- h
+---
+---
+i: j
+`)
+	if !assert.NoError(t, err) {
+		assert.FailNow(t, "")
+	}
+	return b
+}
+
+func TestByteReader_Read_wrappedResourceßßList(t *testing.T) {
+	r := &ByteReader{Reader: bytes.NewBufferString(`apiVersion: kyaml.kustomize.dev/v1alpha1
+kind: ResourceList
+functionConfig:
+  foo: bar
+  elems:
+  - a
+  - b
+  - c
+items:
+-  kind: Deployment
+   spec:
+     replicas: 1
+- kind: Service
+  spec:
+    selectors:
+      foo: bar
+`)}
+	nodes, err := r.Read()
+	if !assert.NoError(t, err) {
+		return
+	}
+
+	// verify the contents
+	if !assert.Len(t, nodes, 2) {
+		return
+	}
+	expected := []string{
+		`kind: Deployment
+spec:
+  replicas: 1
+`,
+		`kind: Service
+spec:
+  selectors:
+    foo: bar
+`,
+	}
+	for i := range nodes {
+		if !assert.Equal(t, expected[i], nodes[i].MustString()) {
+			return
+		}
+	}
+
+	// verify the function config
+	assert.Equal(t, `foo: bar
+elems:
+- a
+- b
+- c
+`, r.FunctionConfig.MustString())
+
+	assert.Equal(t, ResourceListKind, r.WrappingKind)
+	assert.Equal(t, ResourceListApiVersion, r.WrappingApiVersion)
+
+}
+
+func TestByteReader_Read_wrappedList(t *testing.T) {
+	r := &ByteReader{Reader: bytes.NewBufferString(`apiVersion: v1
+kind: List
+items:
+-  kind: Deployment
+   spec:
+     replicas: 1
+- kind: Service
+  spec:
+    selectors:
+      foo: bar
+`)}
+	nodes, err := r.Read()
+	if !assert.NoError(t, err) {
+		return
+	}
+
+	// verify the contents
+	if !assert.Len(t, nodes, 2) {
+		return
+	}
+	expected := []string{
+		`kind: Deployment
+spec:
+  replicas: 1
+`,
+		`kind: Service
+spec:
+  selectors:
+    foo: bar
+`,
+	}
+	for i := range nodes {
+		if !assert.Equal(t, expected[i], nodes[i].MustString()) {
+			return
+		}
+	}
+
+	// verify the function config
+	assert.Nil(t, r.FunctionConfig)
+	assert.Equal(t, "List", r.WrappingKind)
+	assert.Equal(t, "v1", r.WrappingApiVersion)
+}
+
+// TestByteReader_Read tests the default Read behavior
+// - Resources are read into a slice
+// - ReaderAnnotations are set on the ResourceNodes
+func TestByteReader_Read(t *testing.T) {
+	nodes, err := (&ByteReader{Reader: getByteReaderTestInput(t)}).Read()
+	if !assert.NoError(t, err) {
+		return
+	}
+
+	if !assert.Len(t, nodes, 3) {
+		return
+	}
+	expected := []string{
+		`a: b # first resource
+c: d
+metadata:
+  annotations:
+    kyaml.kustomize.dev/kio/index: 0
+`,
+		`# second resource
+e: f
+g:
+- h
+metadata:
+  annotations:
+    kyaml.kustomize.dev/kio/index: 1
+`,
+		`i: j
+metadata:
+  annotations:
+    kyaml.kustomize.dev/kio/index: 2
+`,
+	}
+	for i := range nodes {
+		val, err := nodes[i].String()
+		if !assert.NoError(t, err) {
+			return
+		}
+		if !assert.Equal(t, expected[i], val) {
+			return
+		}
+	}
+}
+
+// TestByteReader_Read_omitReaderAnnotations tests
+// - Resources are read into a slice
+// - ReaderAnnotations are not set on the ResourceNodes
+func TestByteReader_Read_omitReaderAnnotations(t *testing.T) {
+	nodes, err := (&ByteReader{
+		Reader:                getByteReaderTestInput(t),
+		OmitReaderAnnotations: true}).Read()
+	if !assert.NoError(t, err) {
+		return
+	}
+
+	// should have parsed 3 resources
+	if !assert.Len(t, nodes, 3) {
+		return
+	}
+	expected := []string{
+		"a: b # first resource\nc: d\n",
+		"# second resource\ne: f\ng:\n- h\n",
+		"i: j\n",
+	}
+	for i := range nodes {
+		val, err := nodes[i].String()
+		if !assert.NoError(t, err) {
+			return
+		}
+		if !assert.Equal(t, expected[i], val) {
+			return
+		}
+	}
+}
+
+// TestByteReader_Read_omitReaderAnnotations tests
+// - Resources are read into a slice
+// - ReaderAnnotations are NOT set on the ResourceNodes
+// - Additional annotations ARE set on the ResourceNodes
+func TestByteReader_Read_setAnnotationsOmitReaderAnnotations(t *testing.T) {
+	nodes, err := (&ByteReader{
+		Reader:                getByteReaderTestInput(t),
+		SetAnnotations:        map[string]string{"foo": "bar"},
+		OmitReaderAnnotations: true,
+	}).Read()
+	if !assert.NoError(t, err) {
+		return
+	}
+
+	if !assert.Len(t, nodes, 3) {
+		return
+	}
+	expected := []string{
+		`a: b # first resource
+c: d
+metadata:
+  annotations:
+    foo: bar
+`,
+		`# second resource
+e: f
+g:
+- h
+metadata:
+  annotations:
+    foo: bar
+`,
+		`i: j
+metadata:
+  annotations:
+    foo: bar
+`,
+	}
+	for i := range nodes {
+		val, err := nodes[i].String()
+		if !assert.NoError(t, err) {
+			return
+		}
+		if !assert.Equal(t, expected[i], val) {
+			return
+		}
+	}
+}
+
+// TestByteReader_Read_omitReaderAnnotations tests
+// - Resources are read into a slice
+// - ReaderAnnotations ARE set on the ResourceNodes
+// - Additional annotations ARE set on the ResourceNodes
+func TestByteReader_Read_setAnnotations(t *testing.T) {
+	nodes, err := (&ByteReader{
+		Reader:         getByteReaderTestInput(t),
+		SetAnnotations: map[string]string{"foo": "bar"},
+	}).Read()
+	if !assert.NoError(t, err) {
+		return
+	}
+
+	if !assert.Len(t, nodes, 3) {
+		return
+	}
+	expected := []string{
+		`a: b # first resource
+c: d
+metadata:
+  annotations:
+    foo: bar
+    kyaml.kustomize.dev/kio/index: 0
+`,
+		`# second resource
+e: f
+g:
+- h
+metadata:
+  annotations:
+    foo: bar
+    kyaml.kustomize.dev/kio/index: 1
+`,
+		`i: j
+metadata:
+  annotations:
+    foo: bar
+    kyaml.kustomize.dev/kio/index: 2
+`,
+	}
+	for i := range nodes {
+		val, err := nodes[i].String()
+		if !assert.NoError(t, err) {
+			return
+		}
+		if !assert.Equal(t, expected[i], val) {
+			return
+		}
+	}
+}

kyaml/kio/byteio_writer.go

@@ -0,0 +1,122 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package kio
+
+import (
+	"io"
+
+	"sigs.k8s.io/kustomize/kyaml/kio/kioutil"
+	"sigs.k8s.io/kustomize/kyaml/yaml"
+)
+
+// Writer writes ResourceNodes to bytes.
+type ByteWriter struct {
+	// Writer is where ResourceNodes are encoded.
+	Writer io.Writer
+
+	// KeepReaderAnnotations if set will keep the Reader specific annotations when writing
+	// the Resources, otherwise they will be cleared.
+	KeepReaderAnnotations bool
+
+	// ClearAnnotations is a list of annotations to clear when writing the Resources.
+	ClearAnnotations []string
+
+	// Style is a style that is set on the Resource Node Document.
+	Style yaml.Style
+
+	// FunctionConfig is the function config for an ResourceList.  If non-nil
+	// wrap the results in an ResourceList.
+	FunctionConfig *yaml.RNode
+
+	// WrappingKind if set will cause ByteWriter to wrap the Resources in
+	// an 'items' field in this kind.  e.g. if WrappingKind is 'List',
+	// ByteWriter will wrap the Resources in a List .items field.
+	WrappingKind string
+
+	// WrappingApiVersion is the apiVersion for WrappingKind
+	WrappingApiVersion string
+
+	// Sort if set, will cause ByteWriter to sort the the nodes before writing them.
+	Sort bool
+}
+
+var _ Writer = ByteWriter{}
+
+func (w ByteWriter) Write(nodes []*yaml.RNode) error {
+	if w.Sort {
+		if err := kioutil.SortNodes(nodes); err != nil {
+			return err
+		}
+	}
+
+	encoder := yaml.NewEncoder(w.Writer)
+	defer encoder.Close()
+	for i := range nodes {
+
+		// clean resources by removing annotations set by the Reader
+		if !w.KeepReaderAnnotations {
+			_, err := nodes[i].Pipe(yaml.ClearAnnotation(kioutil.IndexAnnotation))
+			if err != nil {
+				return err
+			}
+		}
+		for _, a := range w.ClearAnnotations {
+			_, err := nodes[i].Pipe(yaml.ClearAnnotation(a))
+			if err != nil {
+				return err
+			}
+		}
+
+		// TODO(pwittrock): factor this into a a common module for pruning empty values
+		_, err := nodes[i].Pipe(yaml.Lookup("metadata"), yaml.FieldClearer{
+			Name: "annotations", IfEmpty: true})
+		if err != nil {
+			return err
+		}
+		_, err = nodes[i].Pipe(yaml.FieldClearer{Name: "metadata", IfEmpty: true})
+		if err != nil {
+			return err
+		}
+
+		if w.Style != 0 {
+			nodes[i].YNode().Style = w.Style
+		}
+	}
+
+	// don't wrap the elements
+	if w.WrappingKind == "" {
+		for i := range nodes {
+			err := encoder.Encode(nodes[i].Document())
+			if err != nil {
+				return err
+			}
+		}
+		return nil
+	}
+	// wrap the elements in a list
+	items := &yaml.Node{Kind: yaml.SequenceNode}
+	list := &yaml.Node{
+		Kind:  yaml.MappingNode,
+		Style: w.Style,
+		Content: []*yaml.Node{
+			{Kind: yaml.ScalarNode, Value: "apiVersion"},
+			{Kind: yaml.ScalarNode, Value: w.WrappingApiVersion},
+			{Kind: yaml.ScalarNode, Value: "kind"},
+			{Kind: yaml.ScalarNode, Value: w.WrappingKind},
+			{Kind: yaml.ScalarNode, Value: "items"}, items,
+		}}
+	if w.FunctionConfig != nil {
+		list.Content = append(list.Content,
+			&yaml.Node{Kind: yaml.ScalarNode, Value: "functionConfig"},
+			w.FunctionConfig.YNode())
+	}
+	doc := &yaml.Node{
+		Kind:    yaml.DocumentNode,
+		Style:   yaml.FoldedStyle,
+		Content: []*yaml.Node{list}}
+	for i := range nodes {
+		items.Content = append(items.Content, nodes[i].YNode())
+	}
+	return encoder.Encode(doc)
+}

kyaml/kio/byteio_writer_test.go

@@ -0,0 +1,284 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package kio
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"sigs.k8s.io/kustomize/kyaml/yaml"
+)
+
+// TestByteWriter_Write_withoutAnnotations tests:
+// - Resource Config ordering is preserved if no annotations are present
+func TestByteWriter_Write_wrapped(t *testing.T) {
+	node1, err := yaml.Parse(`a: b #first
+`)
+	if !assert.NoError(t, err) {
+		return
+	}
+	node2, err := yaml.Parse(`c: d # second
+`)
+	if !assert.NoError(t, err) {
+		return
+	}
+	node3, err := yaml.Parse(`e: f
+g:
+  h:
+  - i # has a list
+  - j
+`)
+	if !assert.NoError(t, err) {
+		return
+	}
+
+	buff := &bytes.Buffer{}
+	err = ByteWriter{
+		Sort:               true,
+		Writer:             buff,
+		FunctionConfig:     node3,
+		WrappingKind:       ResourceListKind,
+		WrappingApiVersion: ResourceListApiVersion}.
+		Write([]*yaml.RNode{node2, node1})
+	if !assert.NoError(t, err) {
+		return
+	}
+	assert.Equal(t, `apiVersion: kyaml.kustomize.dev/v1alpha1
+kind: ResourceList
+items:
+- c: d # second
+- a: b #first
+functionConfig:
+  e: f
+  g:
+    h:
+    - i # has a list
+    - j
+`, buff.String())
+}
+
+// TestByteWriter_Write_withoutAnnotations tests:
+// - Resource Config ordering is preserved if no annotations are present
+func TestByteWriter_Write_withoutAnnotations(t *testing.T) {
+	node1, err := yaml.Parse(`a: b #first
+`)
+	if !assert.NoError(t, err) {
+		return
+	}
+	node2, err := yaml.Parse(`c: d # second
+`)
+	if !assert.NoError(t, err) {
+		return
+	}
+	node3, err := yaml.Parse(`e: f
+g:
+  h:
+  - i # has a list
+  - j
+`)
+	if !assert.NoError(t, err) {
+		return
+	}
+
+	buff := &bytes.Buffer{}
+	err = ByteWriter{Writer: buff}.
+		Write([]*yaml.RNode{node2, node3, node1})
+	if !assert.NoError(t, err) {
+		return
+	}
+	assert.Equal(t, `c: d # second
+---
+e: f
+g:
+  h:
+  - i # has a list
+  - j
+---
+a: b #first
+`, buff.String())
+}
+
+// TestByteWriter_Write_withAnnotationsKeepAnnotations tests:
+// - Resource Config is sorted by annotations if present
+// - IndexAnnotations are retained
+func TestByteWriter_Write_withAnnotationsKeepAnnotations(t *testing.T) {
+	node1, err := yaml.Parse(`a: b #first
+metadata:
+  annotations:
+    kyaml.kustomize.dev/kio/index: 0
+    kyaml.kustomize.dev/kio/path: "a/b/a_test.yaml"
+`)
+	if !assert.NoError(t, err) {
+		return
+	}
+	node2, err := yaml.Parse(`c: d # second
+metadata:
+  annotations:
+    kyaml.kustomize.dev/kio/index: 1
+    kyaml.kustomize.dev/kio/path: "a/b/a_test.yaml"
+`)
+	if !assert.NoError(t, err) {
+		return
+	}
+	node3, err := yaml.Parse(`e: f
+g:
+  h:
+  - i # has a list
+  - j
+metadata:
+  annotations:
+    kyaml.kustomize.dev/kio/index: 0
+    kyaml.kustomize.dev/kio/path: "a/b/b_test.yaml"
+`)
+	if !assert.NoError(t, err) {
+		return
+	}
+
+	buff := &bytes.Buffer{}
+	err = ByteWriter{Sort: true, Writer: buff, KeepReaderAnnotations: true}.
+		Write([]*yaml.RNode{node2, node3, node1})
+	if !assert.NoError(t, err) {
+		return
+	}
+	assert.Equal(t, `a: b #first
+metadata:
+  annotations:
+    kyaml.kustomize.dev/kio/index: 0
+    kyaml.kustomize.dev/kio/path: "a/b/a_test.yaml"
+---
+c: d # second
+metadata:
+  annotations:
+    kyaml.kustomize.dev/kio/index: 1
+    kyaml.kustomize.dev/kio/path: "a/b/a_test.yaml"
+---
+e: f
+g:
+  h:
+  - i # has a list
+  - j
+metadata:
+  annotations:
+    kyaml.kustomize.dev/kio/index: 0
+    kyaml.kustomize.dev/kio/path: "a/b/b_test.yaml"
+`, buff.String())
+}
+
+// TestByteWriter_Write_withAnnotations tests:
+// - Resource Config is sorted by annotations if present
+// - IndexAnnotations are pruned
+func TestByteWriter_Write_withAnnotations(t *testing.T) {
+	node1, err := yaml.Parse(`a: b #first
+metadata:
+  annotations:
+    kyaml.kustomize.dev/kio/index: 0
+    kyaml.kustomize.dev/kio/path: "a/b/a_test.yaml"
+`)
+	if !assert.NoError(t, err) {
+		return
+	}
+	node2, err := yaml.Parse(`c: d # second
+metadata:
+  annotations:
+    kyaml.kustomize.dev/kio/index: 1
+    kyaml.kustomize.dev/kio/path: "a/b/a_test.yaml"
+`)
+	if !assert.NoError(t, err) {
+		return
+	}
+	node3, err := yaml.Parse(`e: f
+g:
+  h:
+  - i # has a list
+  - j
+metadata:
+  annotations:
+    kyaml.kustomize.dev/kio/index: 0
+    kyaml.kustomize.dev/kio/path: "a/b/b_test.yaml"
+`)
+	if !assert.NoError(t, err) {
+		return
+	}
+
+	buff := &bytes.Buffer{}
+	err = ByteWriter{Sort: true, Writer: buff}.
+		Write([]*yaml.RNode{node2, node3, node1})
+	if !assert.NoError(t, err) {
+		return
+	}
+	assert.Equal(t, `a: b #first
+metadata:
+  annotations:
+    kyaml.kustomize.dev/kio/path: "a/b/a_test.yaml"
+---
+c: d # second
+metadata:
+  annotations:
+    kyaml.kustomize.dev/kio/path: "a/b/a_test.yaml"
+---
+e: f
+g:
+  h:
+  - i # has a list
+  - j
+metadata:
+  annotations:
+    kyaml.kustomize.dev/kio/path: "a/b/b_test.yaml"
+`, buff.String())
+}
+
+// TestByteWriter_Write_partialValues tests:
+// - Resource Config is sorted when annotations are present on some but not all ResourceNodes
+func TestByteWriter_Write_partialAnnotations(t *testing.T) {
+	node1, err := yaml.Parse(`a: b #first
+metadata:
+  annotations:
+    kyaml.kustomize.dev/kio/path: "a/b/a_test.yaml"
+`)
+	if !assert.NoError(t, err) {
+		return
+	}
+	node2, err := yaml.Parse(`c: d # second
+metadata:
+  annotations:
+    kyaml.kustomize.dev/kio/index: 1
+    kyaml.kustomize.dev/kio/path: "a/b/a_test.yaml"
+`)
+	if !assert.NoError(t, err) {
+		return
+	}
+	node3, err := yaml.Parse(`e: f
+g:
+  h:
+  - i # has a list
+  - j
+`)
+	if !assert.NoError(t, err) {
+		return
+	}
+
+	buff := &bytes.Buffer{}
+	rw := ByteWriter{Sort: true, Writer: buff}
+	err = rw.Write([]*yaml.RNode{node2, node3, node1})
+	if !assert.NoError(t, err) {
+		return
+	}
+	assert.Equal(t, `e: f
+g:
+  h:
+  - i # has a list
+  - j
+---
+a: b #first
+metadata:
+  annotations:
+    kyaml.kustomize.dev/kio/path: "a/b/a_test.yaml"
+---
+c: d # second
+metadata:
+  annotations:
+    kyaml.kustomize.dev/kio/path: "a/b/a_test.yaml"
+`, buff.String())
+}

kyaml/kio/example_test.go

@@ -0,0 +1,110 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package kio_test
+
+import (
+	"bytes"
+	"log"
+	"os"
+
+	"sigs.k8s.io/kustomize/kyaml/kio"
+	"sigs.k8s.io/kustomize/kyaml/yaml"
+)
+
+func Example() {
+	input := bytes.NewReader([]byte(`apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: nginx
+  labels:
+    app: nginx
+spec:
+  replicas: 3
+  selector:
+    matchLabels:
+      app: nginx
+  template:
+    metadata:
+      labels:
+        app: nginx
+    spec:
+      containers:
+      - name: nginx
+        image: nginx:1.7.9
+        ports:
+        - containerPort: 80
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: nginx
+spec:
+  selector:
+    app: nginx
+  ports:
+    - protocol: TCP
+      port: 80
+      targetPort: 80
+`))
+
+	// setAnnotationFn
+	setAnnotationFn := kio.FilterFunc(func(operand []*yaml.RNode) ([]*yaml.RNode, error) {
+		for i := range operand {
+			resource := operand[i]
+			_, err := resource.Pipe(yaml.SetAnnotation("foo", "bar"))
+			if err != nil {
+				return nil, err
+			}
+		}
+		return operand, nil
+	})
+
+	err := kio.Pipeline{
+		Inputs:  []kio.Reader{&kio.ByteReader{Reader: input}},
+		Filters: []kio.Filter{setAnnotationFn},
+		Outputs: []kio.Writer{kio.ByteWriter{Writer: os.Stdout}},
+	}.Execute()
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	// Output:
+	// apiVersion: apps/v1
+	// kind: Deployment
+	// metadata:
+	//   name: nginx
+	//   labels:
+	//     app: nginx
+	//   annotations:
+	//     foo: bar
+	// spec:
+	//   replicas: 3
+	//   selector:
+	//     matchLabels:
+	//       app: nginx
+	//   template:
+	//     metadata:
+	//       labels:
+	//         app: nginx
+	//     spec:
+	//       containers:
+	//       - name: nginx
+	//         image: nginx:1.7.9
+	//         ports:
+	//         - containerPort: 80
+	// ---
+	// apiVersion: v1
+	// kind: Service
+	// metadata:
+	//   name: nginx
+	//   annotations:
+	//     foo: bar
+	// spec:
+	//   selector:
+	//     app: nginx
+	//   ports:
+	//   - protocol: TCP
+	//     port: 80
+	//     targetPort: 80
+}

kyaml/kio/filters/container.go

@@ -0,0 +1,171 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package filters
+
+import (
+	"bytes"
+	"os"
+	"os/exec"
+	"regexp"
+	"strings"
+
+	"sigs.k8s.io/kustomize/kyaml/kio"
+
+	"sigs.k8s.io/kustomize/kyaml/yaml"
+)
+
+// GrepFilter filters Resources using a container image.
+// The container must start a process that reads the list of
+// input Resources from stdin, reads the Configuration from the env
+// API_CONFIG, and writes the filtered Resources to stdout.
+// If there is a error or validation failure, the process must exit
+// non-zero.
+// The full set of environment variables from the parent process
+// are passed to the container.
+type ContainerFilter struct {
+	// Image is the container image to use to create a container.
+	Image string `yaml:"image,omitempty"`
+
+	// Config is the API configuration for the container and passed through the
+	// API_CONFIG env var to the container.
+	// Typically a Kubernetes style Resource Config.
+	Config *yaml.RNode `yaml:"config,omitempty"`
+
+	// args may be specified by tests to override how a container is spawned
+	args []string
+
+	checkInput func(string)
+}
+
+// GrepFilter implements kio.GrepFilter
+func (c *ContainerFilter) Filter(input []*yaml.RNode) ([]*yaml.RNode, error) {
+	// get the command to filter the Resources
+	cmd, err := c.getCommand()
+	if err != nil {
+		return nil, err
+	}
+
+	in := &bytes.Buffer{}
+	out := &bytes.Buffer{}
+
+	// write the input
+	err = kio.ByteWriter{
+		WrappingApiVersion: kio.ResourceListApiVersion,
+		WrappingKind:       kio.ResourceListKind,
+		Writer:             in, KeepReaderAnnotations: true, FunctionConfig: c.Config}.Write(input)
+	if err != nil {
+		return nil, err
+	}
+
+	// capture the command stdout for the return value
+	r := &kio.ByteReader{Reader: out}
+
+	// do the filtering
+	if c.checkInput != nil {
+		c.checkInput(in.String())
+	}
+	cmd.Stdin = in
+	cmd.Stdout = out
+	if err := cmd.Run(); err != nil {
+		return nil, err
+	}
+
+	return r.Read()
+}
+
+// getArgs returns the command + args to run to spawn the container
+func (c *ContainerFilter) getArgs() []string {
+	// run the container using docker.  this is simpler than using the docker
+	// libraries, and ensures things like auth work the same as if the container
+	// was run from the cli.
+	args := []string{"docker", "run",
+		"--rm",                                              // delete the container afterward
+		"-i", "-a", "STDIN", "-a", "STDOUT", "-a", "STDERR", // attach stdin, stdout, stderr
+
+		// added security options
+		"--network", "none", // disable the network
+		"--user", "nobody", // run as nobody
+		// don't make fs readonly because things like heredoc rely on writing tmp files
+		"--security-opt=no-new-privileges", // don't allow the user to escalate privileges
+	}
+
+	// export the local environment vars to the container
+	for _, pair := range os.Environ() {
+		args = append(args, "-e", strings.Split(pair, "=")[0])
+	}
+	return append(args, c.Image)
+
+}
+
+// getCommand returns a command which will apply the GrepFilter using the container image
+func (c *ContainerFilter) getCommand() (*exec.Cmd, error) {
+	// encode the filter command API configuration
+	cfg := &bytes.Buffer{}
+	if err := func() error {
+		e := yaml.NewEncoder(cfg)
+		defer e.Close()
+		// make it fit on a single line
+		c.Config.YNode().Style = yaml.FlowStyle
+		return e.Encode(c.Config.YNode())
+	}(); err != nil {
+		return nil, err
+	}
+
+	if len(c.args) == 0 {
+		c.args = c.getArgs()
+	}
+
+	cmd := exec.Command(c.args[0], c.args[1:]...)
+	cmd.Stderr = os.Stderr
+	cmd.Env = os.Environ()
+
+	// set stderr for err messaging
+	return cmd, nil
+}
+
+// IsReconcilerFilter filters Resources based on whether or not they are Reconciler Resource.
+// Resources with an apiVersion starting with '*.gcr.io', 'gcr.io' or 'docker.io' are considered
+// Reconciler Resources.
+type IsReconcilerFilter struct {
+	// ExcludeReconcilers if set to true, then Reconcilers will be excluded -- e.g.
+	// Resources with a reconcile container through the apiVersion (gcr.io prefix) or
+	// through the annotations
+	ExcludeReconcilers bool `yaml:"excludeReconcilers,omitempty"`
+
+	// IncludeNonReconcilers if set to true, the NonReconciler will be included.
+	IncludeNonReconcilers bool `yaml:"includeNonReconcilers,omitempty"`
+}
+
+// Filter implements kio.Filter
+func (c *IsReconcilerFilter) Filter(inputs []*yaml.RNode) ([]*yaml.RNode, error) {
+	var out []*yaml.RNode
+	for i := range inputs {
+		isContainerResource := GetContainerName(inputs[i]) != ""
+		if isContainerResource && !c.ExcludeReconcilers {
+			out = append(out, inputs[i])
+		}
+		if !isContainerResource && c.IncludeNonReconcilers {
+			out = append(out, inputs[i])
+		}
+	}
+	return out, nil
+}
+
+// GetContainerName returns the container image for an API if one exists
+func GetContainerName(n *yaml.RNode) string {
+	meta, _ := n.GetMeta()
+	container := meta.Annotations["kyaml.kustomize.dev/container"]
+	if container != "" {
+		return container
+	}
+
+	if match.MatchString(meta.ApiVersion) {
+		return meta.ApiVersion
+	}
+
+	return ""
+}
+
+// match specifies the set of apiVersions to recognize as being container images
+var match = regexp.MustCompile(`(docker\.io|.*\.?gcr\.io)/.*(:.*)?`)

kyaml/kio/filters/container_test.go

@@ -0,0 +1,281 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package filters
+
+import (
+	"bytes"
+	"os"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"sigs.k8s.io/kustomize/kyaml/kio"
+	"sigs.k8s.io/kustomize/kyaml/yaml"
+)
+
+func TestFilter_command(t *testing.T) {
+	cfg, err := yaml.Parse(`apiversion: apps/v1
+kind: Deployment
+metadata:
+  name: foo
+`)
+	if !assert.NoError(t, err) {
+		return
+	}
+	instance := &ContainerFilter{
+		Image:  "example.com:version",
+		Config: cfg,
+	}
+	os.Setenv("KYAML_TEST", "FOO")
+	cmd, err := instance.getCommand()
+	if !assert.NoError(t, err) {
+		return
+	}
+
+	expected := []string{
+		"docker", "run",
+		"--rm",
+		"-i", "-a", "STDIN", "-a", "STDOUT", "-a", "STDERR",
+		"--network", "none",
+		"--user", "nobody",
+		"--security-opt=no-new-privileges",
+	}
+	for _, e := range os.Environ() {
+		// the process env
+		expected = append(expected, "-e", strings.Split(e, "=")[0])
+	}
+	expected = append(expected, "example.com:version")
+	assert.Equal(t, expected, cmd.Args)
+
+	foundKyaml := false
+	for _, e := range cmd.Env {
+		// verify the command has the right environment variables to pass to the container
+		split := strings.Split(e, "=")
+		if split[0] == "KYAML_TEST" {
+			assert.Equal(t, "FOO", split[1])
+			foundKyaml = true
+		}
+	}
+	assert.True(t, foundKyaml)
+}
+
+func TestFilter_Filter(t *testing.T) {
+	cfg, err := yaml.Parse(`apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: foo
+`)
+	if !assert.NoError(t, err) {
+		return
+	}
+
+	input, err := (&kio.ByteReader{Reader: bytes.NewBufferString(`
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: deployment-foo
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: service-foo
+`)}).Read()
+	if !assert.NoError(t, err) {
+		return
+	}
+
+	called := false
+	result, err := (&ContainerFilter{
+		Image:  "example.com:version",
+		Config: cfg,
+		args:   []string{"sed", "s/Deployment/StatefulSet/g"},
+		checkInput: func(s string) {
+			called = true
+			if !assert.Equal(t, `apiVersion: kyaml.kustomize.dev/v1alpha1
+kind: ResourceList
+items:
+- apiVersion: apps/v1
+  kind: Deployment
+  metadata:
+    name: deployment-foo
+    annotations:
+      kyaml.kustomize.dev/kio/index: 0
+- apiVersion: v1
+  kind: Service
+  metadata:
+    name: service-foo
+    annotations:
+      kyaml.kustomize.dev/kio/index: 1
+functionConfig: {apiVersion: apps/v1, kind: Deployment, metadata: {name: foo}}
+`, s) {
+				t.FailNow()
+			}
+		},
+	}).Filter(input)
+	if !assert.NoError(t, err) {
+		return
+	}
+	if !assert.True(t, called) {
+		return
+	}
+
+	b := &bytes.Buffer{}
+	err = kio.ByteWriter{Writer: b, KeepReaderAnnotations: true}.Write(result)
+	if !assert.NoError(t, err) {
+		return
+	}
+
+	assert.Equal(t, `apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: deployment-foo
+  annotations:
+    kyaml.kustomize.dev/kio/index: 0
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: service-foo
+  annotations:
+    kyaml.kustomize.dev/kio/index: 1
+`, b.String())
+}
+
+func TestFilter_Filter_noChange(t *testing.T) {
+	cfg, err := yaml.Parse(`apiversion: apps/v1
+kind: Deployment
+metadata:
+  name: foo
+`)
+	if !assert.NoError(t, err) {
+		return
+	}
+
+	input, err := (&kio.ByteReader{Reader: bytes.NewBufferString(`
+apiversion: apps/v1
+kind: Deployment
+metadata:
+  name: deployment-foo
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: service-foo
+`)}).Read()
+	if !assert.NoError(t, err) {
+		return
+	}
+
+	called := false
+	result, err := (&ContainerFilter{
+		Image:  "example.com:version",
+		Config: cfg,
+		args:   []string{"sh", "-c", "cat <&0"},
+		checkInput: func(s string) {
+			called = true
+			if !assert.Equal(t, `apiVersion: kyaml.kustomize.dev/v1alpha1
+kind: ResourceList
+items:
+- apiversion: apps/v1
+  kind: Deployment
+  metadata:
+    name: deployment-foo
+    annotations:
+      kyaml.kustomize.dev/kio/index: 0
+- apiVersion: v1
+  kind: Service
+  metadata:
+    name: service-foo
+    annotations:
+      kyaml.kustomize.dev/kio/index: 1
+functionConfig: {apiversion: apps/v1, kind: Deployment, metadata: {name: foo}}
+`, s) {
+				t.FailNow()
+			}
+		},
+	}).Filter(input)
+	if !assert.NoError(t, err) {
+		return
+	}
+	if !assert.True(t, called) {
+		return
+	}
+
+	b := &bytes.Buffer{}
+	err = kio.ByteWriter{Writer: b, KeepReaderAnnotations: true}.Write(result)
+	if !assert.NoError(t, err) {
+		return
+	}
+
+	assert.Equal(t, `apiversion: apps/v1
+kind: Deployment
+metadata:
+  name: deployment-foo
+  annotations:
+    kyaml.kustomize.dev/kio/index: 0
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: service-foo
+  annotations:
+    kyaml.kustomize.dev/kio/index: 1
+`, b.String())
+}
+
+func Test_GetContainerName(t *testing.T) {
+	// make sure gcr.io works
+	n, err := yaml.Parse(`apiVersion: gcr.io/foo/bar:something
+kind: MyThing
+`)
+	if !assert.NoError(t, err) {
+		return
+	}
+	c := GetContainerName(n)
+	assert.Equal(t, "gcr.io/foo/bar:something", c)
+
+	// make sure regional gcr.io works
+	n, err = yaml.Parse(`apiVersion: us.gcr.io/foo/bar:something
+kind: MyThing
+`)
+	if !assert.NoError(t, err) {
+		return
+	}
+	c = GetContainerName(n)
+	assert.Equal(t, "us.gcr.io/foo/bar:something", c)
+
+	// container from annotation
+	n, err = yaml.Parse(`apiVersion: v1
+kind: MyThing
+metadata:
+  annotations:
+    kyaml.kustomize.dev/container: gcr.io/foo/bar:something
+`)
+	if !assert.NoError(t, err) {
+		return
+	}
+	c = GetContainerName(n)
+	assert.Equal(t, "gcr.io/foo/bar:something", c)
+
+	// doesn't have a container
+	n, err = yaml.Parse(`apiVersion: v1
+kind: MyThing
+metadata:
+`)
+	if !assert.NoError(t, err) {
+		return
+	}
+	c = GetContainerName(n)
+	assert.Equal(t, "", c)
+
+	// make sure docker.io works
+	n, err = yaml.Parse(`apiVersion: docker.io/foo/bar:something
+kind: MyThing
+`)
+	if !assert.NoError(t, err) {
+		return
+	}
+	c = GetContainerName(n)
+	assert.Equal(t, "docker.io/foo/bar:something", c)
+}

kyaml/kio/filters/filters.go

@@ -0,0 +1,198 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package filters
+
+import (
+	"fmt"
+	"sort"
+	"strings"
+
+	"sigs.k8s.io/kustomize/kyaml/kio"
+	"sigs.k8s.io/kustomize/kyaml/kio/kioutil"
+	"sigs.k8s.io/kustomize/kyaml/yaml"
+)
+
+// Filters are the list of known filters for unmarshalling a filter into a concrete
+// implementation.
+var Filters = map[string]func() kio.Filter{
+	"FileSetter":    func() kio.Filter { return &FileSetter{} },
+	"FormatFilter":  func() kio.Filter { return &FormatFilter{} },
+	"GrepFilter":    func() kio.Filter { return GrepFilter{} },
+	"MatchModifier": func() kio.Filter { return &MatchModifyFilter{} },
+	"Modifier":      func() kio.Filter { return &Modifier{} },
+}
+
+// filter wraps a kio.filter so that it can be unmarshalled from yaml.
+type KFilter struct {
+	kio.Filter
+}
+
+func (t KFilter) MarshalYAML() (interface{}, error) {
+	return t.Filter, nil
+}
+
+func (t *KFilter) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	i := map[string]interface{}{}
+	if err := unmarshal(i); err != nil {
+		return err
+	}
+	meta := &yaml.ResourceMeta{}
+	if err := unmarshal(meta); err != nil {
+		return err
+	}
+	if filter, found := Filters[meta.Kind]; !found {
+		var knownFilters []string
+		for k := range Filters {
+			knownFilters = append(knownFilters, k)
+		}
+		sort.Strings(knownFilters)
+		return fmt.Errorf("unsupported filter Kind %v:  may be one of: [%s]",
+			meta, strings.Join(knownFilters, ","))
+	} else {
+		t.Filter = filter()
+	}
+	return unmarshal(t.Filter)
+}
+
+// Modifier modifies the input Resources by invoking the provided pipeline.
+// Modifier will return any Resources for which the pipeline does not return an error.
+type Modifier struct {
+	Kind string `yaml:"kind,omitempty"`
+
+	Filters yaml.YFilters `yaml:"pipeline,omitempty"`
+}
+
+var _ kio.Filter = &Modifier{}
+
+func (f Modifier) Filter(input []*yaml.RNode) ([]*yaml.RNode, error) {
+	for i := range input {
+		if _, err := input[i].Pipe(f.Filters.Filters()...); err != nil {
+			return nil, err
+		}
+	}
+	return input, nil
+}
+
+type MatchModifyFilter struct {
+	Kind string `yaml:"kind,omitempty"`
+
+	MatchFilters []yaml.YFilters `yaml:"match,omitempty"`
+
+	ModifyFilters yaml.YFilters `yaml:"modify,omitempty"`
+}
+
+var _ kio.Filter = &MatchModifyFilter{}
+
+func (f MatchModifyFilter) Filter(input []*yaml.RNode) ([]*yaml.RNode, error) {
+	var matches = input
+	var err error
+	for _, filter := range f.MatchFilters {
+		matches, err = MatchFilter{Filters: filter}.Filter(matches)
+		if err != nil {
+			return nil, err
+		}
+	}
+	_, err = Modifier{Filters: f.ModifyFilters}.Filter(matches)
+	if err != nil {
+		return nil, err
+	}
+	return input, nil
+}
+
+type MatchFilter struct {
+	Kind string `yaml:"kind,omitempty"`
+
+	Filters yaml.YFilters `yaml:"pipeline,omitempty"`
+}
+
+var _ kio.Filter = &MatchFilter{}
+
+func (f MatchFilter) Filter(input []*yaml.RNode) ([]*yaml.RNode, error) {
+	var output []*yaml.RNode
+	for i := range input {
+		if v, err := input[i].Pipe(f.Filters.Filters()...); err != nil {
+			return nil, err
+		} else if v == nil {
+			continue
+		}
+		output = append(output, input[i])
+	}
+	return output, nil
+}
+
+type FilenameFmtVerb string
+
+const (
+	// KindFmt substitutes kind
+	KindFmt FilenameFmtVerb = "%k"
+
+	// NameFmt substitutes metadata.name
+	NameFmt FilenameFmtVerb = "%n"
+
+	// NamespaceFmt substitutes metdata.namespace
+	NamespaceFmt FilenameFmtVerb = "%s"
+)
+
+// FileSetter sets the file name and mode annotations on Resources.
+type FileSetter struct {
+	Kind string `yaml:"kind,omitempty"`
+
+	// FilenamePattern is the pattern to use for generating filenames.  FilenameFmtVerb
+	// FielnameFmtVerbs may be specified to substitute Resource metadata into the filename.
+	FilenamePattern string `yaml:"filenamePattern,omitempty"`
+
+	// Mode is the filemode to write.
+	Mode string `yaml:"mode,omitempty"`
+
+	// Override will override the existing filename if it is set on the pattern.
+	// Otherwise the existing filename is kept.
+	Override bool `yaml:"override,omitempty"`
+}
+
+var _ kio.Filter = &FileSetter{}
+
+const DefaultFilenamePattern = "%n_%k.yaml"
+
+func (f *FileSetter) Filter(input []*yaml.RNode) ([]*yaml.RNode, error) {
+	if f.Mode == "" {
+		f.Mode = fmt.Sprintf("%d", 0600)
+	}
+	if f.FilenamePattern == "" {
+		f.FilenamePattern = DefaultFilenamePattern
+	}
+
+	resources := map[string][]*yaml.RNode{}
+	for i := range input {
+		m, err := input[i].GetMeta()
+		if err != nil {
+			return nil, err
+		}
+		file := f.FilenamePattern
+		file = strings.Replace(file, string(KindFmt), strings.ToLower(m.Kind), -1)
+		file = strings.Replace(file, string(NameFmt), strings.ToLower(m.Name), -1)
+		file = strings.Replace(file, string(NamespaceFmt), strings.ToLower(m.Namespace), -1)
+
+		if _, found := m.Annotations[kioutil.PathAnnotation]; !found || f.Override {
+			if _, err := input[i].Pipe(yaml.SetAnnotation(kioutil.PathAnnotation, file)); err != nil {
+				return nil, err
+			}
+		}
+		resources[file] = append(resources[file], input[i])
+	}
+
+	var output []*yaml.RNode
+	for i := range resources {
+		if err := kioutil.SortNodes(resources[i]); err != nil {
+			return nil, err
+		}
+		for j := range resources[i] {
+			if _, err := resources[i][j].Pipe(
+				yaml.SetAnnotation(kioutil.IndexAnnotation, fmt.Sprintf("%d", j))); err != nil {
+				return nil, err
+			}
+			output = append(output, resources[i][j])
+		}
+	}
+	return output, nil
+}

kyaml/kio/filters/filters_test.go

@@ -0,0 +1,170 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package filters_test
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	. "sigs.k8s.io/kustomize/kyaml/kio"
+	. "sigs.k8s.io/kustomize/kyaml/kio/filters"
+)
+
+var r = `
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: foo1
+  namespace: bar
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: foo2
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: foo2
+  namespace: bar
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: foo1
+`
+
+func TestFileSetter_Filter(t *testing.T) {
+	in := bytes.NewBufferString(r)
+	out := &bytes.Buffer{}
+	err := Pipeline{
+		Inputs:  []Reader{&ByteReader{Reader: in}},
+		Filters: []Filter{&FileSetter{}},
+		Outputs: []Writer{ByteWriter{Sort: true, Writer: out}},
+	}.Execute()
+	if !assert.NoError(t, err) {
+		return
+	}
+	assert.Equal(t, `apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: foo1
+  namespace: bar
+  annotations:
+    kyaml.kustomize.dev/kio/path: foo1_deployment.yaml
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: foo1
+  annotations:
+    kyaml.kustomize.dev/kio/path: foo1_service.yaml
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: foo2
+  annotations:
+    kyaml.kustomize.dev/kio/path: foo2_deployment.yaml
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: foo2
+  namespace: bar
+  annotations:
+    kyaml.kustomize.dev/kio/path: foo2_service.yaml
+`, out.String())
+}
+
+func TestFileSetter_Filter_pattern(t *testing.T) {
+	in := bytes.NewBufferString(r)
+	out := &bytes.Buffer{}
+	err := Pipeline{
+		Inputs: []Reader{&ByteReader{Reader: in}},
+		Filters: []Filter{&FileSetter{
+			FilenamePattern: "%n_%s_%k.yaml",
+		}},
+		Outputs: []Writer{ByteWriter{Sort: true, Writer: out}},
+	}.Execute()
+	if !assert.NoError(t, err) {
+		return
+	}
+	assert.Equal(t, `apiVersion: v1
+kind: Service
+metadata:
+  name: foo1
+  annotations:
+    kyaml.kustomize.dev/kio/path: foo1__service.yaml
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: foo1
+  namespace: bar
+  annotations:
+    kyaml.kustomize.dev/kio/path: foo1_bar_deployment.yaml
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: foo2
+  annotations:
+    kyaml.kustomize.dev/kio/path: foo2__deployment.yaml
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: foo2
+  namespace: bar
+  annotations:
+    kyaml.kustomize.dev/kio/path: foo2_bar_service.yaml
+`, out.String())
+}
+
+func TestFileSetter_Filter_empty(t *testing.T) {
+	in := bytes.NewBufferString(r)
+	out := &bytes.Buffer{}
+	err := Pipeline{
+		Inputs: []Reader{&ByteReader{Reader: in}},
+		Filters: []Filter{&FileSetter{
+			FilenamePattern: "resource.yaml",
+		}},
+		Outputs: []Writer{ByteWriter{Writer: out}},
+	}.Execute()
+	if !assert.NoError(t, err) {
+		return
+	}
+	assert.Equal(t, `apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: foo1
+  namespace: bar
+  annotations:
+    kyaml.kustomize.dev/kio/path: resource.yaml
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: foo2
+  annotations:
+    kyaml.kustomize.dev/kio/path: resource.yaml
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: foo2
+  namespace: bar
+  annotations:
+    kyaml.kustomize.dev/kio/path: resource.yaml
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: foo1
+  annotations:
+    kyaml.kustomize.dev/kio/path: resource.yaml
+`, out.String())
+}

kyaml/kio/filters/fmtr.go

@@ -0,0 +1,220 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+// Package yamlfmt contains libraries for formatting yaml files containing
+// Kubernetes Resource configuration.
+//
+// Yaml files are formatted by:
+// - Sorting fields and map values
+// - Sorting unordered lists for whitelisted types
+// - Applying a canonical yaml Style
+//
+// Fields are ordered using a relative ordering applied to commonly
+// encountered Resource fields.  All Resources,  including non-builtin
+// Resources such as CRDs, share the same field precedence.
+//
+// Fields that do not appear in the explicit ordering are ordered
+// lexicographically.
+//
+// A subset of well known known unordered lists are sorted by element field
+// values.
+package filters
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"sort"
+
+	"sigs.k8s.io/kustomize/kyaml/kio"
+	"sigs.k8s.io/kustomize/kyaml/yaml"
+)
+
+// FormatInput returns the formatted input.
+func FormatInput(input io.Reader) (*bytes.Buffer, error) {
+	buff := &bytes.Buffer{}
+	err := kio.Pipeline{
+		Inputs:  []kio.Reader{&kio.ByteReader{Reader: input}},
+		Filters: []kio.Filter{FormatFilter{}},
+		Outputs: []kio.Writer{kio.ByteWriter{Writer: buff}},
+	}.Execute()
+
+	return buff, err
+}
+
+// FormatFileOrDirectory reads the file or directory and formats each file's
+// contents by writing it back to the file.
+func FormatFileOrDirectory(path string) error {
+	return kio.Pipeline{
+		Inputs: []kio.Reader{kio.LocalPackageReader{
+			PackagePath: path,
+		}},
+		Filters: []kio.Filter{FormatFilter{}},
+		Outputs: []kio.Writer{kio.LocalPackageWriter{PackagePath: path}},
+	}.Execute()
+}
+
+type FormatFilter struct{}
+
+var _ kio.Filter = FormatFilter{}
+
+func (f FormatFilter) Filter(slice []*yaml.RNode) ([]*yaml.RNode, error) {
+	for i := range slice {
+		kindNode, err := slice[i].Pipe(yaml.Get("kind"))
+		if err != nil {
+			return nil, err
+		}
+		if kindNode == nil {
+			continue
+		}
+		apiVersionNode, err := slice[i].Pipe(yaml.Get("apiVersion"))
+		if err != nil {
+			return nil, err
+		}
+		if apiVersionNode == nil {
+			continue
+		}
+		kind, apiVersion := kindNode.YNode().Value, apiVersionNode.YNode().Value
+		err = (&formatter{apiVersion: apiVersion, kind: kind}).fmtNode(slice[i].YNode(), "")
+		if err != nil {
+			return nil, err
+		}
+	}
+	return slice, nil
+}
+
+type formatter struct {
+	apiVersion string
+	kind       string
+}
+
+// fmtNode recursively formats the Document Contents.
+func (f *formatter) fmtNode(n *yaml.Node, path string) error {
+	n.Style = 0
+	// sort the order of mapping fields
+	if n.Kind == yaml.MappingNode {
+		sort.Sort(sortedMapContents(*n))
+	}
+
+	// sort the order of sequence elements if it is whitelisted
+	if n.Kind == yaml.SequenceNode {
+		if yaml.WhitelistedListSortKinds.Has(f.kind) &&
+			yaml.WhitelistedListSortApis.Has(f.apiVersion) {
+			if sortField, found := yaml.WhitelistedListSortFields[path]; found {
+				sort.Sort(sortedSeqContents{Node: *n, sortField: sortField})
+			}
+		}
+	}
+	for i := range n.Content {
+		p := path
+		if n.Kind == yaml.MappingNode && i%2 == 1 {
+			p = fmt.Sprintf("%s.%s", path, n.Content[i-1].Value)
+		}
+		err := f.fmtNode(n.Content[i], p)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// sortedMapContents sorts the Contents field of a MappingNode by the field names using a statically
+// defined field precedence, and falling back on lexicographical sorting
+type sortedMapContents yaml.Node
+
+func (s sortedMapContents) Len() int {
+	return len(s.Content) / 2
+}
+func (s sortedMapContents) Swap(i, j int) {
+	// yaml MappingNode Contents are a list of field names followed by
+	// field values, rather than a list of field <name, value> pairs.
+	// increment.
+	//
+	// e.g. ["field1Name", "field1Value", "field2Name", "field2Value"]
+	iFieldNameIndex := i * 2
+	jFieldNameIndex := j * 2
+	iFieldValueIndex := iFieldNameIndex + 1
+	jFieldValueIndex := jFieldNameIndex + 1
+
+	// swap field names
+	s.Content[iFieldNameIndex], s.Content[jFieldNameIndex] =
+		s.Content[jFieldNameIndex], s.Content[iFieldNameIndex]
+
+	// swap field values
+	s.Content[iFieldValueIndex], s.Content[jFieldValueIndex] = s.
+		Content[jFieldValueIndex], s.Content[iFieldValueIndex]
+}
+func (s sortedMapContents) Less(i, j int) bool {
+	iFieldNameIndex := i * 2
+	jFieldNameIndex := j * 2
+	iFieldName := s.Content[iFieldNameIndex].Value
+	jFieldName := s.Content[jFieldNameIndex].Value
+
+	// order by their precedence values looked up from the index
+	iOrder, foundI := yaml.FieldOrder[iFieldName]
+	jOrder, foundJ := yaml.FieldOrder[jFieldName]
+	if foundI && foundJ {
+		return iOrder < jOrder
+	}
+
+	// known fields come before unknown fields
+	if foundI {
+		return true
+	}
+	if foundJ {
+		return false
+	}
+
+	// neither field is known, sort them lexicographically
+	return iFieldName < jFieldName
+}
+
+// sortedSeqContents sorts the Contents field of a SequenceNode by the value of
+// the elements sortField.
+// e.g. it will sort spec.template.spec.containers by the value of the container `name` field
+type sortedSeqContents struct {
+	yaml.Node
+	sortField string
+}
+
+func (s sortedSeqContents) Len() int {
+	return len(s.Content)
+}
+func (s sortedSeqContents) Swap(i, j int) {
+	s.Content[i], s.Content[j] = s.Content[j], s.Content[i]
+}
+func (s sortedSeqContents) Less(i, j int) bool {
+	// primitive lists -- sort by the element's primitive values
+	if s.sortField == "" {
+		iValue := s.Content[i].Value
+		jValue := s.Content[j].Value
+		return iValue < jValue
+	}
+
+	// map lists -- sort by the element's sortField values
+	var iValue, jValue string
+	for a := range s.Content[i].Content {
+		if a%2 != 0 {
+			continue // not a fieldNameIndex
+		}
+		// locate the index of the sortField field
+		if s.Content[i].Content[a].Value == s.sortField {
+			// a is the yaml node for the field key, a+1 is the node for the field value
+			iValue = s.Content[i].Content[a+1].Value
+		}
+	}
+	for a := range s.Content[j].Content {
+		if a%2 != 0 {
+			continue // not a fieldNameIndex
+		}
+
+		// locate the index of the sortField field
+		if s.Content[j].Content[a].Value == s.sortField {
+			// a is the yaml node for the field key, a+1 is the node for the field value
+			jValue = s.Content[j].Content[a+1].Value
+		}
+	}
+
+	// compare the field values
+	return iValue < jValue
+}

kyaml/kio/filters/fmtr_test.go

@@ -0,0 +1,623 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package filters_test
+
+import (
+	"bytes"
+	"io"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	. "sigs.k8s.io/kustomize/kyaml/kio/filters"
+	"sigs.k8s.io/kustomize/kyaml/kio/filters/testyaml"
+)
+
+// TestFormatInput_configMap verifies a ConfigMap yaml is formatted correctly
+func TestFormatInput_configMap(t *testing.T) {
+	y := `
+
+
+# this formatting is intentionally weird
+
+apiVersion: v1
+# this is data
+data:
+  # this is color
+  color: purple
+  # that was color
+
+  # this is textmode
+  textmode: "true"
+  # this is how
+  how: fairlyNice
+
+
+
+kind: ConfigMap
+
+
+metadata:
+  selfLink: /api/v1/namespaces/default/configmaps/config-multi-env-files
+  namespace: default
+  creationTimestamp: 2017-12-27T18:38:34Z
+  name: config-multi-env-files
+  resourceVersion: "810136"
+  uid: 252c4572-eb35-11e7-887b-42010a8002b8  # keep no trailing linefeed`
+
+	expected := `# this formatting is intentionally weird
+
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: config-multi-env-files
+  namespace: default
+  creationTimestamp: 2017-12-27T18:38:34Z
+  resourceVersion: "810136"
+  selfLink: /api/v1/namespaces/default/configmaps/config-multi-env-files
+  uid: 252c4572-eb35-11e7-887b-42010a8002b8 # keep no trailing linefeed
+# this is data
+data:
+  # this is color
+  color: purple
+  # that was color
+
+  # this is how
+  how: fairlyNice
+  # this is textmode
+  textmode: "true"
+`
+
+	s, err := FormatInput(strings.NewReader(y))
+	assert.NoError(t, err)
+	assert.Equal(t, expected, s.String())
+}
+
+// TestFormatInput_deployment verifies a Deployment yaml is formatted correctly
+func TestFormatInput_deployment(t *testing.T) {
+	y := `
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: nginx-deployment
+  labels:
+    app: nginx
+spec:
+  selector:
+    matchLabels:
+      app: nginx
+  replicas: 3
+  template:
+    metadata:
+      labels:
+        app: nginx
+    spec:
+      containers:
+      # this is a container
+      - ports:
+        # this is a port
+        - containerPort: 80
+        name: b-nginx
+        image: nginx:1.7.9
+      # this is another container
+      - name: a-nginx
+        image: nginx:1.7.9
+        ports:
+        - containerPort: 80
+`
+	expected := `apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: nginx-deployment
+  labels:
+    app: nginx
+spec:
+  replicas: 3
+  selector:
+    matchLabels:
+      app: nginx
+  template:
+    metadata:
+      labels:
+        app: nginx
+    spec:
+      containers:
+      - # this is another container
+        name: a-nginx
+        image: nginx:1.7.9
+        ports:
+        - containerPort: 80
+      - name: b-nginx
+        image: nginx:1.7.9
+        # this is a container
+        ports:
+        - # this is a port
+          containerPort: 80
+`
+	s, err := FormatInput(strings.NewReader(y))
+	assert.NoError(t, err)
+	assert.Equal(t, expected, s.String())
+}
+
+// TestFormatInput_service verifies a Service yaml is formatted correctly
+func TestFormatInput_service(t *testing.T) {
+
+	y := `
+apiVersion: v1
+kind: Service
+metadata:
+  name: my-service
+spec:
+  selector:
+    app: MyApp
+  ports:
+    - protocol: TCP
+      port: 80
+      targetPort: 9376
+`
+	expected := `apiVersion: v1
+kind: Service
+metadata:
+  name: my-service
+spec:
+  selector:
+    app: MyApp
+  ports:
+  - protocol: TCP
+    port: 80
+    targetPort: 9376
+`
+	s, err := FormatInput(strings.NewReader(y))
+	assert.NoError(t, err)
+	assert.Equal(t, expected, s.String())
+}
+
+// TestFormatInput_service verifies a Service yaml is formatted correctly
+func TestFormatInput_validatingWebhookConfiguration(t *testing.T) {
+
+	y := `
+apiVersion: admissionregistration.k8s.io/v1beta1
+kind: ValidatingWebhookConfiguration
+metadata:
+  name: <name of this configuration object>
+webhooks:
+- name: <webhook name, e.g., pod-policy.example.io>
+  rules:
+  - apiGroups:
+    - ""
+    apiVersions:
+    - v1
+    operations:
+      - UPDATE # this list is indented by 2
+      - CREATE
+      - CONNECT
+    resources:
+    - pods # this list is not indented by 2
+    scope: "Namespaced"
+  clientConfig:
+    service:
+      namespace: <namespace of the front-end service>
+      name: <name of the front-end service>
+    caBundle: <pem encoded ca cert that signs the server cert used by the webhook>
+  admissionReviewVersions:
+  - v1beta1
+  timeoutSeconds: 1
+`
+	expected := `apiVersion: admissionregistration.k8s.io/v1beta1
+kind: ValidatingWebhookConfiguration
+metadata:
+  name: <name of this configuration object>
+webhooks:
+- name: <webhook name, e.g., pod-policy.example.io>
+  admissionReviewVersions:
+  - v1beta1
+  clientConfig:
+    service:
+      name: <name of the front-end service>
+      namespace: <namespace of the front-end service>
+    caBundle: <pem encoded ca cert that signs the server cert used by the webhook>
+  rules:
+  - resources:
+    - pods # this list is not indented by 2
+    apiGroups:
+    - ""
+    apiVersions:
+    - v1
+    operations:
+    - CONNECT
+    - CREATE
+    - UPDATE # this list is indented by 2
+    scope: Namespaced
+  timeoutSeconds: 1
+`
+	s, err := FormatInput(strings.NewReader(y))
+	assert.NoError(t, err)
+	assert.Equal(t, expected, s.String())
+}
+
+// TestFormatInput_unKnownType verifies an unknown type yaml is formatted correctly
+func TestFormatInput_unKnownType(t *testing.T) {
+	y := `
+spec:
+  template:
+    spec:
+      # these shouldn't be sorted because the type isn't whitelisted
+      containers:
+      - name: b
+      - name: a
+  replicas: 1
+status:
+  conditions:
+  - 3
+  - 1
+  - 2
+other:
+  b: a1
+  a: b1
+apiVersion: example.com/v1beta1
+kind: MyType
+`
+
+	expected := `apiVersion: example.com/v1beta1
+kind: MyType
+spec:
+  replicas: 1
+  template:
+    spec:
+      # these shouldn't be sorted because the type isn't whitelisted
+      containers:
+      - name: b
+      - name: a
+status:
+  conditions:
+  - 3
+  - 1
+  - 2
+other:
+  a: b1
+  b: a1
+`
+	s, err := FormatInput(strings.NewReader(y))
+	assert.NoError(t, err)
+	assert.Equal(t, expected, s.String())
+}
+
+// TestFormatInput_deployment verifies a Deployment yaml is formatted correctly
+func TestFormatInput_resources(t *testing.T) {
+	input := &bytes.Buffer{}
+	_, err := io.Copy(input, bytes.NewReader(testyaml.UnformattedYaml1))
+	assert.NoError(t, err)
+	_, err = io.Copy(input, strings.NewReader("---\n"))
+	assert.NoError(t, err)
+	_, err = io.Copy(input, bytes.NewReader(testyaml.UnformattedYaml2))
+	assert.NoError(t, err)
+
+	expectedOutput := &bytes.Buffer{}
+	_, err = io.Copy(expectedOutput, bytes.NewReader(testyaml.FormattedYaml1))
+	assert.NoError(t, err)
+	_, err = io.Copy(expectedOutput, strings.NewReader("---\n"))
+	assert.NoError(t, err)
+	_, err = io.Copy(expectedOutput, bytes.NewReader(testyaml.FormattedYaml2))
+	assert.NoError(t, err)
+
+	s, err := FormatInput(input)
+	assert.NoError(t, err)
+	assert.Equal(t, expectedOutput.String(), s.String())
+}
+
+//
+func TestFormatInput_failMissingKind(t *testing.T) {
+	y := `
+spec:
+  template:
+    spec:
+      containers:
+      - b
+      - a
+  replicas: 1
+status:
+  conditions:
+  - 3
+  - 1
+  - 2
+other:
+  b: a1
+  a: b1
+apiVersion: example.com/v1beta1
+`
+
+	b, err := FormatInput(strings.NewReader(y))
+	assert.NoError(t, err)
+	assert.Equal(t, strings.TrimLeft(y, "\n"), b.String())
+}
+
+func TestFormatInput_failMissingApiVersion(t *testing.T) {
+	y := `
+spec:
+  template:
+    spec:
+      containers:
+      - a
+      - b
+  replicas: 1
+status:
+  conditions:
+  - 3
+  - 1
+  - 2
+other:
+  b: a1
+  a: b1
+kind: MyKind
+`
+
+	b, err := FormatInput(strings.NewReader(y))
+	assert.NoError(t, err)
+	assert.Equal(t, strings.TrimLeft(y, "\n"), b.String())
+}
+
+func TestFormatInput_failUnmarshal(t *testing.T) {
+	y := `
+spec:
+  template:
+    spec:
+      containers:
+      - a
+      - b
+  replicas: 1
+status:
+  conditions:
+  - 3
+  - 1
+  - 2
+other:
+	b: a1
+	a: b1
+kind: MyKind
+apiVersion: example.com/v1beta1
+`
+
+	_, err := FormatInput(strings.NewReader(y))
+	assert.EqualError(t, err, "yaml: line 15: found character that cannot start any token")
+}
+
+// TestFormatFileOrDirectory_yamlExtFile verifies that FormatFileOrDirectory will format a file
+// with a .yaml extension.
+func TestFormatFileOrDirectory_yamlExtFile(t *testing.T) {
+	// write the unformatted file
+	f, err := ioutil.TempFile("", "yamlfmt*.yaml")
+	if !assert.NoError(t, err) {
+		return
+	}
+	defer os.Remove(f.Name())
+	err = ioutil.WriteFile(f.Name(), testyaml.UnformattedYaml1, 0600)
+	if !assert.NoError(t, err) {
+		return
+	}
+
+	// format the file
+	err = FormatFileOrDirectory(f.Name())
+	if !assert.NoError(t, err) {
+		return
+	}
+
+	// check the result is formatted
+	b, err := ioutil.ReadFile(f.Name())
+	if !assert.NoError(t, err) {
+		return
+	}
+	assert.Equal(t, string(testyaml.FormattedYaml1), string(b))
+}
+
+func TestFormatFileOrDirectory_multipleYamlEntries(t *testing.T) {
+	// write the unformatted file
+	f, err := ioutil.TempFile("", "yamlfmt*.yaml")
+	assert.NoError(t, err)
+	defer os.Remove(f.Name())
+	err = ioutil.WriteFile(f.Name(),
+		[]byte(string(testyaml.UnformattedYaml1)+"---\n"+string(testyaml.UnformattedYaml2)), 0600)
+	assert.NoError(t, err)
+
+	// format the file
+	err = FormatFileOrDirectory(f.Name())
+	assert.NoError(t, err)
+
+	// check the result is formatted
+	b, err := ioutil.ReadFile(f.Name())
+	assert.NoError(t, err)
+	assert.Equal(t, string(testyaml.FormattedYaml1)+"---\n"+string(testyaml.FormattedYaml2), string(b))
+}
+
+// TestFormatFileOrDirectory_ymlExtFile verifies that FormatFileOrDirectory will format a file
+// with a .yml extension.
+func TestFormatFileOrDirectory_ymlExtFile(t *testing.T) {
+	// write the unformatted file
+	f, err := ioutil.TempFile("", "yamlfmt*.yml")
+	assert.NoError(t, err)
+	defer os.Remove(f.Name())
+	err = ioutil.WriteFile(f.Name(), testyaml.UnformattedYaml1, 0600)
+	assert.NoError(t, err)
+
+	// format the file
+	err = FormatFileOrDirectory(f.Name())
+	assert.NoError(t, err)
+
+	// check the result is formatted
+	b, err := ioutil.ReadFile(f.Name())
+	assert.NoError(t, err)
+	assert.Equal(t, string(testyaml.FormattedYaml1), string(b))
+}
+
+// TestFormatFileOrDirectory_skipYamlExtFileWithJson verifies that the json content is formatted
+// as yaml
+func TestFormatFileOrDirectory_YamlExtFileWithJson(t *testing.T) {
+	// write the unformatted JSON file contents
+	f, err := ioutil.TempFile("", "yamlfmt*.yaml")
+	assert.NoError(t, err)
+	defer os.Remove(f.Name())
+	err = ioutil.WriteFile(f.Name(), testyaml.UnformattedJson1, 0600)
+	assert.NoError(t, err)
+
+	// format the file
+	err = FormatFileOrDirectory(f.Name())
+	assert.NoError(t, err)
+
+	// check the result is formatted as yaml
+	b, err := ioutil.ReadFile(f.Name())
+	assert.NoError(t, err)
+	assert.Equal(t, string(testyaml.FormattedYaml1), string(b))
+}
+
+// TestFormatFileOrDirectory_partialKubernetesYamlFile verifies that if a yaml file contains both
+// Kubernetes and non-Kubernetes documents, it will only format the Kubernetes documents
+func TestFormatFileOrDirectory_partialKubernetesYamlFile(t *testing.T) {
+	// write the unformatted file
+	f, err := ioutil.TempFile("", "yamlfmt*.yaml")
+	assert.NoError(t, err)
+	defer os.Remove(f.Name())
+	err = ioutil.WriteFile(f.Name(), []byte(string(testyaml.UnformattedYaml1)+`---
+status:
+  conditions:
+  - 3
+  - 1
+  - 2
+spec: a
+---
+`+string(testyaml.UnformattedYaml2)), 0600)
+	assert.NoError(t, err)
+
+	// format the file
+	err = FormatFileOrDirectory(f.Name())
+	assert.NoError(t, err)
+
+	// check the result is  NOT formatted
+	b, err := ioutil.ReadFile(f.Name())
+	assert.NoError(t, err)
+	assert.Equal(t, string(testyaml.FormattedYaml1)+`---
+status:
+  conditions:
+  - 3
+  - 1
+  - 2
+spec: a
+---
+`+string(testyaml.FormattedYaml2), string(b))
+}
+
+// TestFormatFileOrDirectory_nonKubernetesYamlFile verifies that if a yaml file does not contain
+// kubernetes
+func TestFormatFileOrDirectory_skipNonKubernetesYamlFile(t *testing.T) {
+	// write the unformatted JSON file contents
+	f, err := ioutil.TempFile("", "yamlfmt*.yaml")
+	assert.NoError(t, err)
+	defer os.Remove(f.Name())
+	err = ioutil.WriteFile(f.Name(), []byte(`
+status:
+  conditions:
+  - 3
+  - 1
+  - 2
+spec: a
+`), 0600)
+	assert.NoError(t, err)
+
+	// format the file
+	err = FormatFileOrDirectory(f.Name())
+	assert.NoError(t, err)
+
+	// check the result is formatted as yaml
+	b, err := ioutil.ReadFile(f.Name())
+	assert.NoError(t, err)
+	assert.Equal(t, `status:
+  conditions:
+  - 3
+  - 1
+  - 2
+spec: a
+`, string(b))
+}
+
+// TestFormatFileOrDirectory_jsonFile should not fmt the file even though it contains yaml.
+func TestFormatFileOrDirectory_skipJsonExtFile(t *testing.T) {
+	f, err := ioutil.TempFile("", "yamlfmt*.json")
+	assert.NoError(t, err)
+	defer os.Remove(f.Name())
+	err = ioutil.WriteFile(f.Name(), testyaml.UnformattedYaml1, 0600)
+	assert.NoError(t, err)
+
+	err = FormatFileOrDirectory(f.Name())
+	assert.NoError(t, err)
+
+	b, err := ioutil.ReadFile(f.Name())
+	assert.NoError(t, err)
+
+	assert.Equal(t, string(testyaml.UnformattedYaml1), string(b))
+}
+
+// TestFormatFileOrDirectory_directory verifies that yaml files will be formatted,
+// and other files will be ignored
+func TestFormatFileOrDirectory_directory(t *testing.T) {
+	d, err := ioutil.TempDir("", "yamlfmt")
+	assert.NoError(t, err)
+
+	err = os.Mkdir(filepath.Join(d, "config"), 0700)
+	assert.NoError(t, err)
+
+	err = ioutil.WriteFile(filepath.Join(d, "c1.yaml"), testyaml.UnformattedYaml1, 0600)
+	assert.NoError(t, err)
+
+	err = ioutil.WriteFile(filepath.Join(d, "config", "c2.yaml"), testyaml.UnformattedYaml2, 0600)
+	assert.NoError(t, err)
+
+	err = ioutil.WriteFile(filepath.Join(d, "README.md"), []byte(`# Markdown`), 0600)
+	assert.NoError(t, err)
+
+	err = FormatFileOrDirectory(d)
+	assert.NoError(t, err)
+
+	b, err := ioutil.ReadFile(filepath.Join(d, "c1.yaml"))
+	assert.NoError(t, err)
+	assert.Equal(t, string(testyaml.FormattedYaml1), string(b))
+
+	b, err = ioutil.ReadFile(filepath.Join(d, "config", "c2.yaml"))
+	assert.NoError(t, err)
+	assert.Equal(t, string(testyaml.FormattedYaml2), string(b))
+
+	b, err = ioutil.ReadFile(filepath.Join(d, "README.md"))
+	assert.NoError(t, err)
+	assert.Equal(t, `# Markdown`, string(b))
+
+	// verify no additional files were created
+	files := []string{
+		".", "c1.yaml", "README.md", "config", filepath.Join("config", "c2.yaml")}
+	err = filepath.Walk(d, func(path string, info os.FileInfo, err error) error {
+		assert.NoError(t, err)
+		path, err = filepath.Rel(d, path)
+		assert.NoError(t, err)
+		assert.Contains(t, files, path)
+		return nil
+	})
+	assert.NoError(t, err)
+}
+
+// TestFormatFileOrDirectory_trimWhiteSpace verifies that trailling and leading whitespace is
+// trimmed
+func TestFormatFileOrDirectory_trimWhiteSpace(t *testing.T) {
+	f, err := ioutil.TempFile("", "yamlfmt*.yaml")
+	assert.NoError(t, err)
+	defer os.Remove(f.Name())
+	err = ioutil.WriteFile(f.Name(), []byte("\n\n"+string(testyaml.UnformattedYaml1)+"\n\n"), 0600)
+	assert.NoError(t, err)
+
+	err = FormatFileOrDirectory(f.Name())
+	assert.NoError(t, err)
+
+	b, err := ioutil.ReadFile(f.Name())
+	assert.NoError(t, err)
+
+	assert.Equal(t, string(testyaml.FormattedYaml1), string(b))
+}

kyaml/kio/filters/grep.go

@@ -0,0 +1,117 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package filters
+
+import (
+	"regexp"
+	"strings"
+
+	"sigs.k8s.io/kustomize/kyaml/kio"
+	"sigs.k8s.io/kustomize/kyaml/yaml"
+)
+
+type GrepType int
+
+const (
+	Regexp GrepType = 1 << iota
+	GreaterThanEq
+	GreaterThan
+	LessThan
+	LessThanEq
+)
+
+// GrepFilter filters RNodes with a matching field
+type GrepFilter struct {
+	Path        []string `yaml:"path,omitempty"`
+	Value       string   `yaml:"value,omitempty"`
+	MatchType   GrepType `yaml:"matchType,omitempty"`
+	InvertMatch bool     `yaml:"invertMatch,omitempty"`
+	Compare     func(a, b string) (int, error)
+}
+
+var _ kio.Filter = GrepFilter{}
+
+func (f GrepFilter) Filter(input []*yaml.RNode) ([]*yaml.RNode, error) {
+	// compile the regular expression 1 time if we are matching using regex
+	var reg *regexp.Regexp
+	var err error
+	if f.MatchType == Regexp || f.MatchType == 0 {
+		reg, err = regexp.Compile(f.Value)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	var output kio.ResourceNodeSlice
+	for i := range input {
+		node := input[i]
+		val, err := node.Pipe(&yaml.PathMatcher{Path: f.Path})
+		if err != nil {
+			return nil, err
+		}
+		if val == nil || len(val.Content()) == 0 {
+			if f.InvertMatch {
+				output = append(output, input[i])
+			}
+			continue
+		}
+		found := false
+		err = val.VisitElements(func(elem *yaml.RNode) error {
+			// get the value
+			var str string
+			if f.MatchType == Regexp {
+				style := elem.YNode().Style
+				defer func() { elem.YNode().Style = style }()
+				elem.YNode().Style = yaml.FlowStyle
+				str, err = elem.String()
+				if err != nil {
+					return err
+				}
+				str = strings.TrimSpace(strings.Replace(str, `"`, "", -1))
+			} else {
+				// if not regexp, then it needs to parse into a quantity and comments will
+				// break that
+				str = elem.YNode().Value
+				if str == "" {
+					return nil
+				}
+			}
+
+			if f.MatchType == Regexp || f.MatchType == 0 {
+				if reg.MatchString(str) {
+					found = true
+				}
+				return nil
+			}
+
+			comp, err := f.Compare(str, f.Value)
+			if err != nil {
+				return err
+			}
+
+			if f.MatchType == GreaterThan && comp > 0 {
+				found = true
+			}
+			if f.MatchType == GreaterThanEq && comp >= 0 {
+				found = true
+			}
+			if f.MatchType == LessThan && comp < 0 {
+				found = true
+			}
+			if f.MatchType == LessThanEq && comp <= 0 {
+				found = true
+			}
+			return nil
+		})
+		if err != nil {
+			return nil, err
+		}
+		if found == f.InvertMatch {
+			continue
+		}
+
+		output = append(output, input[i])
+	}
+	return output, nil
+}

kyaml/kio/filters/grep_test.go

@@ -0,0 +1,157 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package filters_test
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"sigs.k8s.io/kustomize/kyaml/kio"
+	. "sigs.k8s.io/kustomize/kyaml/kio/filters"
+	"sigs.k8s.io/kustomize/kyaml/yaml"
+)
+
+func TestGrepFilter_Filter(t *testing.T) {
+	in := `kind: Deployment
+metadata:
+  labels:
+    app: nginx2
+  name: foo
+  annotations:
+    app: nginx2
+spec:
+  replicas: 1
+---
+kind: Deployment
+metadata:
+  labels:
+    app: nginx
+  annotations:
+    app: nginx
+  name: bar
+spec:
+  replicas: 3
+---
+kind: Service
+metadata:
+  name: foo
+  annotations:
+    app: nginx
+spec:
+  selector:
+    app: nginx
+`
+	out := &bytes.Buffer{}
+	err := kio.Pipeline{
+		Inputs:  []kio.Reader{&kio.ByteReader{Reader: bytes.NewBufferString(in)}},
+		Filters: []kio.Filter{GrepFilter{Path: []string{"metadata", "name"}, Value: "foo"}},
+		Outputs: []kio.Writer{kio.ByteWriter{Writer: out}},
+	}.Execute()
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
+
+	if !assert.Equal(t, `kind: Deployment
+metadata:
+  labels:
+    app: nginx2
+  name: foo
+  annotations:
+    app: nginx2
+spec:
+  replicas: 1
+---
+kind: Service
+metadata:
+  name: foo
+  annotations:
+    app: nginx
+spec:
+  selector:
+    app: nginx
+`, out.String()) {
+		t.FailNow()
+	}
+
+	out = &bytes.Buffer{}
+	err = kio.Pipeline{
+		Inputs:  []kio.Reader{&kio.ByteReader{Reader: bytes.NewBufferString(in)}},
+		Filters: []kio.Filter{GrepFilter{Path: []string{"kind"}, Value: "Deployment"}},
+		Outputs: []kio.Writer{kio.ByteWriter{Writer: out}},
+	}.Execute()
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
+	if !assert.Equal(t, `kind: Deployment
+metadata:
+  labels:
+    app: nginx2
+  name: foo
+  annotations:
+    app: nginx2
+spec:
+  replicas: 1
+---
+kind: Deployment
+metadata:
+  labels:
+    app: nginx
+  annotations:
+    app: nginx
+  name: bar
+spec:
+  replicas: 3
+`, out.String()) {
+		t.FailNow()
+	}
+
+	out = &bytes.Buffer{}
+	err = kio.Pipeline{
+		Inputs:  []kio.Reader{&kio.ByteReader{Reader: bytes.NewBufferString(in)}},
+		Filters: []kio.Filter{GrepFilter{Path: []string{"spec", "replicas"}, Value: "3"}},
+		Outputs: []kio.Writer{kio.ByteWriter{Writer: out}},
+	}.Execute()
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
+	if !assert.Equal(t, `kind: Deployment
+metadata:
+  labels:
+    app: nginx
+  annotations:
+    app: nginx
+  name: bar
+spec:
+  replicas: 3
+`, out.String()) {
+		t.FailNow()
+	}
+
+	out = &bytes.Buffer{}
+	err = kio.Pipeline{
+		Inputs:  []kio.Reader{&kio.ByteReader{Reader: bytes.NewBufferString(in)}},
+		Filters: []kio.Filter{GrepFilter{Path: []string{"spec", "not-present"}, Value: "3"}},
+		Outputs: []kio.Writer{kio.ByteWriter{Writer: out}},
+	}.Execute()
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
+	if !assert.Equal(t, ``, out.String()) {
+		t.FailNow()
+	}
+}
+
+func TestGrepFilter_init(t *testing.T) {
+	assert.Equal(t, GrepFilter{}, Filters["GrepFilter"]())
+}
+
+func TestGrepFilter_error(t *testing.T) {
+	v, err := GrepFilter{Path: []string{"metadata", "name"},
+		Value: "foo"}.Filter([]*yaml.RNode{{}})
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
+	assert.Nil(t, v)
+}

kyaml/kio/filters/merge.go

@@ -0,0 +1,76 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+// Package merge contains libraries for merging Resources and Patches
+package filters
+
+import (
+	"sigs.k8s.io/kustomize/kyaml/yaml"
+	"sigs.k8s.io/kustomize/kyaml/yaml/merge2"
+)
+
+// GrepFilter merges Resources with the Group/Version/Kind/Namespace/Name together using
+// a 2-way merge strategy.
+//
+// - Fields set to null in the source will be cleared from the destination
+// - Fields with matching keys will be merged recursively
+// - Lists with an associative key (e.g. name) will have their elements merged using the key
+// - List without an associative key will have the dest list replaced by the source list
+type MergeFilter struct {
+	Reverse bool
+}
+
+type mergeKey struct {
+	apiVersion string
+	kind       string
+	namespace  string
+	name       string
+}
+
+// GrepFilter implements kio.GrepFilter by merge Resources with the same G/V/K/NS/N
+func (c MergeFilter) Filter(input []*yaml.RNode) ([]*yaml.RNode, error) {
+	// invert the merge precedence
+	if c.Reverse {
+		for i, j := 0, len(input)-1; i < j; i, j = i+1, j-1 {
+			input[i], input[j] = input[j], input[i]
+		}
+	}
+
+	// index the Resources by G/V/K/NS/N
+	index := map[mergeKey][]*yaml.RNode{}
+	for i := range input {
+		meta, err := input[i].GetMeta()
+		if err != nil {
+			return nil, err
+		}
+		key := mergeKey{
+			apiVersion: meta.ApiVersion,
+			kind:       meta.Kind,
+			namespace:  meta.Namespace,
+			name:       meta.Name,
+		}
+		index[key] = append(index[key], input[i])
+	}
+
+	// merge each of the G/V/K/NS/N lists
+	var output []*yaml.RNode
+	var err error
+	for k := range index {
+		var merged *yaml.RNode
+		resources := index[k]
+		for i := range resources {
+			patch := resources[i]
+			if merged == nil {
+				// first resources, don't merge it
+				merged = resources[i]
+			} else {
+				merged, err = merge2.Merge(patch, merged)
+				if err != nil {
+					return nil, err
+				}
+			}
+		}
+		output = append(output, merged)
+	}
+	return output, nil
+}

kyaml/kio/filters/merge3.go

@@ -0,0 +1,167 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package filters
+
+import (
+	"fmt"
+
+	"sigs.k8s.io/kustomize/kyaml/kio"
+	"sigs.k8s.io/kustomize/kyaml/yaml"
+	"sigs.k8s.io/kustomize/kyaml/yaml/merge3"
+)
+
+const (
+	mergeSourceAnnotation = "kyaml.kustomize.dev/merge-source"
+	mergeSourceOriginal   = "original"
+	mergeSourceUpdated    = "updated"
+	mergeSourceDest       = "dest"
+)
+
+// Merge3 performs a 3-way merge on the original, updated, and destination packages.
+type Merge3 struct {
+	OriginalPath   string
+	UpdatedPath    string
+	DestPath       string
+	MatchFilesGlob []string
+}
+
+func (m Merge3) Merge() error {
+	// Read the destination package.  The ReadWriter will take take of deleting files
+	// for removed resources.
+	var inputs []kio.Reader
+	dest := &kio.LocalPackageReadWriter{
+		PackagePath:    m.DestPath,
+		MatchFilesGlob: m.MatchFilesGlob,
+		SetAnnotations: map[string]string{mergeSourceAnnotation: mergeSourceDest},
+	}
+	inputs = append(inputs, dest)
+
+	// Read the original package
+	inputs = append(inputs, kio.LocalPackageReader{
+		PackagePath:    m.OriginalPath,
+		MatchFilesGlob: m.MatchFilesGlob,
+		SetAnnotations: map[string]string{mergeSourceAnnotation: mergeSourceOriginal},
+	})
+
+	// Read the updated package
+	inputs = append(inputs, kio.LocalPackageReader{
+		PackagePath:    m.UpdatedPath,
+		MatchFilesGlob: m.MatchFilesGlob,
+		SetAnnotations: map[string]string{mergeSourceAnnotation: mergeSourceUpdated},
+	})
+
+	return kio.Pipeline{
+		Inputs:  inputs,
+		Filters: []kio.Filter{m, FormatFilter{}}, // format the merged output
+		Outputs: []kio.Writer{dest},
+	}.Execute()
+}
+
+// Filter combines Resources with the same GVK + N + NS into tuples, and then merges them
+func (m Merge3) Filter(nodes []*yaml.RNode) ([]*yaml.RNode, error) {
+	// index the nodes by their identity
+	tl := tuples{}
+	for i := range nodes {
+		if err := tl.add(nodes[i]); err != nil {
+			return nil, err
+		}
+	}
+
+	// iterate over the inputs, merging as needed
+	var output []*yaml.RNode
+	for i := range tl.list {
+		t := tl.list[i]
+		switch {
+		case t.original == nil && t.updated == nil && t.dest != nil:
+			// added locally -- keep dest
+			output = append(output, t.dest)
+		case t.original == nil && t.updated != nil && t.dest == nil:
+			// added in the update -- add update
+			output = append(output, t.updated)
+		case t.original != nil && t.updated == nil:
+			// deleted in the update
+		// don't include the resource in the output
+		case t.original != nil && t.dest == nil:
+			// deleted locally
+			// don't include the resource in the output
+		default:
+			// dest and updated are non-nil -- merge them
+			node, err := t.merge()
+			if err != nil {
+				return nil, err
+			}
+			if node != nil {
+				output = append(output, node)
+			}
+		}
+	}
+	return output, nil
+}
+
+// tuples combines nodes with the same GVK + N + NS
+type tuples struct {
+	list []*tuple
+}
+
+// add adds a node to the list, combining it with an existing matching Resource if found
+func (ts *tuples) add(node *yaml.RNode) error {
+	nodeMeta, err := node.GetMeta()
+	if err != nil {
+		return err
+	}
+	for i := range ts.list {
+		t := ts.list[i]
+		if t.meta.Name == nodeMeta.Name && t.meta.Namespace == nodeMeta.Namespace &&
+			t.meta.ApiVersion == nodeMeta.ApiVersion && t.meta.Kind == nodeMeta.Kind {
+			return t.add(node)
+		}
+	}
+	t := &tuple{meta: nodeMeta}
+	if err := t.add(node); err != nil {
+		return err
+	}
+	ts.list = append(ts.list, t)
+	return nil
+}
+
+// tuple wraps an original, updated, and dest tuple for a given Resource
+type tuple struct {
+	meta     yaml.ResourceMeta
+	original *yaml.RNode
+	updated  *yaml.RNode
+	dest     *yaml.RNode
+}
+
+// add sets the corresponding tuple field for the node
+func (t *tuple) add(node *yaml.RNode) error {
+	meta, err := node.GetMeta()
+	if err != nil {
+		return err
+	}
+	switch meta.Annotations[mergeSourceAnnotation] {
+	case mergeSourceDest:
+		if t.dest != nil {
+			return fmt.Errorf("dest source already specified")
+		}
+		t.dest = node
+	case mergeSourceOriginal:
+		if t.original != nil {
+			return fmt.Errorf("original source already specified")
+		}
+		t.original = node
+	case mergeSourceUpdated:
+		if t.updated != nil {
+			return fmt.Errorf("updated source already specified")
+		}
+		t.updated = node
+	default:
+		return fmt.Errorf("no source annotation for Resource")
+	}
+	return nil
+}
+
+// merge performs a 3-way merge on the tuple
+func (t *tuple) merge() (*yaml.RNode, error) {
+	return merge3.Merge(t.dest, t.original, t.updated)
+}

kyaml/kio/filters/merge3_test.go

@@ -0,0 +1,56 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package filters_test
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"runtime"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"sigs.k8s.io/kustomize/kyaml/copyutil"
+	"sigs.k8s.io/kustomize/kyaml/kio/filters"
+)
+
+func TestMerge3_Merge(t *testing.T) {
+	_, datadir, _, ok := runtime.Caller(0)
+	if !assert.True(t, ok) {
+		t.FailNow()
+	}
+	datadir = filepath.Join(filepath.Dir(datadir), "testdata")
+
+	// setup the local directory
+	dir, err := ioutil.TempDir("", "kyaml-test")
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
+	defer os.RemoveAll(dir)
+
+	if !assert.NoError(t, copyutil.CopyDir(
+		filepath.Join(datadir, "dataset1-localupdates"),
+		filepath.Join(dir, "dataset1"))) {
+		t.FailNow()
+	}
+
+	err = filters.Merge3{
+		OriginalPath: filepath.Join(datadir, "dataset1"),
+		UpdatedPath:  filepath.Join(datadir, "dataset1-remoteupdates"),
+		DestPath:     filepath.Join(dir, "dataset1"),
+	}.Merge()
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
+
+	diffs, err := copyutil.Diff(
+		filepath.Join(dir, "dataset1"),
+		filepath.Join(datadir, "dataset1-expected"))
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
+	if !assert.Empty(t, diffs.List()) {
+		t.FailNow()
+	}
+}

kyaml/kio/filters/modify.go

@@ -0,0 +1,4 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package filters

kyaml/kio/filters/stripcomments.go

@@ -0,0 +1,43 @@
+// Copyright 2019 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package filters
+
+import (
+	"sigs.k8s.io/kustomize/kyaml/kio"
+	"sigs.k8s.io/kustomize/kyaml/yaml"
+)
+
+type StripCommentsFilter struct{}
+
+var _ kio.Filter = StripCommentsFilter{}
+
+func (f StripCommentsFilter) Filter(slice []*yaml.RNode) ([]*yaml.RNode, error) {
+	for i := range slice {
+		stripComments(slice[i].YNode())
+	}
+	return slice, nil
+}
+
+func stripComments(node *yaml.Node) {
+	if node == nil {
+		return
+	}
+	node.HeadComment = ""
+	node.LineComment = ""
+	node.FootComment = ""
+	for i := range node.Content {
+		stripComments(node.Content[i])
+	}
+}

kyaml/kio/filters/testdata/dataset1-expected/java/java-configmap.resource.yaml

@@ -0,0 +1,11 @@
+# Copyright 2019 The Kubernetes Authors.
+# SPDX-License-Identifier: Apache-2.0
+# 
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: app-config
+  labels:
+    app.kubernetes.io/component: undefined
+    app.kubernetes.io/instance: undefined
+data: {}

kyaml/kio/filters/testdata/dataset1-expected/java/java-deployment.resource.yaml

@@ -0,0 +1,43 @@
+# Copyright 2019 The Kubernetes Authors.
+# SPDX-License-Identifier: Apache-2.0
+# 
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: app
+  labels:
+    app: java
+    new-local: label
+    new-remote: label
+spec:
+  replicas: 3
+  selector:
+    matchLabels:
+      app: java
+  template:
+    metadata:
+      labels:
+        app: java
+    spec:
+      restartPolicy: Always
+      containers:
+      - name: app
+        image: gcr.io/project/app:version
+        command:
+        - java
+        - -jar
+        - /app.jar
+        - otherstuff
+        args:
+        - foo
+        ports:
+        - containerPort: 8080
+        envFrom:
+        - configMapRef:
+            name: app-config
+        env:
+        - name: JAVA_OPTS
+          value: -XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap
+            -Djava.security.egd=file:/dev/./urandom
+        imagePullPolicy: Always
+  minReadySeconds: 20

kyaml/kio/filters/testdata/dataset1-expected/java/java-service.resource.yaml

@@ -0,0 +1,16 @@
+# Copyright 2019 The Kubernetes Authors.
+# SPDX-License-Identifier: Apache-2.0
+# 
+apiVersion: v1
+kind: Service
+metadata:
+  name: app
+  labels:
+    app: java
+spec:
+  selector:
+    app: java
+  ports:
+  - name: "8080"
+    port: 8080
+    targetPort: 8080

kyaml/kio/filters/testdata/dataset1-expected/mysql/mysql-configmap.resource.yaml

@@ -0,0 +1,23 @@
+# Copyright 2019 The Kubernetes Authors.
+# SPDX-License-Identifier: Apache-2.0
+# 
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: mysql
+  labels:
+    app: mysql
+  annotations:
+    file/index: "0"
+    file/path: mysql-configmap.resource.yaml
+    package/name: mysql
+    package/original-name: mysql
+data:
+  master.cnf: |
+    # Apply this config only on the master.
+    [mysqld]
+    log-bin
+  slave.cnf: |
+    # Apply this config only on slaves.
+    [mysqld]
+    super-read-only

kyaml/kio/filters/testdata/dataset1-expected/mysql/mysql-service.resource.yaml

@@ -0,0 +1,29 @@
+# Copyright 2019 The Kubernetes Authors.
+# SPDX-License-Identifier: Apache-2.0
+# 
+apiVersion: v1
+kind: Service
+metadata:
+  name: mysql
+  labels:
+    app: mysql
+spec:
+  selector:
+    app: mysql
+  ports:
+  - name: mysql
+    port: 3306
+  clusterIP: None
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: mysql-read
+  labels:
+    app: mysql
+spec:
+  selector:
+    app: mysql
+  ports:
+  - name: mysql
+    port: 3306

kyaml/kio/filters/testdata/dataset1-expected/mysql/mysql-statefulset.resource.yaml

@@ -0,0 +1,173 @@
+# Copyright 2019 The Kubernetes Authors.
+# SPDX-License-Identifier: Apache-2.0
+# 
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: mysql
+spec:
+  replicas: 3
+  selector:
+    matchLabels:
+      app: mysql
+  template:
+    metadata:
+      labels:
+        app: mysql
+    spec:
+      initContainers:
+      - name: init-mysql
+        image: mysql:5.7
+        command:
+        - bash
+        - -c
+        - |
+          set -ex
+          # Generate mysql server-id from pod ordinal index.
+          [[ `hostname` =~ -([0-9]+)$ ]] || exit 1
+          ordinal=${BASH_REMATCH[1]}
+          echo [mysqld] > /mnt/conf.d/server-id.cnf
+          # Add an offset to avoid reserved server-id=0 value.
+          echo server-id=$((100 + $ordinal)) >> /mnt/conf.d/server-id.cnf
+          # Copy appropriate conf.d files from config-map to emptyDir.
+          if [[ $ordinal -eq 0 ]]; then
+            cp /mnt/config-map/master.cnf /mnt/conf.d/
+          else
+            cp /mnt/config-map/slave.cnf /mnt/conf.d/
+          fi
+        volumeMounts:
+        - name: conf
+          mountPath: /mnt/conf.d
+        - name: config-map
+          mountPath: /mnt/config-map
+      - name: clone-mysql
+        image: gcr.io/google-samples/xtrabackup:1.0
+        command:
+        - bash
+        - -c
+        - |
+          set -ex
+          # Skip the clone if data already exists.
+          [[ -d /var/lib/mysql/mysql ]] && exit 0
+          # Skip the clone on master (ordinal index 0).
+          [[ `hostname` =~ -([0-9]+)$ ]] || exit 1
+          ordinal=${BASH_REMATCH[1]}
+          [[ $ordinal -eq 0 ]] && exit 0
+          # Clone data from previous peer.
+          ncat --recv-only mysql-$(($ordinal-1)).mysql 3307 | xbstream -x -C /var/lib/mysql
+          # Prepare the backup.
+          xtrabackup --prepare --target-dir=/var/lib/mysql
+        volumeMounts:
+        - name: data
+          mountPath: /var/lib/mysql
+          subPath: mysql
+        - name: conf
+          mountPath: /etc/mysql/conf.d
+      containers:
+      - name: mysql
+        image: mysql:5.7
+        ports:
+        - name: mysql
+          containerPort: 3306
+        env:
+        - name: MYSQL_ALLOW_EMPTY_PASSWORD
+          value: "1"
+        resources:
+          requests:
+            cpu: 500m
+            memory: 1Gi
+        volumeMounts:
+        - name: data
+          mountPath: /var/lib/mysql
+          subPath: mysql
+        - name: conf
+          mountPath: /etc/mysql/conf.d
+        livenessProbe:
+          exec:
+            command:
+            - mysqladmin
+            - ping
+          initialDelaySeconds: 30
+          periodSeconds: 10
+          timeoutSeconds: 5
+        readinessProbe:
+          exec:
+            command:
+            - mysql
+            - -h
+            - 127.0.0.1
+            - -e
+            - SELECT 1
+          initialDelaySeconds: 5
+          periodSeconds: 2
+          timeoutSeconds: 1
+      - name: xtrabackup
+        image: gcr.io/google-samples/xtrabackup:1.0
+        command:
+        - bash
+        - -c
+        - |
+          set -ex
+          cd /var/lib/mysql
+          # Determine binlog position of cloned data, if any.
+          if [[ -f xtrabackup_slave_info ]]; then
+            # XtraBackup already generated a partial "CHANGE MASTER TO" query
+            # because we're cloning from an existing slave.
+            mv xtrabackup_slave_info change_master_to.sql.in
+            # Ignore xtrabackup_binlog_info in this case (it's useless).
+            rm -f xtrabackup_binlog_info
+          elif [[ -f xtrabackup_binlog_info ]]; then
+            # We're cloning directly from master. Parse binlog position.
+            [[ `cat xtrabackup_binlog_info` =~ ^(.*?)[[:space:]]+(.*?)$ ]] || exit 1
+            rm xtrabackup_binlog_info
+            echo "CHANGE MASTER TO MASTER_LOG_FILE='${BASH_REMATCH[1]}',\
+                  MASTER_LOG_POS=${BASH_REMATCH[2]}" > change_master_to.sql.in
+          fi
+          # Check if we need to complete a clone by starting replication.
+          if [[ -f change_master_to.sql.in ]]; then
+            echo "Waiting for mysqld to be ready (accepting connections)"
+            until mysql -h 127.0.0.1 -e "SELECT 1"; do sleep 1; done
+            echo "Initializing replication from clone position"
+            # In case of container restart, attempt this at-most-once.
+            mv change_master_to.sql.in change_master_to.sql.orig
+            mysql -h 127.0.0.1 <<EOF
+          $(<change_master_to.sql.orig),
+            MASTER_HOST='mysql-0.mysql',
+            MASTER_USER='root',
+            MASTER_PASSWORD='',
+            MASTER_CONNECT_RETRY=10;
+          START SLAVE;
+          EOF
+          fi
+          # Start a server to send backups when requested by peers.
+          exec ncat --listen --keep-open --send-only --max-conns=1 3307 -c \
+            "xtrabackup --backup --slave-info --stream=xbstream --host=127.0.0.1 --user=root"
+        ports:
+        - name: xtrabackup
+          containerPort: 3307
+        resources:
+          requests:
+            cpu: 100m
+            memory: 100Mi
+        volumeMounts:
+        - name: data
+          mountPath: /var/lib/mysql
+          subPath: mysql
+        - name: conf
+          mountPath: /etc/mysql/conf.d
+      volumes:
+      - name: conf
+        emptyDir: {}
+      - name: config-map
+        configMap:
+          name: mysql
+  volumeClaimTemplates:
+  - metadata:
+      name: data
+    spec:
+      resources:
+        requests:
+          storage: 10Gi
+      accessModes:
+      - ReadWriteOnce
+  serviceName: mysql

kyaml/kio/filters/testdata/dataset1-expected/wordpress/wordpress-service.resource.yaml

@@ -0,0 +1,31 @@
+# Copyright 2019 The Kubernetes Authors.
+# SPDX-License-Identifier: Apache-2.0
+# 
+apiVersion: v1
+kind: Service
+metadata:
+  name: wordpress
+  labels:
+    app: wordpress
+spec:
+  type: NodePort
+  selector:
+    app: wordpress
+    tier: frontend
+  ports:
+  - port: 80
+    nodePort: 30000
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: wordpress-identity
+  labels:
+    app: wordpress-identity
+spec:
+  selector:
+    app: wordpress
+    tier: frontend
+  ports:
+  - port: 80
+    nodePort: 30000

kyaml/kio/filters/testdata/dataset1-expected/wordpress/wordpress-statefulset.resource.yaml

@@ -0,0 +1,42 @@
+# Copyright 2019 The Kubernetes Authors.
+# SPDX-License-Identifier: Apache-2.0
+# 
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: wordpress
+  labels:
+    app: wordpress
+spec:
+  selector:
+    matchLabels:
+      app: wordpress
+      tier: frontend
+  template:
+    metadata:
+      labels:
+        app: wordpress
+        tier: frontend
+    spec:
+      containers:
+      - name: wordpress
+        image: buddy/wordpress:latest
+        ports:
+        - name: wordpress
+          containerPort: 80
+        env:
+        - name: WORDPRESS_DB_HOST
+          value: wordpress-mysql
+        - name: WORDPRESS_DB_PASSWORD
+          valueFrom:
+            secretKeyRef:
+              name: mysql-pass
+              key: password
+        volumeMounts:
+        - name: wordpress-persistent-storage
+          mountPath: /var/www/html
+      volumes:
+      - name: wordpress-persistent-storage
+        persistentVolumeClaim:
+          claimName: wp-pv-claim
+  serviceName: wordpress-identity

kyaml/kio/filters/testdata/dataset1-localupdates/java/java-configmap.resource.yaml

@@ -0,0 +1,11 @@
+# Copyright 2019 The Kubernetes Authors.
+# SPDX-License-Identifier: Apache-2.0
+# 
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: app-config
+  labels:
+    app.kubernetes.io/component: undefined
+    app.kubernetes.io/instance: undefined
+data: {}

kyaml/kio/filters/testdata/dataset1-localupdates/java/java-deployment.resource.yaml

@@ -0,0 +1,41 @@
+# Copyright 2019 The Kubernetes Authors.
+# SPDX-License-Identifier: Apache-2.0
+# 
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: app
+  labels:
+    app: java
+    new-local: label
+spec:
+  replicas: 2
+  selector:
+    matchLabels:
+      app: java
+  template:
+    metadata:
+      labels:
+        app: java
+    spec:
+      restartPolicy: Always
+      containers:
+      - name: app
+        image: gcr.io/project/app:version
+        command:
+        - java
+        - -jar
+        - /app.jar
+        args:
+        - foo
+        ports:
+        - containerPort: 8080
+        envFrom:
+        - configMapRef:
+            name: app-config
+        env:
+        - name: JAVA_OPTS
+          value: -XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap
+            -Djava.security.egd=file:/dev/./urandom
+        imagePullPolicy: Always
+  minReadySeconds: 20

kyaml/kio/filters/testdata/dataset1-localupdates/java/java-service.resource.yaml

@@ -0,0 +1,16 @@
+# Copyright 2019 The Kubernetes Authors.
+# SPDX-License-Identifier: Apache-2.0
+# 
+apiVersion: v1
+kind: Service
+metadata:
+  name: app
+  labels:
+    app: java
+spec:
+  selector:
+    app: java
+  ports:
+  - name: "8080"
+    port: 8080
+    targetPort: 8080

kyaml/kio/filters/testdata/dataset1-localupdates/wordpress/wordpress-service.resource.yaml

@@ -0,0 +1,31 @@
+# Copyright 2019 The Kubernetes Authors.
+# SPDX-License-Identifier: Apache-2.0
+# 
+apiVersion: v1
+kind: Service
+metadata:
+  name: wordpress
+  labels:
+    app: wordpress
+spec:
+  type: NodePort
+  selector:
+    app: wordpress
+    tier: frontend
+  ports:
+  - port: 80
+    nodePort: 30000
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: wordpress-identity
+  labels:
+    app: wordpress-identity
+spec:
+  selector:
+    app: wordpress
+    tier: frontend
+  ports:
+  - port: 80
+    nodePort: 30000

kyaml/kio/filters/testdata/dataset1-localupdates/wordpress/wordpress-statefulset.resource.yaml

@@ -0,0 +1,42 @@
+# Copyright 2019 The Kubernetes Authors.
+# SPDX-License-Identifier: Apache-2.0
+# 
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: wordpress
+  labels:
+    app: wordpress
+spec:
+  selector:
+    matchLabels:
+      app: wordpress
+      tier: frontend
+  template:
+    metadata:
+      labels:
+        app: wordpress
+        tier: frontend
+    spec:
+      containers:
+      - name: wordpress
+        image: buddy/wordpress:latest
+        ports:
+        - name: wordpress
+          containerPort: 80
+        env:
+        - name: WORDPRESS_DB_HOST
+          value: wordpress-mysql
+        - name: WORDPRESS_DB_PASSWORD
+          valueFrom:
+            secretKeyRef:
+              name: mysql-pass
+              key: password
+        volumeMounts:
+        - name: wordpress-persistent-storage
+          mountPath: /var/www/html
+      volumes:
+      - name: wordpress-persistent-storage
+        persistentVolumeClaim:
+          claimName: wp-pv-claim
+  serviceName: wordpress-identity

kyaml/kio/filters/testdata/dataset1-remoteupdates/java/java-configmap.resource.yaml

@@ -0,0 +1,11 @@
+# Copyright 2019 The Kubernetes Authors.
+# SPDX-License-Identifier: Apache-2.0
+# 
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: app-config
+  labels:
+    app.kubernetes.io/component: undefined
+    app.kubernetes.io/instance: undefined
+data: {}

kyaml/kio/filters/testdata/dataset1-remoteupdates/java/java-deployment.resource.yaml

@@ -0,0 +1,40 @@
+# Copyright 2019 The Kubernetes Authors.
+# SPDX-License-Identifier: Apache-2.0
+# 
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: app
+  labels:
+    app: java
+    new-remote: label
+spec:
+  replicas: 3
+  selector:
+    matchLabels:
+      app: java
+  template:
+    metadata:
+      labels:
+        app: java
+    spec:
+      restartPolicy: Always
+      containers:
+      - name: app
+        image: gcr.io/project/app:version
+        command:
+        - java
+        - -jar
+        - /app.jar
+        - otherstuff
+        ports:
+        - containerPort: 8080
+        envFrom:
+        - configMapRef:
+            name: app-config
+        env:
+        - name: JAVA_OPTS
+          value: -XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap
+            -Djava.security.egd=file:/dev/./urandom
+        imagePullPolicy: Always
+  minReadySeconds: 5

kyaml/kio/filters/testdata/dataset1-remoteupdates/java/java-service.resource.yaml

@@ -0,0 +1,16 @@
+# Copyright 2019 The Kubernetes Authors.
+# SPDX-License-Identifier: Apache-2.0
+# 
+apiVersion: v1
+kind: Service
+metadata:
+  name: app
+  labels:
+    app: java
+spec:
+  selector:
+    app: java
+  ports:
+  - name: "8080"
+    port: 8080
+    targetPort: 8080

kyaml/kio/filters/testdata/dataset1-remoteupdates/mysql/mysql-configmap.resource.yaml

@@ -0,0 +1,23 @@
+# Copyright 2019 The Kubernetes Authors.
+# SPDX-License-Identifier: Apache-2.0
+# 
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: mysql
+  labels:
+    app: mysql
+  annotations:
+    file/index: "0"
+    file/path: mysql-configmap.resource.yaml
+    package/name: mysql
+    package/original-name: mysql
+data:
+  master.cnf: |
+    # Apply this config only on the master.
+    [mysqld]
+    log-bin
+  slave.cnf: |
+    # Apply this config only on slaves.
+    [mysqld]
+    super-read-only

kyaml/kio/filters/testdata/dataset1-remoteupdates/mysql/mysql-service.resource.yaml

@@ -0,0 +1,29 @@
+# Copyright 2019 The Kubernetes Authors.
+# SPDX-License-Identifier: Apache-2.0
+# 
+apiVersion: v1
+kind: Service
+metadata:
+  name: mysql
+  labels:
+    app: mysql
+spec:
+  selector:
+    app: mysql
+  ports:
+  - name: mysql
+    port: 3306
+  clusterIP: None
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: mysql-read
+  labels:
+    app: mysql
+spec:
+  selector:
+    app: mysql
+  ports:
+  - name: mysql
+    port: 3306

kyaml/kio/filters/testdata/dataset1-remoteupdates/mysql/mysql-statefulset.resource.yaml

@@ -0,0 +1,173 @@
+# Copyright 2019 The Kubernetes Authors.
+# SPDX-License-Identifier: Apache-2.0
+# 
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: mysql
+spec:
+  replicas: 3
+  selector:
+    matchLabels:
+      app: mysql
+  template:
+    metadata:
+      labels:
+        app: mysql
+    spec:
+      initContainers:
+      - name: init-mysql
+        image: mysql:5.7
+        command:
+        - bash
+        - -c
+        - |
+          set -ex
+          # Generate mysql server-id from pod ordinal index.
+          [[ `hostname` =~ -([0-9]+)$ ]] || exit 1
+          ordinal=${BASH_REMATCH[1]}
+          echo [mysqld] > /mnt/conf.d/server-id.cnf
+          # Add an offset to avoid reserved server-id=0 value.
+          echo server-id=$((100 + $ordinal)) >> /mnt/conf.d/server-id.cnf
+          # Copy appropriate conf.d files from config-map to emptyDir.
+          if [[ $ordinal -eq 0 ]]; then
+            cp /mnt/config-map/master.cnf /mnt/conf.d/
+          else
+            cp /mnt/config-map/slave.cnf /mnt/conf.d/
+          fi
+        volumeMounts:
+        - name: conf
+          mountPath: /mnt/conf.d
+        - name: config-map
+          mountPath: /mnt/config-map
+      - name: clone-mysql
+        image: gcr.io/google-samples/xtrabackup:1.0
+        command:
+        - bash
+        - -c
+        - |
+          set -ex
+          # Skip the clone if data already exists.
+          [[ -d /var/lib/mysql/mysql ]] && exit 0
+          # Skip the clone on master (ordinal index 0).
+          [[ `hostname` =~ -([0-9]+)$ ]] || exit 1
+          ordinal=${BASH_REMATCH[1]}
+          [[ $ordinal -eq 0 ]] && exit 0
+          # Clone data from previous peer.
+          ncat --recv-only mysql-$(($ordinal-1)).mysql 3307 | xbstream -x -C /var/lib/mysql
+          # Prepare the backup.
+          xtrabackup --prepare --target-dir=/var/lib/mysql
+        volumeMounts:
+        - name: data
+          mountPath: /var/lib/mysql
+          subPath: mysql
+        - name: conf
+          mountPath: /etc/mysql/conf.d
+      containers:
+      - name: mysql
+        image: mysql:5.7
+        ports:
+        - name: mysql
+          containerPort: 3306
+        env:
+        - name: MYSQL_ALLOW_EMPTY_PASSWORD
+          value: "1"
+        resources:
+          requests:
+            cpu: 500m
+            memory: 1Gi
+        volumeMounts:
+        - name: data
+          mountPath: /var/lib/mysql
+          subPath: mysql
+        - name: conf
+          mountPath: /etc/mysql/conf.d
+        livenessProbe:
+          exec:
+            command:
+            - mysqladmin
+            - ping
+          initialDelaySeconds: 30
+          periodSeconds: 10
+          timeoutSeconds: 5
+        readinessProbe:
+          exec:
+            command:
+            - mysql
+            - -h
+            - 127.0.0.1
+            - -e
+            - SELECT 1
+          initialDelaySeconds: 5
+          periodSeconds: 2
+          timeoutSeconds: 1
+      - name: xtrabackup
+        image: gcr.io/google-samples/xtrabackup:1.0
+        command:
+        - bash
+        - -c
+        - |
+          set -ex
+          cd /var/lib/mysql
+          # Determine binlog position of cloned data, if any.
+          if [[ -f xtrabackup_slave_info ]]; then
+            # XtraBackup already generated a partial "CHANGE MASTER TO" query
+            # because we're cloning from an existing slave.
+            mv xtrabackup_slave_info change_master_to.sql.in
+            # Ignore xtrabackup_binlog_info in this case (it's useless).
+            rm -f xtrabackup_binlog_info
+          elif [[ -f xtrabackup_binlog_info ]]; then
+            # We're cloning directly from master. Parse binlog position.
+            [[ `cat xtrabackup_binlog_info` =~ ^(.*?)[[:space:]]+(.*?)$ ]] || exit 1
+            rm xtrabackup_binlog_info
+            echo "CHANGE MASTER TO MASTER_LOG_FILE='${BASH_REMATCH[1]}',\
+                  MASTER_LOG_POS=${BASH_REMATCH[2]}" > change_master_to.sql.in
+          fi
+          # Check if we need to complete a clone by starting replication.
+          if [[ -f change_master_to.sql.in ]]; then
+            echo "Waiting for mysqld to be ready (accepting connections)"
+            until mysql -h 127.0.0.1 -e "SELECT 1"; do sleep 1; done
+            echo "Initializing replication from clone position"
+            # In case of container restart, attempt this at-most-once.
+            mv change_master_to.sql.in change_master_to.sql.orig
+            mysql -h 127.0.0.1 <<EOF
+          $(<change_master_to.sql.orig),
+            MASTER_HOST='mysql-0.mysql',
+            MASTER_USER='root',
+            MASTER_PASSWORD='',
+            MASTER_CONNECT_RETRY=10;
+          START SLAVE;
+          EOF
+          fi
+          # Start a server to send backups when requested by peers.
+          exec ncat --listen --keep-open --send-only --max-conns=1 3307 -c \
+            "xtrabackup --backup --slave-info --stream=xbstream --host=127.0.0.1 --user=root"
+        ports:
+        - name: xtrabackup
+          containerPort: 3307
+        resources:
+          requests:
+            cpu: 100m
+            memory: 100Mi
+        volumeMounts:
+        - name: data
+          mountPath: /var/lib/mysql
+          subPath: mysql
+        - name: conf
+          mountPath: /etc/mysql/conf.d
+      volumes:
+      - name: conf
+        emptyDir: {}
+      - name: config-map
+        configMap:
+          name: mysql
+  volumeClaimTemplates:
+  - metadata:
+      name: data
+    spec:
+      resources:
+        requests:
+          storage: 10Gi
+      accessModes:
+      - ReadWriteOnce
+  serviceName: mysql

kyaml/kio/filters/testdata/dataset1/java/java-configmap.resource.yaml

@@ -0,0 +1,11 @@
+# Copyright 2019 The Kubernetes Authors.
+# SPDX-License-Identifier: Apache-2.0
+# 
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: app-config
+  labels:
+    app.kubernetes.io/component: undefined
+    app.kubernetes.io/instance: undefined
+data: {}

kyaml/kio/filters/testdata/dataset1/java/java-deployment.resource.yaml

@@ -0,0 +1,38 @@
+# Copyright 2019 The Kubernetes Authors.
+# SPDX-License-Identifier: Apache-2.0
+# 
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: app
+  labels:
+    app: java
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: java
+  template:
+    metadata:
+      labels:
+        app: java
+    spec:
+      restartPolicy: Always
+      containers:
+      - name: app
+        image: gcr.io/project/app:version
+        command:
+        - java
+        - -jar
+        - /app.jar
+        ports:
+        - containerPort: 8080
+        envFrom:
+        - configMapRef:
+            name: app-config
+        env:
+        - name: JAVA_OPTS
+          value: -XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap
+            -Djava.security.egd=file:/dev/./urandom
+        imagePullPolicy: Always
+  minReadySeconds: 5

kyaml/kio/filters/testdata/dataset1/java/java-service.resource.yaml

@@ -0,0 +1,16 @@
+# Copyright 2019 The Kubernetes Authors.
+# SPDX-License-Identifier: Apache-2.0
+# 
+apiVersion: v1
+kind: Service
+metadata:
+  name: app
+  labels:
+    app: java
+spec:
+  selector:
+    app: java
+  ports:
+  - name: "8080"
+    port: 8080
+    targetPort: 8080

kyaml/kio/filters/testdata/dataset1/rails/rails-configmap.resource.yaml

@@ -0,0 +1,11 @@
+# Copyright 2019 The Kubernetes Authors.
+# SPDX-License-Identifier: Apache-2.0
+# 
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: rails-app-config
+  labels:
+    app.kubernetes.io/component: undefined
+    app.kubernetes.io/instance: undefined
+data: {}

kyaml/kio/filters/testdata/dataset1/rails/rails-deployment.resource.yaml

@@ -0,0 +1,36 @@
+# Copyright 2019 The Kubernetes Authors.
+# SPDX-License-Identifier: Apache-2.0
+# 
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: rails-app
+  labels:
+    app: rails
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: rails
+  template:
+    metadata:
+      labels:
+        app: rails
+    spec:
+      restartPolicy: Always
+      containers:
+      - name: rails
+        image: gcr.io/project/app:version
+        command:
+        - rails
+        ports:
+        - containerPort: 8080
+        envFrom:
+        - configMapRef:
+            name: rails-app-config
+        env:
+        - name: JAVA_OPTS
+          value: -XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap
+            -Djava.security.egd=file:/dev/./urandom
+        imagePullPolicy: Always
+  minReadySeconds: 5

kyaml/kio/filters/testdata/dataset1/rails/rails-service.resource.yaml

@@ -0,0 +1,16 @@
+# Copyright 2019 The Kubernetes Authors.
+# SPDX-License-Identifier: Apache-2.0
+# 
+apiVersion: v1
+kind: Service
+metadata:
+  name: rails-app
+  labels:
+    app: rails
+spec:
+  selector:
+    app: rails
+  ports:
+  - name: "8080"
+    port: 8080
+    targetPort: 8080

kyaml/kio/filters/testyaml/testyaml.go

@@ -0,0 +1,57 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+// Package testyaml contains test data and libraries for formatting
+// Kubernetes configuration
+package testyaml
+
+var UnformattedYaml1 = []byte(`
+spec: a
+status:
+  conditions:
+  - 3
+  - 1
+  - 2
+apiVersion: example.com/v1beta1
+kind: MyType
+`)
+
+var UnformattedYaml2 = []byte(`
+spec2: a
+status2:
+  conditions:
+  - 3
+  - 1
+  - 2
+apiVersion: example.com/v1beta1
+kind: MyType2
+`)
+
+var UnformattedJson1 = []byte(`
+{
+  "spec": "a",
+  "status": {"conditions": [3, 1, 2]},
+  "apiVersion": "example.com/v1beta1",
+  "kind": "MyType"
+}
+`)
+
+var FormattedYaml1 = []byte(`apiVersion: example.com/v1beta1
+kind: MyType
+spec: a
+status:
+  conditions:
+  - 3
+  - 1
+  - 2
+`)
+
+var FormattedYaml2 = []byte(`apiVersion: example.com/v1beta1
+kind: MyType2
+spec2: a
+status2:
+  conditions:
+  - 3
+  - 1
+  - 2
+`)

kyaml/kio/kio.go

@@ -0,0 +1,106 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+// Package kio contains low-level libraries for reading, modifying and writing
+// Resource Configuration and packages.
+package kio
+
+import (
+	"sigs.k8s.io/kustomize/kyaml/yaml"
+)
+
+// Reader reads ResourceNodes. Analogous to io.Reader.
+type Reader interface {
+	Read() ([]*yaml.RNode, error)
+}
+
+// ResourceNodeSlice is a collection of ResourceNodes.
+// While ResourceNodeSlice has no inherent constraints on ordering or uniqueness, specific
+// Readers, Filters or Writers may have constraints.
+type ResourceNodeSlice []*yaml.RNode
+
+var _ Reader = ResourceNodeSlice{}
+
+func (o ResourceNodeSlice) Read() ([]*yaml.RNode, error) {
+	return o, nil
+}
+
+// Writer writes ResourceNodes. Analogous to io.Writer.
+type Writer interface {
+	Write([]*yaml.RNode) error
+}
+
+type WriterFunc func([]*yaml.RNode) error
+
+func (fn WriterFunc) Write(o []*yaml.RNode) error {
+	return fn(o)
+}
+
+// GrepFilter modifies a collection of Resource Configuration by returning the modified slice.
+// When possible, Filters should be serializable to yaml so that they can be described
+// declaratively as data.
+//
+// Analogous to http://www.linfo.org/filters.html
+type Filter interface {
+	Filter([]*yaml.RNode) ([]*yaml.RNode, error)
+}
+
+// FilterFunc can be used to implement GrepFilter by defining a function.
+type FilterFunc func([]*yaml.RNode) ([]*yaml.RNode, error)
+
+func (fn FilterFunc) Filter(o []*yaml.RNode) ([]*yaml.RNode, error) {
+	return fn(o)
+}
+
+// Pipeline reads Resource Configuration from a set of Inputs, applies some
+// transformations, and writes the results to a set of Outputs.
+//
+// Analogous to http://www.linfo.org/pipes.html
+type Pipeline struct {
+	// Inputs provide sources for Resource Configuration to be read.
+	Inputs []Reader `yaml:"inputs,omitempty"`
+
+	// Filters are transformations applied to the Resource Configuration.
+	// They are applied in the order they are specified.
+	// Analogous to http://www.linfo.org/filters.html
+	Filters []Filter `yaml:"filters,omitempty"`
+
+	// Outputs are where the transformed Resource Configuration is written.
+	Outputs []Writer `yaml:"outputs,omitempty"`
+}
+
+// Execute implements the Pipeline pipeline.
+func (p Pipeline) Execute() error {
+	var result []*yaml.RNode
+
+	// read from the inputs
+	for _, i := range p.Inputs {
+		nodes, err := i.Read()
+		if err != nil {
+			return err
+		}
+		result = append(result, nodes...)
+	}
+	if len(result) == 0 {
+		// no inputs to operate on
+		return nil
+	}
+
+	// apply operations
+	var err error
+	for i := range p.Filters {
+		op := p.Filters[i]
+		result, err = op.Filter(result)
+		if len(result) == 0 || err != nil {
+			return err
+		}
+	}
+
+	// write to the outputs
+	for _, o := range p.Outputs {
+		if err := o.Write(result); err != nil {
+			return err
+		}
+	}
+	return nil
+}

kyaml/kio/kio_test.go

@@ -0,0 +1,28 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package kio_test
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	. "sigs.k8s.io/kustomize/kyaml/kio"
+)
+
+func TestPipe(t *testing.T) {
+	p := Pipeline{
+		Inputs:  []Reader{},
+		Filters: []Filter{},
+		Outputs: []Writer{},
+	}
+
+	err := p.Execute()
+	if !assert.NoError(t, err) {
+		assert.FailNow(t, err.Error())
+	}
+}
+
+func TestSlice_Write(t *testing.T) {
+
+}

kyaml/kio/kioutil/kioutil.go

@@ -0,0 +1,111 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package kioutil
+
+import (
+	"fmt"
+	"sort"
+	"strconv"
+
+	"sigs.k8s.io/kustomize/kyaml/yaml"
+)
+
+type AnnotationKey = string
+
+const (
+	// IndexAnnotation records the index of a specific resource in a file or input stream.
+	IndexAnnotation AnnotationKey = "kyaml.kustomize.dev/kio/index"
+
+	// PathAnnotation records the path to the file the Resource was read from
+	PathAnnotation AnnotationKey = "kyaml.kustomize.dev/kio/path"
+
+	// PackageAnnotation records the name of the package the Resource was read from
+	PackageAnnotation AnnotationKey = "kyaml.kustomize.dev/kio/package"
+)
+
+func GetFileAnnotations(rn *yaml.RNode) (string, string, error) {
+	meta, err := rn.GetMeta()
+	if err != nil {
+		return "", "", err
+	}
+	path := meta.Annotations[PathAnnotation]
+	index := meta.Annotations[IndexAnnotation]
+	return path, index, nil
+}
+
+// ErrorIfMissingAnnotation validates the provided annotations are present on the given resources
+func ErrorIfMissingAnnotation(nodes []*yaml.RNode, keys ...AnnotationKey) error {
+	for _, key := range keys {
+		for _, node := range nodes {
+			val, err := node.Pipe(yaml.GetAnnotation(key))
+			if err != nil {
+				return err
+			}
+			if val == nil {
+				return fmt.Errorf("missing package annotation %s", key)
+			}
+		}
+	}
+	return nil
+}
+
+// SortNodes sorts nodes in place:
+// - by PathAnnotation annotation
+// - by IndexAnnotation annotation
+func SortNodes(nodes []*yaml.RNode) error {
+	var err error
+	// use stable sort to keep ordering of equal elements
+	sort.SliceStable(nodes, func(i, j int) bool {
+		if err != nil {
+			return false
+		}
+		var iMeta, jMeta yaml.ResourceMeta
+		if iMeta, _ = nodes[i].GetMeta(); err != nil {
+			return false
+		}
+		if jMeta, _ = nodes[j].GetMeta(); err != nil {
+			return false
+		}
+
+		iValue := iMeta.Annotations[PathAnnotation]
+		jValue := jMeta.Annotations[PathAnnotation]
+		if iValue != jValue {
+			return iValue < jValue
+		}
+
+		iValue = iMeta.Annotations[IndexAnnotation]
+		jValue = jMeta.Annotations[IndexAnnotation]
+
+		// put resource config without an index first
+		if iValue == jValue {
+			return false
+		}
+		if iValue == "" {
+			return true
+		}
+		if jValue == "" {
+			return false
+		}
+
+		// sort by index
+		var iIndex, jIndex int
+		iIndex, err = strconv.Atoi(iValue)
+		if err != nil {
+			err = fmt.Errorf("unable to parse kyaml.kustomize.dev/kio/index %s :%v", iValue, err)
+			return false
+		}
+		jIndex, err = strconv.Atoi(jValue)
+		if err != nil {
+			err = fmt.Errorf("unable to parse kyaml.kustomize.dev/kio/index %s :%v", jValue, err)
+			return false
+		}
+		if iIndex != jIndex {
+			return iValue < jValue
+		}
+
+		// elements are equal
+		return false
+	})
+	return err
+}

kyaml/kio/pkgio_reader.go

@@ -0,0 +1,281 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package kio
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"sigs.k8s.io/kustomize/kyaml/kio/kioutil"
+	"sigs.k8s.io/kustomize/kyaml/sets"
+	"sigs.k8s.io/kustomize/kyaml/yaml"
+)
+
+// requiredResourcePackageAnnotations are annotations that are required to write resources back to
+// files.
+var requiredResourcePackageAnnotations = []string{kioutil.IndexAnnotation, kioutil.PathAnnotation}
+
+type PackageBuffer struct {
+	Nodes []*yaml.RNode
+}
+
+func (r *PackageBuffer) Read() ([]*yaml.RNode, error) {
+	return r.Nodes, nil
+}
+
+func (r *PackageBuffer) Write(nodes []*yaml.RNode) error {
+	r.Nodes = nodes
+	return nil
+}
+
+type LocalPackageReadWriter struct {
+	Kind string `yaml:"kind,omitempty"`
+
+	KeepReaderAnnotations bool `yaml:"keepReaderAnnotations,omitempty"`
+
+	// PackagePath is the path to the package directory.
+	PackagePath string `yaml:"path,omitempty"`
+
+	// PackageFileName is the name of file containing package metadata.
+	// It will be used to identify package.
+	PackageFileName string `yaml:"packageFileName,omitempty"`
+
+	// MatchFilesGlob configures Read to only read Resources from files matching any of the
+	// provided patterns.
+	// Defaults to ["*.yaml", "*.yml"] if empty.  To match all files specify ["*"].
+	MatchFilesGlob []string `yaml:"matchFilesGlob,omitempty"`
+
+	// IncludeSubpackages will configure Read to read Resources from subpackages.
+	// Subpackages are identified by presence of PackageFileName.
+	IncludeSubpackages bool `yaml:"includeSubpackages,omitempty"`
+
+	// ErrorIfNonResources will configure Read to throw an error if yaml missing missing
+	// apiVersion or kind is read.
+	ErrorIfNonResources bool `yaml:"errorIfNonResources,omitempty"`
+
+	// OmitReaderAnnotations will cause the reader to skip annotating Resources with the file
+	// path and mode.
+	OmitReaderAnnotations bool `yaml:"omitReaderAnnotations,omitempty"`
+
+	// SetAnnotations are annotations to set on the Resources as they are read.
+	SetAnnotations map[string]string `yaml:"setAnnotations,omitempty"`
+
+	// NoDeleteFiles if set to true, LocalPackageReadWriter won't delete any files
+	NoDeleteFiles bool `yaml:"noDeleteFiles,omitempty"`
+
+	files sets.String
+}
+
+func (r *LocalPackageReadWriter) Read() ([]*yaml.RNode, error) {
+	nodes, err := LocalPackageReader{
+		PackagePath:         r.PackagePath,
+		MatchFilesGlob:      r.MatchFilesGlob,
+		IncludeSubpackages:  r.IncludeSubpackages,
+		ErrorIfNonResources: r.ErrorIfNonResources,
+		SetAnnotations:      r.SetAnnotations,
+	}.Read()
+	if err != nil {
+		return nil, err
+	}
+	// keep track of all the files
+	if !r.NoDeleteFiles {
+		r.files, err = r.getFiles(nodes)
+		if err != nil {
+			return nil, err
+		}
+	}
+	return nodes, nil
+}
+
+func (r *LocalPackageReadWriter) Write(nodes []*yaml.RNode) error {
+	newFiles, err := r.getFiles(nodes)
+	if err != nil {
+		return err
+	}
+	var clear []string
+	for k := range r.SetAnnotations {
+		clear = append(clear, k)
+	}
+	err = LocalPackageWriter{
+		PackagePath:           r.PackagePath,
+		ClearAnnotations:      clear,
+		KeepReaderAnnotations: r.KeepReaderAnnotations,
+	}.Write(nodes)
+	if err != nil {
+		return err
+	}
+	deleteFiles := r.files.Difference(newFiles)
+	for f := range deleteFiles {
+		if err = os.Remove(filepath.Join(r.PackagePath, f)); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (r *LocalPackageReadWriter) getFiles(nodes []*yaml.RNode) (sets.String, error) {
+	val := sets.String{}
+	for _, n := range nodes {
+		path, _, err := kioutil.GetFileAnnotations(n)
+		if err != nil {
+			return nil, err
+		}
+		val.Insert(path)
+	}
+	return val, nil
+}
+
+// LocalPackageReader reads ResourceNodes from a local package.
+type LocalPackageReader struct {
+	Kind string `yaml:"kind,omitempty"`
+
+	// PackagePath is the path to the package directory.
+	PackagePath string `yaml:"path,omitempty"`
+
+	// PackageFileName is the name of file containing package metadata.
+	// It will be used to identify package.
+	PackageFileName string `yaml:"packageFileName,omitempty"`
+
+	// MatchFilesGlob configures Read to only read Resources from files matching any of the
+	// provided patterns.
+	// Defaults to ["*.yaml", "*.yml"] if empty.  To match all files specify ["*"].
+	MatchFilesGlob []string `yaml:"matchFilesGlob,omitempty"`
+
+	// IncludeSubpackages will configure Read to read Resources from subpackages.
+	// Subpackages are identified by presence of PackageFileName.
+	IncludeSubpackages bool `yaml:"includeSubpackages,omitempty"`
+
+	// ErrorIfNonResources will configure Read to throw an error if yaml missing missing
+	// apiVersion or kind is read.
+	ErrorIfNonResources bool `yaml:"errorIfNonResources,omitempty"`
+
+	// OmitReaderAnnotations will cause the reader to skip annotating Resources with the file
+	// path and mode.
+	OmitReaderAnnotations bool `yaml:"omitReaderAnnotations,omitempty"`
+
+	// SetAnnotations are annotations to set on the Resources as they are read.
+	SetAnnotations map[string]string `yaml:"setAnnotations,omitempty"`
+}
+
+var _ Reader = LocalPackageReader{}
+
+var defaultMatch = []string{"*.yaml", "*.yml"}
+
+// Read reads the Resources.
+func (r LocalPackageReader) Read() ([]*yaml.RNode, error) {
+	if r.PackagePath == "" {
+		return nil, fmt.Errorf("must specify package path")
+	}
+	if len(r.MatchFilesGlob) == 0 {
+		r.MatchFilesGlob = defaultMatch
+	}
+
+	var operand ResourceNodeSlice
+	var pathRelativeTo string
+	r.PackagePath = filepath.Clean(r.PackagePath)
+	err := filepath.Walk(r.PackagePath, func(
+		path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+
+		// is this the user specified path?
+		if path == r.PackagePath {
+			if info.IsDir() {
+				// skip the root package directory
+				pathRelativeTo = r.PackagePath
+				return nil
+			}
+
+			// user specified path is a file rather than a directory.
+			// make its path relative to its parent so it can be written to another file.
+			pathRelativeTo = filepath.Dir(r.PackagePath)
+		}
+
+		// check if we should skip the directory or file
+		if info.IsDir() {
+			return r.shouldSkipDir(path, info)
+		}
+		if match, err := r.shouldSkipFile(path, info); err != nil {
+			return err
+		} else if !match {
+			// skip this file
+			return nil
+		}
+
+		// get the relative path to file within the package so we can write the files back out
+		// to another location.
+		path, err = filepath.Rel(pathRelativeTo, path)
+		if err != nil {
+			return err
+		}
+
+		r.initReaderAnnotations(path, info)
+		nodes, err := r.readFile(filepath.Join(pathRelativeTo, path), info)
+		if err != nil {
+			return err
+		}
+		operand = append(operand, nodes...)
+		return nil
+	})
+	return operand, err
+}
+
+// readFile reads the ResourceNodes from a file
+func (r *LocalPackageReader) readFile(path string, info os.FileInfo) ([]*yaml.RNode, error) {
+	f, err := os.Open(path)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+	rr := &ByteReader{
+		Reader:                f,
+		OmitReaderAnnotations: r.OmitReaderAnnotations,
+		SetAnnotations:        r.SetAnnotations,
+	}
+	return rr.Read()
+}
+
+// shouldSkipFile returns true if the file should be skipped
+func (r *LocalPackageReader) shouldSkipFile(path string, info os.FileInfo) (bool, error) {
+	// check if the files are in scope
+	for _, g := range r.MatchFilesGlob {
+		if match, err := filepath.Match(g, info.Name()); err != nil {
+			return false, err
+		} else if match {
+			return true, nil
+		}
+	}
+	return false, nil
+}
+
+// initReaderAnnotations adds the LocalPackageReader Annotations to r.SetAnnotations
+func (r *LocalPackageReader) initReaderAnnotations(path string, info os.FileInfo) {
+	if r.SetAnnotations == nil {
+		r.SetAnnotations = map[string]string{}
+	}
+	if !r.OmitReaderAnnotations {
+		r.SetAnnotations[kioutil.PackageAnnotation] = filepath.Dir(path)
+		r.SetAnnotations[kioutil.PathAnnotation] = path
+	}
+}
+
+// shouldSkipDir returns a filepath.SkipDir if the directory should be skipped
+func (r *LocalPackageReader) shouldSkipDir(path string, info os.FileInfo) error {
+	if r.PackageFileName == "" {
+		return nil
+	}
+	// check if this is a subpackage
+	_, err := os.Stat(filepath.Join(path, r.PackageFileName))
+	if os.IsNotExist(err) {
+		return nil
+	} else if err != nil {
+		return err
+	}
+	if !r.IncludeSubpackages {
+		return filepath.SkipDir
+	}
+	return nil
+}

kyaml/kio/pkgio_reader_test.go

@@ -0,0 +1,486 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package kio_test
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	. "sigs.k8s.io/kustomize/kyaml/kio"
+	// "sigs.k8s.io/kustomize/kyaml/testutil"
+)
+
+// setup creates directories and files for testing
+type setup struct {
+	// root is the tmp directory
+	root string
+}
+
+// setupDirectories creates directories for reading test configuration from
+func setupDirectories(t *testing.T, dirs ...string) setup {
+	d, err := ioutil.TempDir("", "kyaml-test")
+	if !assert.NoError(t, err) {
+		assert.FailNow(t, err.Error())
+	}
+	err = os.Chdir(d)
+	if !assert.NoError(t, err) {
+		assert.FailNow(t, err.Error())
+	}
+	for _, s := range dirs {
+		err = os.MkdirAll(s, 0700)
+		if !assert.NoError(t, err) {
+			assert.FailNow(t, err.Error())
+		}
+	}
+	return setup{root: d}
+}
+
+// writeFile writes a file under the test directory
+func (s setup) writeFile(t *testing.T, path string, value []byte) {
+	err := os.MkdirAll(filepath.Dir(filepath.Join(s.root, path)), 0700)
+	if !assert.NoError(t, err) {
+		assert.FailNow(t, err.Error())
+	}
+	err = ioutil.WriteFile(filepath.Join(s.root, path), value, 0600)
+	if !assert.NoError(t, err) {
+		assert.FailNow(t, err.Error())
+	}
+}
+
+// clean deletes the test config
+func (s setup) clean() {
+	os.RemoveAll(s.root)
+}
+
+var readFileA = []byte(`---
+a: b #first
+---
+c: d # second
+`)
+
+var readFileB = []byte(`# second thing
+e: f
+g:
+  h:
+  - i # has a list
+  - j
+`)
+
+var pkgFile = []byte(``)
+
+func TestLocalPackageReader_Read_empty(t *testing.T) {
+	var r LocalPackageReader
+	nodes, err := r.Read()
+	if assert.Error(t, err) {
+		assert.Contains(t, err.Error(), "must specify package path")
+	}
+	assert.Nil(t, nodes)
+
+}
+
+func TestLocalPackageReader_Read_pkg(t *testing.T) {
+	s := setupDirectories(t, filepath.Join("a", "b"), filepath.Join("a", "c"))
+	defer s.clean()
+	s.writeFile(t, filepath.Join("a_test.yaml"), readFileA)
+	s.writeFile(t, filepath.Join("b_test.yaml"), readFileB)
+
+	paths := []struct {
+		path string
+	}{
+		{path: "./"},
+		{path: s.root},
+	}
+	for _, p := range paths {
+		rfr := LocalPackageReader{PackagePath: p.path}
+		nodes, err := rfr.Read()
+		if !assert.NoError(t, err) {
+			return
+		}
+
+		if !assert.Len(t, nodes, 3) {
+			return
+		}
+		expected := []string{
+			`a: b #first
+metadata:
+  annotations:
+    kyaml.kustomize.dev/kio/index: 0
+    kyaml.kustomize.dev/kio/package: .
+    kyaml.kustomize.dev/kio/path: a_test.yaml
+`,
+			`c: d # second
+metadata:
+  annotations:
+    kyaml.kustomize.dev/kio/index: 1
+    kyaml.kustomize.dev/kio/package: .
+    kyaml.kustomize.dev/kio/path: a_test.yaml
+`,
+			`# second thing
+e: f
+g:
+  h:
+  - i # has a list
+  - j
+metadata:
+  annotations:
+    kyaml.kustomize.dev/kio/index: 0
+    kyaml.kustomize.dev/kio/package: .
+    kyaml.kustomize.dev/kio/path: b_test.yaml
+`,
+		}
+		for i := range nodes {
+			val, err := nodes[i].String()
+			if !assert.NoError(t, err) {
+				return
+			}
+			if !assert.Equal(t, expected[i], val) {
+				return
+			}
+		}
+	}
+}
+
+func TestLocalPackageReader_Read_file(t *testing.T) {
+	s := setupDirectories(t, filepath.Join("a", "b"), filepath.Join("a", "c"))
+	defer s.clean()
+	s.writeFile(t, filepath.Join("a_test.yaml"), readFileA)
+	s.writeFile(t, filepath.Join("b_test.yaml"), readFileB)
+
+	paths := []struct {
+		path string
+	}{
+		{path: "./"},
+		{path: s.root},
+	}
+	for _, p := range paths {
+		rfr := LocalPackageReader{PackagePath: filepath.Join(p.path, "a_test.yaml")}
+		nodes, err := rfr.Read()
+		if !assert.NoError(t, err) {
+			return
+		}
+
+		if !assert.Len(t, nodes, 2) {
+			return
+		}
+		expected := []string{
+			`a: b #first
+metadata:
+  annotations:
+    kyaml.kustomize.dev/kio/index: 0
+    kyaml.kustomize.dev/kio/package: .
+    kyaml.kustomize.dev/kio/path: a_test.yaml
+`,
+			`c: d # second
+metadata:
+  annotations:
+    kyaml.kustomize.dev/kio/index: 1
+    kyaml.kustomize.dev/kio/package: .
+    kyaml.kustomize.dev/kio/path: a_test.yaml
+`,
+		}
+		for i := range nodes {
+			val, err := nodes[i].String()
+			if !assert.NoError(t, err) {
+				return
+			}
+			if !assert.Equal(t, expected[i], val) {
+				return
+			}
+		}
+	}
+}
+
+func TestLocalPackageReader_Read_pkgOmitAnnotations(t *testing.T) {
+	s := setupDirectories(t, filepath.Join("a", "b"), filepath.Join("a", "c"))
+	defer s.clean()
+	s.writeFile(t, filepath.Join("a_test.yaml"), readFileA)
+	s.writeFile(t, filepath.Join("b_test.yaml"), readFileB)
+
+	paths := []struct {
+		path string
+	}{
+		{path: "./"},
+		{path: s.root},
+	}
+	for _, p := range paths {
+
+		// empty path
+		rfr := LocalPackageReader{PackagePath: p.path, OmitReaderAnnotations: true}
+		nodes, err := rfr.Read()
+		if !assert.NoError(t, err) {
+			return
+		}
+
+		if !assert.Len(t, nodes, 3) {
+			return
+		}
+		expected := []string{
+			`a: b #first
+`,
+			`c: d # second
+`,
+			`# second thing
+e: f
+g:
+  h:
+  - i # has a list
+  - j
+`,
+		}
+		for i := range nodes {
+			val, err := nodes[i].String()
+			if !assert.NoError(t, err) {
+				return
+			}
+			if !assert.Equal(t, expected[i], val) {
+				return
+			}
+		}
+	}
+}
+
+func TestLocalPackageReader_Read_nestedDirs(t *testing.T) {
+	s := setupDirectories(t, filepath.Join("a", "b"), filepath.Join("a", "c"))
+	defer s.clean()
+	s.writeFile(t, filepath.Join("a", "b", "a_test.yaml"), readFileA)
+	s.writeFile(t, filepath.Join("a", "b", "b_test.yaml"), readFileB)
+
+	paths := []struct {
+		path string
+	}{
+		{path: "./"},
+		{path: s.root},
+	}
+	for _, p := range paths {
+		// empty path
+		rfr := LocalPackageReader{PackagePath: p.path}
+		nodes, err := rfr.Read()
+		if !assert.NoError(t, err) {
+			assert.FailNow(t, err.Error())
+		}
+
+		if !assert.Len(t, nodes, 3) {
+			return
+		}
+		expected := []string{
+			`a: b #first
+metadata:
+  annotations:
+    kyaml.kustomize.dev/kio/index: 0
+    kyaml.kustomize.dev/kio/package: a/b
+    kyaml.kustomize.dev/kio/path: a/b/a_test.yaml
+`,
+			`c: d # second
+metadata:
+  annotations:
+    kyaml.kustomize.dev/kio/index: 1
+    kyaml.kustomize.dev/kio/package: a/b
+    kyaml.kustomize.dev/kio/path: a/b/a_test.yaml
+`,
+			`# second thing
+e: f
+g:
+  h:
+  - i # has a list
+  - j
+metadata:
+  annotations:
+    kyaml.kustomize.dev/kio/index: 0
+    kyaml.kustomize.dev/kio/package: a/b
+    kyaml.kustomize.dev/kio/path: a/b/b_test.yaml
+`,
+		}
+		for i := range nodes {
+			val, err := nodes[i].String()
+			if !assert.NoError(t, err) {
+				return
+			}
+			if !assert.Equal(t, expected[i], val) {
+				return
+			}
+		}
+	}
+}
+
+func TestLocalPackageReader_Read_matchRegex(t *testing.T) {
+	s := setupDirectories(t, filepath.Join("a", "b"), filepath.Join("a", "c"))
+	defer s.clean()
+	s.writeFile(t, filepath.Join("a", "b", "a_test.yaml"), readFileA)
+	s.writeFile(t, filepath.Join("a", "b", "b_test.yaml"), readFileB)
+
+	// empty path
+	rfr := LocalPackageReader{PackagePath: s.root, MatchFilesGlob: []string{`a*.yaml`}}
+	nodes, err := rfr.Read()
+	if !assert.NoError(t, err) {
+		assert.FailNow(t, err.Error())
+	}
+
+	if !assert.Len(t, nodes, 2) {
+		assert.FailNow(t, "wrong number items")
+	}
+
+	val, err := nodes[0].String()
+	assert.NoError(t, err)
+	assert.Equal(t, `a: b #first
+metadata:
+  annotations:
+    kyaml.kustomize.dev/kio/index: 0
+    kyaml.kustomize.dev/kio/package: a/b
+    kyaml.kustomize.dev/kio/path: a/b/a_test.yaml
+`, val)
+
+	val, err = nodes[1].String()
+	assert.NoError(t, err)
+	assert.Equal(t, `c: d # second
+metadata:
+  annotations:
+    kyaml.kustomize.dev/kio/index: 1
+    kyaml.kustomize.dev/kio/package: a/b
+    kyaml.kustomize.dev/kio/path: a/b/a_test.yaml
+`, val)
+}
+
+func TestLocalPackageReader_Read_skipSubpackage(t *testing.T) {
+	s := setupDirectories(t, filepath.Join("a", "b"), filepath.Join("a", "c"))
+	defer s.clean()
+	s.writeFile(t, filepath.Join("a", "b", "a_test.yaml"), readFileA)
+	s.writeFile(t, filepath.Join("a", "c", "c_test.yaml"), readFileB)
+	s.writeFile(t, filepath.Join("a", "c", "pkgFile"), pkgFile)
+
+	// empty path
+	rfr := LocalPackageReader{PackagePath: s.root, PackageFileName: "pkgFile"}
+	nodes, err := rfr.Read()
+	if !assert.NoError(t, err) {
+		assert.FailNow(t, err.Error())
+	}
+
+	if !assert.Len(t, nodes, 2) {
+		assert.FailNow(t, "wrong number items")
+	}
+
+	val, err := nodes[0].String()
+	assert.NoError(t, err)
+	assert.Equal(t, `a: b #first
+metadata:
+  annotations:
+    kyaml.kustomize.dev/kio/index: 0
+    kyaml.kustomize.dev/kio/package: a/b
+    kyaml.kustomize.dev/kio/path: a/b/a_test.yaml
+`, val)
+
+	val, err = nodes[1].String()
+	assert.NoError(t, err)
+	assert.Equal(t, `c: d # second
+metadata:
+  annotations:
+    kyaml.kustomize.dev/kio/index: 1
+    kyaml.kustomize.dev/kio/package: a/b
+    kyaml.kustomize.dev/kio/path: a/b/a_test.yaml
+`, val)
+}
+
+func TestLocalPackageReader_Read_includeSubpackage(t *testing.T) {
+	s := setupDirectories(t, filepath.Join("a", "b"), filepath.Join("a", "c"))
+	defer s.clean()
+	s.writeFile(t, filepath.Join("a", "b", "a_test.yaml"), readFileA)
+	s.writeFile(t, filepath.Join("a", "c", "c_test.yaml"), readFileB)
+	s.writeFile(t, filepath.Join("a", "c", "pkgFile"), pkgFile)
+
+	// empty path
+	rfr := LocalPackageReader{PackagePath: s.root, IncludeSubpackages: true, PackageFileName: "pkgFile"}
+	nodes, err := rfr.Read()
+	if !assert.NoError(t, err) {
+		assert.FailNow(t, err.Error())
+	}
+
+	if !assert.Len(t, nodes, 3) {
+		assert.FailNow(t, "wrong number items")
+	}
+	val, err := nodes[0].String()
+	assert.NoError(t, err)
+	assert.Equal(t, `a: b #first
+metadata:
+  annotations:
+    kyaml.kustomize.dev/kio/index: 0
+    kyaml.kustomize.dev/kio/package: a/b
+    kyaml.kustomize.dev/kio/path: a/b/a_test.yaml
+`, val)
+
+	val, err = nodes[1].String()
+	assert.NoError(t, err)
+	assert.Equal(t, `c: d # second
+metadata:
+  annotations:
+    kyaml.kustomize.dev/kio/index: 1
+    kyaml.kustomize.dev/kio/package: a/b
+    kyaml.kustomize.dev/kio/path: a/b/a_test.yaml
+`, val)
+
+	val, err = nodes[2].String()
+	assert.NoError(t, err)
+	assert.Equal(t, `# second thing
+e: f
+g:
+  h:
+  - i # has a list
+  - j
+metadata:
+  annotations:
+    kyaml.kustomize.dev/kio/index: 0
+    kyaml.kustomize.dev/kio/package: a/c
+    kyaml.kustomize.dev/kio/path: a/c/c_test.yaml
+`, val)
+}
+
+// func TestLocalPackageReaderWriter_DeleteFiles(t *testing.T) {
+// 	g, _, clean := testutil.SetupDefaultRepoAndWorkspace(t)
+// 	defer clean()
+// 	if !assert.NoError(t, os.Chdir(g.RepoDirectory)) {
+// 		return
+// 	}
+//
+// 	rw := LocalPackageReadWriter{PackagePath: "."}
+// 	nodes, err := rw.Read()
+// 	if !assert.NoError(t, err) {
+// 		t.FailNow()
+// 	}
+// 	_, err = os.Stat(filepath.Join("java", "java-deployment.resource.yaml"))
+// 	if !assert.NoError(t, err) {
+// 		t.FailNow()
+// 	}
+//
+// 	// delete one of the nodes
+// 	var newNodes []*yaml.RNode
+// 	for i := range nodes {
+// 		meta, err := nodes[i].GetMeta()
+// 		if !assert.NoError(t, err) {
+// 			t.FailNow()
+// 		}
+// 		if meta.Name == "app" && meta.Kind == "Deployment" {
+// 			continue
+// 		}
+// 		newNodes = append(newNodes, nodes[i])
+// 	}
+//
+// 	if !assert.NoError(t, rw.Write(newNodes)) {
+// 		t.FailNow()
+// 	}
+//
+// 	_, err = os.Stat(filepath.Join("java", "java-deployment.resource.yaml"))
+// 	if !assert.Error(t, err) {
+// 		t.FailNow()
+// 	}
+//
+// 	diff, err := copyutil.Diff(filepath.Join(g.DatasetDirectory, testutil.Dataset1), ".")
+// 	if !assert.NoError(t, err) {
+// 		t.FailNow()
+// 	}
+//
+// 	assert.ElementsMatch(t,
+// 		diff.List(),
+// 		[]string{filepath.Join("java", "java-deployment.resource.yaml")})
+// }

kyaml/kio/pkgio_writer.go

@@ -0,0 +1,152 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package kio
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"sigs.k8s.io/kustomize/kyaml/kio/kioutil"
+	"sigs.k8s.io/kustomize/kyaml/yaml"
+)
+
+// LocalPackageWriter writes ResourceNodes to a filesystem
+type LocalPackageWriter struct {
+	Kind string `yaml:"kind,omitempty"`
+
+	// PackagePath is the path to the package directory.
+	PackagePath string `yaml:"path,omitempty"`
+
+	// KeepReaderAnnotations if set will retain the annotations set by LocalPackageReader
+	KeepReaderAnnotations bool `yaml:"keepReaderAnnotations,omitempty"`
+
+	// ClearAnnotations will clear annotations before writing the resources
+	ClearAnnotations []string `yaml:"clearAnnotations,omitempty"`
+}
+
+var _ Writer = LocalPackageWriter{}
+
+func (r LocalPackageWriter) Write(nodes []*yaml.RNode) error {
+	if err := kioutil.ErrorIfMissingAnnotation(nodes, requiredResourcePackageAnnotations...); err != nil {
+		return err
+	}
+
+	if s, err := os.Stat(r.PackagePath); err != nil {
+		return err
+	} else if !s.IsDir() {
+		// if the user specified input isn't a directory, the package is the directory of the
+		// target
+		r.PackagePath = filepath.Dir(r.PackagePath)
+	}
+
+	// setup indexes for writing Resources back to files
+	if err := r.errorIfMissingRequiredAnnotation(nodes); err != nil {
+		return err
+	}
+	outputFiles, err := r.indexByFilePath(nodes)
+	if err != nil {
+		return err
+	}
+	for k := range outputFiles {
+		if err = kioutil.SortNodes(outputFiles[k]); err != nil {
+			return err
+		}
+	}
+
+	if !r.KeepReaderAnnotations {
+		r.ClearAnnotations = append(r.ClearAnnotations, kioutil.PackageAnnotation)
+		r.ClearAnnotations = append(r.ClearAnnotations, kioutil.PathAnnotation)
+	}
+
+	// validate outputs before writing any
+	for path := range outputFiles {
+		outputPath := filepath.Join(r.PackagePath, path)
+		if st, err := os.Stat(outputPath); !os.IsNotExist(err) {
+			if err != nil {
+				return err
+			}
+			if st.IsDir() {
+				return fmt.Errorf("kyaml.kustomize.dev/kio/path cannot be a directory: %s", path)
+			}
+		}
+
+		err = os.MkdirAll(filepath.Dir(outputPath), 0700)
+		if err != nil {
+			return err
+		}
+	}
+
+	// write files
+	for path := range outputFiles {
+		outputPath := filepath.Join(r.PackagePath, path)
+		err = os.MkdirAll(filepath.Dir(filepath.Join(r.PackagePath, path)), 0700)
+		if err != nil {
+			return err
+		}
+
+		f, err := os.OpenFile(outputPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, os.FileMode(0600))
+		if err != nil {
+			return err
+		}
+		if err := func() error {
+			defer f.Close()
+			w := ByteWriter{
+				Writer:                f,
+				KeepReaderAnnotations: r.KeepReaderAnnotations,
+				ClearAnnotations:      r.ClearAnnotations,
+			}
+			if err = w.Write(outputFiles[path]); err != nil {
+				return err
+			}
+			return nil
+		}(); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (r LocalPackageWriter) errorIfMissingRequiredAnnotation(nodes []*yaml.RNode) error {
+	for i := range nodes {
+		for _, s := range requiredResourcePackageAnnotations {
+			key, err := nodes[i].Pipe(yaml.GetAnnotation(s))
+			if err != nil {
+				return err
+			}
+			if key == nil || key.YNode() == nil || key.YNode().Value == "" {
+				return fmt.Errorf(
+					"resources must be annotated with %s to be written to files", s)
+			}
+		}
+	}
+	return nil
+}
+
+func (r LocalPackageWriter) indexByFilePath(nodes []*yaml.RNode) (map[string][]*yaml.RNode, error) {
+
+	outputFiles := map[string][]*yaml.RNode{}
+	for i := range nodes {
+		// parse the file write path
+		node := nodes[i]
+		value, err := node.Pipe(yaml.GetAnnotation(kioutil.PathAnnotation))
+		if err != nil {
+			// this should never happen if errorIfMissingRequiredAnnotation was run
+			return nil, err
+		}
+		path := value.YNode().Value
+		outputFiles[path] = append(outputFiles[path], node)
+
+		if filepath.IsAbs(path) {
+			return nil, fmt.Errorf("package paths may not be absolute paths")
+		}
+		if strings.Contains(filepath.Clean(path), "..") {
+			return nil, fmt.Errorf("resource must be written under package %s: %s",
+				r.PackagePath, filepath.Clean(path))
+		}
+	}
+	return outputFiles, nil
+}

kyaml/kio/pkgio_writer_test.go

@@ -0,0 +1,329 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package kio_test
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	. "sigs.k8s.io/kustomize/kyaml/kio"
+	"sigs.k8s.io/kustomize/kyaml/yaml"
+)
+
+// TestLocalPackageWriter_Write tests:
+// - ReaderAnnotations are cleared when writing the Resources
+func TestLocalPackageWriter_Write(t *testing.T) {
+	d, node1, node2, node3 := getWriterInputs(t)
+	defer os.RemoveAll(d)
+
+	w := LocalPackageWriter{PackagePath: d}
+	err := w.Write([]*yaml.RNode{node2, node1, node3})
+	if !assert.NoError(t, err) {
+		assert.FailNow(t, err.Error())
+	}
+
+	b, err := ioutil.ReadFile(filepath.Join(d, "a", "b", "a_test.yaml"))
+	if !assert.NoError(t, err) {
+		assert.FailNow(t, err.Error())
+	}
+	assert.Equal(t, `a: b #first
+---
+c: d # second
+`, string(b))
+
+	b, err = ioutil.ReadFile(filepath.Join(d, "a", "b", "b_test.yaml"))
+	if !assert.NoError(t, err) {
+		assert.FailNow(t, err.Error())
+	}
+	assert.Equal(t, `e: f
+g:
+  h:
+  - i # has a list
+  - j
+`, string(b))
+}
+
+// TestLocalPackageWriter_Write_keepReaderAnnotations tests:
+// - ReaderAnnotations are kept when writing the Resources
+func TestLocalPackageWriter_Write_keepReaderAnnotations(t *testing.T) {
+	d, node1, node2, node3 := getWriterInputs(t)
+	defer os.RemoveAll(d)
+
+	w := LocalPackageWriter{PackagePath: d, KeepReaderAnnotations: true}
+	err := w.Write([]*yaml.RNode{node2, node1, node3})
+	if !assert.NoError(t, err) {
+		return
+	}
+
+	b, err := ioutil.ReadFile(filepath.Join(d, "a", "b", "a_test.yaml"))
+	if !assert.NoError(t, err) {
+		return
+	}
+	assert.Equal(t, `a: b #first
+metadata:
+  annotations:
+    kyaml.kustomize.dev/kio/index: 0
+    kyaml.kustomize.dev/kio/path: a/b/a_test.yaml
+---
+c: d # second
+metadata:
+  annotations:
+    kyaml.kustomize.dev/kio/index: 1
+    kyaml.kustomize.dev/kio/path: a/b/a_test.yaml
+`, string(b))
+
+	b, err = ioutil.ReadFile(filepath.Join(d, "a", "b", "b_test.yaml"))
+	if !assert.NoError(t, err) {
+		return
+	}
+	assert.Equal(t, `e: f
+g:
+  h:
+  - i # has a list
+  - j
+metadata:
+  annotations:
+    kyaml.kustomize.dev/kio/index: 0
+    kyaml.kustomize.dev/kio/path: a/b/b_test.yaml
+`, string(b))
+}
+
+// TestLocalPackageWriter_Write_clearAnnotations tests:
+// - ClearAnnotations are removed from Resources
+func TestLocalPackageWriter_Write_clearAnnotations(t *testing.T) {
+	d, node1, node2, node3 := getWriterInputs(t)
+	defer os.RemoveAll(d)
+
+	w := LocalPackageWriter{PackagePath: d, ClearAnnotations: []string{"kyaml.kustomize.dev/kio/mode"}}
+	err := w.Write([]*yaml.RNode{node2, node1, node3})
+	if !assert.NoError(t, err) {
+		assert.FailNow(t, err.Error())
+	}
+
+	b, err := ioutil.ReadFile(filepath.Join(d, "a", "b", "a_test.yaml"))
+	if !assert.NoError(t, err) {
+		assert.FailNow(t, err.Error())
+	}
+	assert.Equal(t, `a: b #first
+---
+c: d # second
+`, string(b))
+
+	b, err = ioutil.ReadFile(filepath.Join(d, "a", "b", "b_test.yaml"))
+	if !assert.NoError(t, err) {
+		assert.FailNow(t, err.Error())
+	}
+	assert.Equal(t, `e: f
+g:
+  h:
+  - i # has a list
+  - j
+`, string(b))
+}
+
+// TestLocalPackageWriter_Write_failRelativePath tests:
+// - If a relative path above the package is defined, write fails
+func TestLocalPackageWriter_Write_failRelativePath(t *testing.T) {
+	d, node1, node2, node3 := getWriterInputs(t)
+	defer os.RemoveAll(d)
+
+	node4, err := yaml.Parse(`e: f
+g:
+  h:
+  - i # has a list
+  - j
+metadata:
+  annotations:
+    kyaml.kustomize.dev/kio/index: 0
+    kyaml.kustomize.dev/kio/path: "a/b/../../../b_test.yaml"
+`)
+	if !assert.NoError(t, err) {
+		assert.FailNow(t, err.Error())
+	}
+
+	w := LocalPackageWriter{PackagePath: d}
+	err = w.Write([]*yaml.RNode{node2, node1, node3, node4})
+	if assert.Error(t, err) {
+		assert.Contains(t, err.Error(), "resource must be written under package")
+	}
+}
+
+// TestLocalPackageWriter_Write_invalidIndex tests:
+// - If a non-int index is given, fail
+func TestLocalPackageWriter_Write_invalidIndex(t *testing.T) {
+	d, node1, node2, node3 := getWriterInputs(t)
+	defer os.RemoveAll(d)
+
+	node4, err := yaml.Parse(`e: f
+g:
+  h:
+  - i # has a list
+  - j
+metadata:
+  annotations:
+    kyaml.kustomize.dev/kio/index: a
+    kyaml.kustomize.dev/kio/path: "a/b/b_test.yaml" # use a different path, should still collide
+`)
+	if !assert.NoError(t, err) {
+		assert.FailNow(t, err.Error())
+	}
+
+	w := LocalPackageWriter{PackagePath: d}
+	err = w.Write([]*yaml.RNode{node2, node1, node3, node4})
+	if assert.Error(t, err) {
+		assert.Contains(t, err.Error(), "unable to parse kyaml.kustomize.dev/kio/index")
+	}
+}
+
+// TestLocalPackageWriter_Write_absPath tests:
+// - If kyaml.kustomize.dev/kio/path is absolute, fail
+func TestLocalPackageWriter_Write_absPath(t *testing.T) {
+	d, node1, node2, node3 := getWriterInputs(t)
+	defer os.RemoveAll(d)
+
+	node4, err := yaml.Parse(fmt.Sprintf(`e: f
+g:
+  h:
+  - i # has a list
+  - j
+metadata:
+  annotations:
+    kyaml.kustomize.dev/kio/index: a
+    kyaml.kustomize.dev/kio/path: "%s/a/b/b_test.yaml" # use a different path, should still collide
+`, d))
+	if !assert.NoError(t, err) {
+		assert.FailNow(t, err.Error())
+	}
+
+	w := LocalPackageWriter{PackagePath: d}
+	err = w.Write([]*yaml.RNode{node2, node1, node3, node4})
+	if assert.Error(t, err) {
+		assert.Contains(t, err.Error(), "package paths may not be absolute paths")
+	}
+}
+
+// TestLocalPackageWriter_Write_missingIndex tests:
+// - If kyaml.kustomize.dev/kio/path is missing, fail
+func TestLocalPackageWriter_Write_missingPath(t *testing.T) {
+	d, node1, node2, node3 := getWriterInputs(t)
+	defer os.RemoveAll(d)
+
+	node4, err := yaml.Parse(`e: f
+g:
+  h:
+  - i # has a list
+  - j
+metadata:
+  annotations:
+    kyaml.kustomize.dev/kio/index: a
+`)
+	if !assert.NoError(t, err) {
+		assert.FailNow(t, err.Error())
+	}
+
+	w := LocalPackageWriter{PackagePath: d}
+	err = w.Write([]*yaml.RNode{node2, node1, node3, node4})
+	if assert.Error(t, err) {
+		assert.Contains(t, err.Error(), "kyaml.kustomize.dev/kio/path")
+	}
+}
+
+// TestLocalPackageWriter_Write_missingIndex tests:
+// - If kyaml.kustomize.dev/kio/index is missing, fail
+func TestLocalPackageWriter_Write_missingIndex(t *testing.T) {
+	d, node1, node2, node3 := getWriterInputs(t)
+	defer os.RemoveAll(d)
+
+	node4, err := yaml.Parse(`e: f
+g:
+  h:
+  - i # has a list
+  - j
+metadata:
+  annotations:
+    kyaml.kustomize.dev/kio/path: a/a.yaml
+`)
+	if !assert.NoError(t, err) {
+		assert.FailNow(t, err.Error())
+	}
+
+	w := LocalPackageWriter{PackagePath: d}
+	err = w.Write([]*yaml.RNode{node2, node1, node3, node4})
+	if assert.Error(t, err) {
+		assert.Contains(t, err.Error(), "kyaml.kustomize.dev/kio/index")
+	}
+}
+
+// TestLocalPackageWriter_Write_pathIsDir tests:
+// - If  kyaml.kustomize.dev/kio/path is a directory, fail
+func TestLocalPackageWriter_Write_pathIsDir(t *testing.T) {
+	d, node1, node2, node3 := getWriterInputs(t)
+	defer os.RemoveAll(d)
+
+	node4, err := yaml.Parse(`e: f
+g:
+  h:
+  - i # has a list
+  - j
+metadata:
+  annotations:
+    kyaml.kustomize.dev/kio/path: a/
+    kyaml.kustomize.dev/kio/index: 0
+`)
+	if !assert.NoError(t, err) {
+		assert.FailNow(t, err.Error())
+	}
+
+	w := LocalPackageWriter{PackagePath: d}
+	err = w.Write([]*yaml.RNode{node2, node1, node3, node4})
+	if assert.Error(t, err) {
+		assert.Contains(t, err.Error(), "kyaml.kustomize.dev/kio/path cannot be a directory")
+	}
+}
+
+func getWriterInputs(t *testing.T) (string, *yaml.RNode, *yaml.RNode, *yaml.RNode) {
+	node1, err := yaml.Parse(`a: b #first
+metadata:
+  annotations:
+    kyaml.kustomize.dev/kio/index: 0
+    kyaml.kustomize.dev/kio/path: "a/b/a_test.yaml"
+`)
+	if !assert.NoError(t, err) {
+		assert.FailNow(t, err.Error())
+	}
+	node2, err := yaml.Parse(`c: d # second
+metadata:
+  annotations:
+    kyaml.kustomize.dev/kio/index: 1
+    kyaml.kustomize.dev/kio/path: "a/b/a_test.yaml"
+`)
+	if !assert.NoError(t, err) {
+		assert.FailNow(t, err.Error())
+	}
+	node3, err := yaml.Parse(`e: f
+g:
+  h:
+  - i # has a list
+  - j
+metadata:
+  annotations:
+    kyaml.kustomize.dev/kio/index: 0
+    kyaml.kustomize.dev/kio/path: "a/b/b_test.yaml"
+`)
+	if !assert.NoError(t, err) {
+		assert.FailNow(t, err.Error())
+	}
+	d, err := ioutil.TempDir("", "kyaml-test")
+	if !assert.NoError(t, err) {
+		assert.FailNow(t, err.Error())
+	}
+	if !assert.NoError(t, os.MkdirAll(filepath.Join(d, "a"), 0700)) {
+		assert.FailNow(t, "")
+	}
+	return d, node1, node2, node3
+}

kyaml/kio/tree.go

@@ -0,0 +1,491 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package kio
+
+import (
+	"fmt"
+	"io"
+	"path/filepath"
+	"sort"
+	"strings"
+
+	"github.com/xlab/treeprint"
+	"sigs.k8s.io/kustomize/kyaml/kio/kioutil"
+	"sigs.k8s.io/kustomize/kyaml/yaml"
+)
+
+type TreeStructure string
+
+const (
+	// TreeStructurePackage configures TreeWriter to generate the tree structure off of the
+	// Resources packages.
+	TreeStructurePackage TreeStructure = "package"
+
+	// TreeStructureOwners configures TreeWriter to generate the tree structure off of the
+	// Resource owners.
+	TreeStructureGraph TreeStructure = "graph"
+)
+
+// TreeWriter prints the package structured as a tree
+// TODO(pwittrock): test this package better.  it is lower-risk since it is only
+// used for printing rather than updating or editing.
+type TreeWriter struct {
+	Writer    io.Writer
+	Root      string
+	Fields    []TreeWriterField
+	Structure TreeStructure
+}
+
+// TreeWriterField configures a Resource field to be included in the tree
+type TreeWriterField struct {
+	yaml.PathMatcher
+	Name    string
+	SubName string
+}
+
+func (p TreeWriter) packageStructure(nodes []*yaml.RNode) error {
+	indexByPackage := p.index(nodes)
+
+	// create the new tree
+	tree := treeprint.New()
+	tree.SetValue(p.Root)
+
+	// add each package to the tree
+	treeIndex := map[string]treeprint.Tree{}
+	keys := p.sort(indexByPackage)
+	for _, pkg := range keys {
+		// create a branch for this package -- search for the parent package and create
+		// the branch under it -- requires that the keys are sorted
+		branch := tree
+		for parent, subTree := range treeIndex {
+			if strings.HasPrefix(pkg, parent) {
+				// found a package whose path is a prefix to our own, use this
+				// package if a closer one isn't found
+				branch = subTree
+				// don't break, continue searching for more closely related ancestors
+			}
+		}
+
+		// create a new branch for the package
+		createOk := pkg != "." // special edge case logic for tree on current working dir
+		if createOk {
+			branch = branch.AddBranch(pkg)
+		}
+
+		// cache the branch for this package
+		treeIndex[pkg] = branch
+
+		// print each resource in the package
+		for i := range indexByPackage[pkg] {
+			var err error
+			if _, err = p.doResource(indexByPackage[pkg][i], "", branch); err != nil {
+				return err
+			}
+		}
+	}
+
+	_, err := io.WriteString(p.Writer, tree.String())
+	return err
+}
+
+// Write writes the ascii tree to p.Writer
+func (p TreeWriter) Write(nodes []*yaml.RNode) error {
+	switch p.Structure {
+	case TreeStructurePackage:
+		return p.packageStructure(nodes)
+	case TreeStructureGraph:
+		return p.graphStructure(nodes)
+	default:
+		return p.packageStructure(nodes)
+	}
+}
+
+// node wraps a tree node, and any children nodes
+type node struct {
+	p TreeWriter
+	*yaml.RNode
+	children []*node
+}
+
+func (a node) Len() int      { return len(a.children) }
+func (a node) Swap(i, j int) { a.children[i], a.children[j] = a.children[j], a.children[i] }
+func (a node) Less(i, j int) bool {
+	return compareNodes(a.children[i].RNode, a.children[j].RNode)
+}
+
+// Tree adds this node to the root
+func (a node) Tree(root treeprint.Tree) error {
+	sort.Sort(a)
+	branch := root
+	var err error
+
+	// generate a node for the Resource
+	if a.RNode != nil {
+		branch, err = a.p.doResource(a.RNode, "Resource", root)
+		if err != nil {
+			return err
+		}
+	}
+
+	// attach children to the branch
+	for _, n := range a.children {
+		if err := n.Tree(branch); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// graphStructure writes the tree using owners for structure
+func (p TreeWriter) graphStructure(nodes []*yaml.RNode) error {
+	resourceToOwner := map[string]*node{}
+	root := &node{}
+	// index each of the nodes by their owner
+	for _, n := range nodes {
+		ownerVal, err := ownerToString(n)
+		if err != nil {
+			return err
+		}
+		var owner *node
+		if ownerVal == "" {
+			// no owner -- attach to the root
+			owner = root
+		} else {
+			// owner found -- attach to the owner
+			var found bool
+			owner, found = resourceToOwner[ownerVal]
+			if !found {
+				// initialize the owner if not found
+				resourceToOwner[ownerVal] = &node{p: p}
+				owner = resourceToOwner[ownerVal]
+			}
+		}
+
+		nodeVal, err := nodeToString(n)
+		if err != nil {
+			return err
+		}
+		val, found := resourceToOwner[nodeVal]
+		if !found {
+			// initialize the node if not found -- may have already been initialized if it
+			// is the owner of another node
+			resourceToOwner[nodeVal] = &node{p: p}
+			val = resourceToOwner[nodeVal]
+		}
+		val.RNode = n
+		owner.children = append(owner.children, val)
+	}
+
+	for k, v := range resourceToOwner {
+		if v.RNode == nil {
+			return fmt.Errorf(
+				"owner '%s' not found in input, but found as an owner of input objects", k)
+		}
+	}
+
+	// print the tree
+	tree := treeprint.New()
+	if err := root.Tree(tree); err != nil {
+		return err
+	}
+
+	_, err := io.WriteString(p.Writer, tree.String())
+	return err
+}
+
+// nodeToString generates a string to identify the node -- matches ownerToString format
+func nodeToString(node *yaml.RNode) (string, error) {
+	meta, err := node.GetMeta()
+	if err != nil {
+		return "", err
+	}
+
+	return fmt.Sprintf("%s %s/%s", meta.Kind, meta.Namespace, meta.Name), nil
+}
+
+// ownerToString generate a string to identify the owner -- matches nodeToString format
+func ownerToString(node *yaml.RNode) (string, error) {
+	meta, err := node.GetMeta()
+	if err != nil {
+		return "", err
+	}
+	namespace := meta.Namespace
+
+	owners, err := node.Pipe(yaml.Lookup("metadata", "ownerReferences"))
+	if err != nil {
+		return "", err
+	}
+	if owners == nil {
+		return "", nil
+	}
+
+	elements, err := owners.Elements()
+	if err != nil {
+		return "", err
+	}
+	if len(elements) == 0 {
+		return "", err
+	}
+	owner := elements[0]
+	var kind, name string
+
+	if value := owner.Field("kind"); !yaml.IsFieldEmpty(value) {
+		kind = value.Value.YNode().Value
+	}
+	if value := owner.Field("name"); !yaml.IsFieldEmpty(value) {
+		name = value.Value.YNode().Value
+	}
+
+	return fmt.Sprintf("%s %s/%s", kind, namespace, name), nil
+}
+
+// index indexes the Resources by their package
+func (p TreeWriter) index(nodes []*yaml.RNode) map[string][]*yaml.RNode {
+	// index the ResourceNodes by package
+	indexByPackage := map[string][]*yaml.RNode{}
+	for i := range nodes {
+		meta, err := nodes[i].GetMeta()
+		if err != nil || meta.Kind == "" {
+			// not a resource
+			continue
+		}
+		pkg := meta.Annotations[kioutil.PackageAnnotation]
+		indexByPackage[pkg] = append(indexByPackage[pkg], nodes[i])
+	}
+	return indexByPackage
+}
+
+func compareNodes(i, j *yaml.RNode) bool {
+	metai, _ := i.GetMeta()
+	metaj, _ := j.GetMeta()
+	pi := metai.Annotations[kioutil.PathAnnotation]
+	pj := metaj.Annotations[kioutil.PathAnnotation]
+
+	// compare file names
+	if filepath.Base(pi) != filepath.Base(pj) {
+		return filepath.Base(pi) < filepath.Base(pj)
+	}
+
+	// compare namespace
+	if metai.Namespace != metaj.Namespace {
+		return metai.Namespace < metaj.Namespace
+	}
+
+	// compare name
+	if metai.Name != metaj.Name {
+		return metai.Name < metaj.Name
+	}
+
+	// compare kind
+	if metai.Kind != metaj.Kind {
+		return metai.Kind < metaj.Kind
+	}
+
+	// compare apiVersion
+	if metai.ApiVersion != metaj.ApiVersion {
+		return metai.ApiVersion < metaj.ApiVersion
+	}
+	return true
+}
+
+// sort sorts the Resources in the index in display order and returns the ordered
+// keys for the index
+//
+// Packages are sorted by package name
+// Resources within a package are sorted by: [filename, namespace, name, kind, apiVersion]
+func (p TreeWriter) sort(indexByPackage map[string][]*yaml.RNode) []string {
+	var keys []string
+	for k := range indexByPackage {
+		pkgNodes := indexByPackage[k]
+		sort.Slice(pkgNodes, func(i, j int) bool { return compareNodes(pkgNodes[i], pkgNodes[j]) })
+		keys = append(keys, k)
+	}
+
+	// return the package names sorted lexicographically
+	sort.Strings(keys)
+	return keys
+}
+
+func (p TreeWriter) doResource(leaf *yaml.RNode, metaString string, branch treeprint.Tree) (treeprint.Tree, error) {
+	meta, _ := leaf.GetMeta()
+	if metaString == "" {
+		path := meta.Annotations[kioutil.PathAnnotation]
+		path = filepath.Base(path)
+		metaString = path
+	}
+
+	value := fmt.Sprintf("%s %s", meta.Kind, meta.Name)
+	if len(meta.Namespace) > 0 {
+		value = fmt.Sprintf("%s %s/%s", meta.Kind, meta.Namespace, meta.Name)
+	}
+
+	fields, err := p.getFields(leaf)
+	if err != nil {
+		return nil, err
+	}
+
+	n := branch.AddMetaBranch(metaString, value)
+	for i := range fields {
+		field := fields[i]
+
+		// do leaf node
+		if len(field.matchingElementsAndFields) == 0 {
+			n.AddNode(fmt.Sprintf("%s: %s", field.name, field.value))
+			continue
+		}
+
+		// do nested nodes
+		b := n.AddBranch(field.name)
+		for j := range field.matchingElementsAndFields {
+			elem := field.matchingElementsAndFields[j]
+			b := b.AddBranch(elem.name)
+			for k := range elem.matchingElementsAndFields {
+				field := elem.matchingElementsAndFields[k]
+				b.AddNode(fmt.Sprintf("%s: %s", field.name, field.value))
+			}
+		}
+	}
+
+	return n, nil
+}
+
+// getFields looks up p.Fields from leaf and structures them into treeFields.
+// TODO(pwittrock): simplify this function
+func (p TreeWriter) getFields(leaf *yaml.RNode) (treeFields, error) {
+	fieldsByName := map[string]*treeField{}
+
+	// index nested and non-nested fields
+	for i := range p.Fields {
+		f := p.Fields[i]
+		seq, err := leaf.Pipe(&f)
+		if err != nil {
+			return nil, err
+		}
+		if seq == nil {
+			continue
+		}
+
+		if fieldsByName[f.Name] == nil {
+			fieldsByName[f.Name] = &treeField{name: f.Name}
+		}
+
+		// non-nested field -- add directly to the treeFields list
+		if f.SubName == "" {
+			// non-nested field -- only 1 element
+			val, err := yaml.String(seq.Content()[0], yaml.Trim, yaml.Flow)
+			if err != nil {
+				return nil, err
+			}
+			fieldsByName[f.Name].value = val
+			continue
+		}
+
+		// nested-field -- create a parent elem, and index by the 'match' value
+		if fieldsByName[f.Name].subFieldByMatch == nil {
+			fieldsByName[f.Name].subFieldByMatch = map[string]treeFields{}
+		}
+		index := fieldsByName[f.Name].subFieldByMatch
+		for j := range seq.Content() {
+			elem := seq.Content()[j]
+			matches := f.Matches[elem]
+			str, err := yaml.String(elem, yaml.Trim, yaml.Flow)
+			if err != nil {
+				return nil, err
+			}
+
+			// map the field by the name of the element
+			// index the subfields by the matching element so we can put all the fields for the
+			// same element under the same branch
+			matchKey := strings.Join(matches, "/")
+			index[matchKey] = append(index[matchKey], &treeField{name: f.SubName, value: str})
+		}
+	}
+
+	// iterate over collection of all queried fields in the Resource
+	for _, field := range fieldsByName {
+		// iterate over collection of elements under the field -- indexed by element name
+		for match, subFields := range field.subFieldByMatch {
+			// create a new element for this collection of fields
+			// note: we will convert name to an index later, but keep the match for sorting
+			elem := &treeField{name: match}
+			field.matchingElementsAndFields = append(field.matchingElementsAndFields, elem)
+
+			// iterate over collection of queried fields for the element
+			for i := range subFields {
+				// add to the list of fields for this element
+				elem.matchingElementsAndFields = append(elem.matchingElementsAndFields, subFields[i])
+			}
+		}
+		// clear this cached data
+		field.subFieldByMatch = nil
+	}
+
+	// put the fields in a list so they are ordered
+	fieldList := treeFields{}
+	for _, v := range fieldsByName {
+		fieldList = append(fieldList, v)
+	}
+
+	// sort the fields
+	sort.Sort(fieldList)
+	for i := range fieldList {
+		field := fieldList[i]
+		// sort the elements under this field
+		sort.Sort(field.matchingElementsAndFields)
+
+		for i := range field.matchingElementsAndFields {
+			element := field.matchingElementsAndFields[i]
+			// sort the elements under a list field by their name
+			sort.Sort(element.matchingElementsAndFields)
+			// set the name of the element to its index
+			element.name = fmt.Sprintf("%d", i)
+		}
+	}
+
+	return fieldList, nil
+}
+
+// treeField wraps a field node
+type treeField struct {
+	// name is the name of the node
+	name string
+
+	// value is the value of the node -- may be empty
+	value string
+
+	// matchingElementsAndFields is a slice of fields that go under this as a branch
+	matchingElementsAndFields treeFields
+
+	// subFieldByMatch caches matchingElementsAndFields indexed by the name of the matching elem
+	subFieldByMatch map[string]treeFields
+}
+
+// treeFields wraps a slice of treeField so they can be sorted
+type treeFields []*treeField
+
+func (nodes treeFields) Len() int { return len(nodes) }
+
+func (nodes treeFields) Less(i, j int) bool {
+	iIndex, iFound := yaml.FieldOrder[nodes[i].name]
+	jIndex, jFound := yaml.FieldOrder[nodes[j].name]
+	if iFound && jFound {
+		return iIndex < jIndex
+	}
+	if iFound {
+		return true
+	}
+	if jFound {
+		return false
+	}
+
+	if nodes[i].name != nodes[j].name {
+		return nodes[i].name < nodes[j].name
+	}
+	if nodes[i].value != nodes[j].value {
+		return nodes[i].value < nodes[j].value
+	}
+	return false
+}
+
+func (nodes treeFields) Swap(i, j int) { nodes[i], nodes[j] = nodes[j], nodes[i] }

kyaml/kio/tree_test.go

@@ -0,0 +1,386 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package kio_test
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	. "sigs.k8s.io/kustomize/kyaml/kio"
+	"sigs.k8s.io/kustomize/kyaml/yaml"
+)
+
+func TestPrinter_Write(t *testing.T) {
+	in := `kind: Deployment
+metadata:
+  labels:
+    app: nginx3
+  name: foo
+  namespace: default
+  annotations:
+    app: nginx3
+    kyaml.kustomize.dev/kio/package: foo-package/3
+    kyaml.kustomize.dev/kio/path: f3.yaml
+spec:
+  replicas: 1
+---
+kind: Deployment
+metadata:
+  labels:
+    app: nginx2
+  name: foo
+  namespace: default
+  annotations:
+    app: nginx2
+    kyaml.kustomize.dev/kio/package: foo-package
+    kyaml.kustomize.dev/kio/path: f1.yaml
+spec:
+  replicas: 1
+---
+kind: Deployment
+metadata:
+  labels:
+    app: nginx
+  annotations:
+    app: nginx
+    kyaml.kustomize.dev/kio/package: bar-package
+    kyaml.kustomize.dev/kio/path: f2.yaml
+  name: bar
+spec:
+  replicas: 3
+---
+kind: Service
+metadata:
+  name: foo
+  namespace: default
+  annotations:
+    app: nginx
+    kyaml.kustomize.dev/kio/package: foo-package
+    kyaml.kustomize.dev/kio/path: f1.yaml
+spec:
+  selector:
+    app: nginx
+`
+	out := &bytes.Buffer{}
+	err := Pipeline{
+		Inputs:  []Reader{&ByteReader{Reader: bytes.NewBufferString(in)}},
+		Outputs: []Writer{TreeWriter{Writer: out}},
+	}.Execute()
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
+
+	if !assert.Equal(t, `
+├── bar-package
+│   └── [f2.yaml]  Deployment bar
+└── foo-package
+    ├── [f1.yaml]  Deployment default/foo
+    ├── [f1.yaml]  Service default/foo
+    └── foo-package/3
+        └── [f3.yaml]  Deployment default/foo
+`, out.String()) {
+		t.FailNow()
+	}
+}
+
+func TestPrinter_Write_base(t *testing.T) {
+	in := `kind: Deployment
+metadata:
+  labels:
+    app: nginx3
+  name: foo
+  namespace: default
+  annotations:
+    app: nginx3
+    kyaml.kustomize.dev/kio/package: .
+    kyaml.kustomize.dev/kio/path: f3.yaml
+spec:
+  replicas: 1
+---
+kind: Deployment
+metadata:
+  labels:
+    app: nginx2
+  name: foo
+  namespace: default
+  annotations:
+    app: nginx2
+    kyaml.kustomize.dev/kio/package: foo-package
+    kyaml.kustomize.dev/kio/path: f1.yaml
+spec:
+  replicas: 1
+---
+kind: Deployment
+metadata:
+  labels:
+    app: nginx
+  annotations:
+    app: nginx
+    kyaml.kustomize.dev/kio/package: bar-package
+    kyaml.kustomize.dev/kio/path: f2.yaml
+  name: bar
+spec:
+  replicas: 3
+---
+kind: Service
+metadata:
+  name: foo
+  namespace: default
+  annotations:
+    app: nginx
+    kyaml.kustomize.dev/kio/package: .
+    kyaml.kustomize.dev/kio/path: f1.yaml
+spec:
+  selector:
+    app: nginx
+`
+	out := &bytes.Buffer{}
+	err := Pipeline{
+		Inputs:  []Reader{&ByteReader{Reader: bytes.NewBufferString(in)}},
+		Outputs: []Writer{TreeWriter{Writer: out}},
+	}.Execute()
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
+
+	if !assert.Equal(t, `
+├── [f1.yaml]  Service default/foo
+├── [f3.yaml]  Deployment default/foo
+├── bar-package
+│   └── [f2.yaml]  Deployment bar
+└── foo-package
+    └── [f1.yaml]  Deployment default/foo
+`, out.String()) {
+		t.FailNow()
+	}
+}
+
+func TestPrinter_Write_sort(t *testing.T) {
+	in := `apiVersion: extensions/v1
+kind: Deployment
+metadata:
+  labels:
+    app: nginx2
+  name: foo3
+  namespace: default
+  annotations:
+    app: nginx2
+    kyaml.kustomize.dev/kio/package: .
+    kyaml.kustomize.dev/kio/path: f1.yaml
+spec:
+  replicas: 1
+---
+apiVersion: extensions/v1
+kind: Deployment
+metadata:
+  labels:
+    app: nginx2
+  name: foo3
+  namespace: default
+  annotations:
+    app: nginx2
+    kyaml.kustomize.dev/kio/package: .
+    kyaml.kustomize.dev/kio/path: f1.yaml
+spec:
+  replicas: 1
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: nginx2
+  name: foo3
+  namespace: default
+  annotations:
+    app: nginx2
+    kyaml.kustomize.dev/kio/package: .
+    kyaml.kustomize.dev/kio/path: f1.yaml
+spec:
+  replicas: 1
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: nginx2
+  name: foo2
+  namespace: default2
+  annotations:
+    app: nginx2
+    kyaml.kustomize.dev/kio/package: .
+    kyaml.kustomize.dev/kio/path: f1.yaml
+spec:
+  replicas: 1
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: nginx3
+  name: foo
+  namespace: default
+  annotations:
+    app: nginx3
+    kyaml.kustomize.dev/kio/package: .
+    kyaml.kustomize.dev/kio/path: f1.yaml
+spec:
+  replicas: 1
+---
+kind: Deployment
+metadata:
+  labels:
+    app: nginx
+  annotations:
+    app: nginx
+    kyaml.kustomize.dev/kio/package: bar-package
+    kyaml.kustomize.dev/kio/path: f2.yaml
+  name: bar
+spec:
+  replicas: 3
+---
+kind: Service
+metadata:
+  name: foo
+  namespace: default
+  annotations:
+    app: nginx
+    kyaml.kustomize.dev/kio/package: .
+    kyaml.kustomize.dev/kio/path: f1.yaml
+spec:
+  selector:
+    app: nginx
+`
+	out := &bytes.Buffer{}
+	err := Pipeline{
+		Inputs:  []Reader{&ByteReader{Reader: bytes.NewBufferString(in)}},
+		Outputs: []Writer{TreeWriter{Writer: out}},
+	}.Execute()
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
+
+	if !assert.Equal(t, `
+├── [f1.yaml]  Deployment default/foo
+├── [f1.yaml]  Service default/foo
+├── [f1.yaml]  Deployment default/foo3
+├── [f1.yaml]  Deployment default/foo3
+├── [f1.yaml]  Deployment default/foo3
+├── [f1.yaml]  Deployment default2/foo2
+└── bar-package
+    └── [f2.yaml]  Deployment bar
+`, out.String()) {
+		t.FailNow()
+	}
+}
+
+func TestPrinter_metaError(t *testing.T) {
+	out := &bytes.Buffer{}
+	err := TreeWriter{Writer: out}.Write([]*yaml.RNode{{}})
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
+	if !assert.Equal(t, `
+`, out.String()) {
+		t.FailNow()
+	}
+}
+
+func TestPrinter_Write_owners(t *testing.T) {
+	in := `
+apiVersion: v1
+kind: Pod
+metadata:
+  name: cockroachdb-0
+  namespace: myapp-staging
+  ownerReferences:
+  - apiVersion: apps/v1
+    kind: StatefulSet
+    name: cockroachdb
+spec:
+  containers:
+  - name: cockroachdb
+    image: cockraochdb:1.1.1
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: cockroachdb-1
+  namespace: myapp-staging
+  ownerReferences:
+  - apiVersion: apps/v1
+    kind: StatefulSet
+    name: cockroachdb
+spec:
+  containers:
+  - name: cockroachdb
+    image: cockraochdb:1.1.1
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: cockroachdb-2
+  namespace: myapp-staging
+  ownerReferences:
+  - apiVersion: apps/v1
+    kind: StatefulSet
+    name: cockroachdb
+spec:
+  containers:
+  - name: cockroachdb
+    image: cockraochdb:1.1.0
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: cockroachdb
+  namespace: myapp-staging
+  ownerReferences:
+  - apiVersion: app.k8s.io/v1beta1
+    kind: Application
+    name: myapp
+spec:
+  replicas: 3
+  containers:
+  - name: cockroachdb
+    image: cockraochdb:1.1.1
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: cockroachdb
+  namespace: myapp-staging
+  ownerReferences:
+  - apiVersion: app.k8s.io/v1beta1
+    kind: Application
+    name: myapp
+---
+apiVersion: app.k8s.io/v1beta1
+kind: Application
+metadata:
+  labels:
+    app.kubernetes.io/name: myapp
+  name: myapp
+  namespace: myapp-staging
+`
+	out := &bytes.Buffer{}
+	err := Pipeline{
+		Inputs:  []Reader{&ByteReader{Reader: bytes.NewBufferString(in)}},
+		Outputs: []Writer{TreeWriter{Writer: out, Structure: TreeStructureGraph}},
+	}.Execute()
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
+
+	if !assert.Equal(t, `.
+└── [Resource]  Application myapp-staging/myapp
+    ├── [Resource]  Service myapp-staging/cockroachdb
+    └── [Resource]  StatefulSet myapp-staging/cockroachdb
+        ├── [Resource]  Pod myapp-staging/cockroachdb-0
+        ├── [Resource]  Pod myapp-staging/cockroachdb-1
+        └── [Resource]  Pod myapp-staging/cockroachdb-2
+`, out.String()) {
+		t.FailNow()
+	}
+}