Merge pull request #2053 from pwittrock/master

cmd/config run scoping and path defaulting

kyaml/kio/filters/container.go

@@ -8,8 +8,10 @@ import (
 	"fmt"
 	"os"
 	"os/exec"
+	"path"
 	"strings"
 
+	"sigs.k8s.io/kustomize/kyaml/errors"
 	"sigs.k8s.io/kustomize/kyaml/kio"
 	"sigs.k8s.io/kustomize/kyaml/kio/kioutil"
 
@@ -24,6 +26,106 @@ import (
 // non-zero.
 // The full set of environment variables from the parent process
 // are passed to the container.
+//
+// Function Scoping:
+// ContainerFilter applies the function only to Resources to which it is scoped.
+//
+// Resources are scoped to a function if any of the following are true:
+// - the Resource were read from the same directory as the function config
+// - the Resource were read from a subdirectory of the function config directory
+// - the function config is in a directory named "functions" and
+//   they were read from a subdirectory of "functions" parent
+// - the function config doesn't have a path annotation (considered globally scoped)
+// - the ContainerFilter has GlobalScope == true
+//
+// In Scope Examples:
+//
+// Example 1: deployment.yaml and service.yaml in function.yaml scope
+//            same directory as the function config directory
+//     .
+//     ├── function.yaml
+//     ├── deployment.yaml
+//     └── service.yaml
+//
+// Example 2: apps/deployment.yaml and apps/service.yaml in function.yaml scope
+//            subdirectory of the function config directory
+//     .
+//     ├── function.yaml
+//     └── apps
+//         ├── deployment.yaml
+//         └── service.yaml
+//
+// Example 3: apps/deployment.yaml and apps/service.yaml in functions/function.yaml scope
+//            function config is in a directory named "functions"
+//     .
+//     ├── functions
+//     │   └── function.yaml
+//     └── apps
+//         ├── deployment.yaml
+//         └── service.yaml
+//
+// Out of Scope Examples:
+//
+// Example 1: apps/deployment.yaml and apps/service.yaml NOT in stuff/function.yaml scope
+//     .
+//     ├── stuff
+//     │   └── function.yaml
+//     └── apps
+//         ├── deployment.yaml
+//         └── service.yaml
+//
+// Example 2: apps/deployment.yaml and apps/service.yaml NOT in stuff/functions/function.yaml scope
+//     .
+//     ├── stuff
+//     │   └── functions
+//     │       └── function.yaml
+//     └── apps
+//         ├── deployment.yaml
+//         └── service.yaml
+//
+// Default Paths:
+// Resources emitted by functions will have default path applied as annotations
+// if none is present.
+// The default path will be the function-dir/ (or parent directory in the case of "functions")
+// + function-file-name/ + namespace/ + kind_name.yaml
+//
+// Example 1: Given a function in fn.yaml that produces a Deployment name foo and a Service named bar
+//     dir
+//     └── fn.yaml
+//
+// Would default newly generated Resources to:
+//
+//     dir
+//     ├── fn.yaml
+//     └── fn
+//         ├── deployment_foo.yaml
+//         └── service_bar.yaml
+//
+// Example 2: Given a function in functions/fn.yaml that produces a Deployment name foo and a Service named bar
+//     dir
+//     └── fn.yaml
+//
+// Would default newly generated Resources to:
+//
+//     dir
+//     ├── functions
+//     │   └── fn.yaml
+//     └── fn
+//         ├── deployment_foo.yaml
+//         └── service_bar.yaml
+//
+// Example 3: Given a function in fn.yaml that produces a Deployment name foo, namespace baz and a Service named bar namespace baz
+//     dir
+//     └── fn.yaml
+//
+// Would default newly generated Resources to:
+//
+//     dir
+//     ├── fn.yaml
+//     └── fn
+//         └── baz
+//             ├── deployment_foo.yaml
+//             └── service_bar.yaml
 type ContainerFilter struct {
 
 	// Image is the container image to use to create a container.
@@ -40,6 +142,10 @@ type ContainerFilter struct {
 	// Typically a Kubernetes style Resource Config.
 	Config *yaml.RNode `yaml:"config,omitempty"`
 
+	// GlobalScope will cause the function to be run against all input
+	// nodes instead of only nodes scoped under the function.
+	GlobalScope bool
+
 	// args may be specified by tests to override how a container is spawned
 	args []string
 
@@ -65,8 +171,76 @@ func (s *StorageMount) String() string {
 	return fmt.Sprintf("type=%s,src=%s,dst=%s:ro", s.MountType, s.Src, s.DstPath)
 }
 
+// functionsDirectoryName is keyword directory name for functions scoped 1 directory higher
+const functionsDirectoryName = "functions"
+
+// getFunctionScope returns the path of the directory containing the function config,
+// or its parent directory if the base directory is named "functions"
+func (c *ContainerFilter) getFunctionScope() (string, error) {
+	m, err := c.Config.GetMeta()
+	if err != nil {
+		return "", errors.Wrap(err)
+	}
+	p, found := m.Annotations[kioutil.PathAnnotation]
+	if !found {
+		return "", nil
+	}
+
+	functionDir := path.Clean(path.Dir(p))
+
+	if path.Base(functionDir) == functionsDirectoryName {
+		// the scope of functions in a directory called "functions" is 1 level higher
+		// this is similar to how the golang "internal" directory scoping works
+		functionDir = path.Dir(functionDir)
+	}
+	return functionDir, nil
+}
+
+// scope partitions the input nodes into 2 slices.  The first slice contains only Resources
+// which are scoped under dir, and the second slice contains the Resources which are not.
+func (c *ContainerFilter) scope(dir string, nodes []*yaml.RNode) ([]*yaml.RNode, []*yaml.RNode, error) {
+	// scope container filtered Resources to Resources under that directory
+	var input, saved []*yaml.RNode
+	if c.GlobalScope {
+		return nodes, nil, nil
+	}
+
+	if dir == "" {
+		// global function
+		return nodes, nil, nil
+	}
+
+	// identify Resources read from directories under the function configuration
+	for i := range nodes {
+		m, err := nodes[i].GetMeta()
+		if err != nil {
+			return nil, nil, err
+		}
+		p, found := m.Annotations[kioutil.PathAnnotation]
+		if !found {
+			// this Resource isn't scoped under the function -- don't know where it came from
+			// consider it out of scope
+			saved = append(saved, nodes[i])
+			continue
+		}
+
+		resourceDir := path.Clean(path.Dir(p))
+		if !strings.HasPrefix(resourceDir, dir) {
+			// this Resource doesn't fall under the function scope if it
+			// isn't in a subdirectory of where the function lives
+			saved = append(saved, nodes[i])
+			continue
+		}
+
+		// this input is scoped under the function
+		input = append(input, nodes[i])
+	}
+
+	return input, saved, nil
+}
+
 // GrepFilter implements kio.GrepFilter
-func (c *ContainerFilter) Filter(input []*yaml.RNode) ([]*yaml.RNode, error) {
+func (c *ContainerFilter) Filter(nodes []*yaml.RNode) ([]*yaml.RNode, error) {
 	// get the command to filter the Resources
 	cmd, err := c.getCommand()
 	if err != nil {
@@ -76,11 +250,23 @@ func (c *ContainerFilter) Filter(input []*yaml.RNode) ([]*yaml.RNode, error) {
 	in := &bytes.Buffer{}
 	out := &bytes.Buffer{}
 
+	// only process Resources scoped to this function, save the others
+	functionDir, err := c.getFunctionScope()
+	if err != nil {
+		return nil, err
+	}
+	input, saved, err := c.scope(functionDir, nodes)
+	if err != nil {
+		return nil, err
+	}
+
 	// write the input
 	err = kio.ByteWriter{
-		WrappingAPIVersion: kio.ResourceListAPIVersion,
+		WrappingAPIVersion:    kio.ResourceListAPIVersion,
-		WrappingKind:       kio.ResourceListKind,
+		WrappingKind:          kio.ResourceListKind,
-		Writer:             in, KeepReaderAnnotations: true, FunctionConfig: c.Config}.Write(input)
+		Writer:                in,
+		KeepReaderAnnotations: true,
+		FunctionConfig:        c.Config}.Write(input)
 	if err != nil {
 		return nil, err
 	}
@@ -98,7 +284,19 @@ func (c *ContainerFilter) Filter(input []*yaml.RNode) ([]*yaml.RNode, error) {
 		return nil, err
 	}
 
-	return r.Read()
+	output, err := r.Read()
+	if err != nil {
+		return nil, err
+	}
+
+	// annotate any generated Resources with a path and index if they don't already have one
+	if err := kioutil.DefaultPathAnnotation(functionDir, output); err != nil {
+		return nil, err
+	}
+
+	// emit both the Resources output from the function, and the out-of-scope Resources
+	// which were not provided to the function
+	return append(output, saved...), nil
 }
 
 // getArgs returns the command + args to run to spawn the container
@@ -139,7 +337,7 @@ func (c *ContainerFilter) getArgs() []string {
 	return append(args, c.Image)
 }
 
-// getCommand returns a command which will apply the GrepFilter using the container image
+// getCommand returns a command which will apply the Filter using the container image
 func (c *ContainerFilter) getCommand() (*exec.Cmd, error) {
 	// encode the filter command API configuration
 	cfg := &bytes.Buffer{}

kyaml/kio/filters/container_test.go

@@ -16,7 +16,7 @@ import (
 )
 
 func TestFilter_command(t *testing.T) {
-	cfg, err := yaml.Parse(`apiversion: apps/v1
+	cfg, err := yaml.Parse(`apiVersion: apps/v1
 kind: Deployment
 metadata:
   name: foo
@@ -62,7 +62,7 @@ metadata:
 }
 
 func TestFilter_command_StorageMount(t *testing.T) {
-	cfg, err := yaml.Parse(`apiversion: apps/v1
+	cfg, err := yaml.Parse(`apiVersion: apps/v1
 kind: Deployment
 metadata:
   name: foo
@@ -103,7 +103,7 @@ metadata:
 }
 
 func TestFilter_command_network(t *testing.T) {
-	cfg, err := yaml.Parse(`apiversion: apps/v1
+	cfg, err := yaml.Parse(`apiVersion: apps/v1
 kind: Deployment
 metadata:
   name: foo
@@ -213,6 +213,7 @@ metadata:
   name: deployment-foo
   annotations:
     config.kubernetes.io/index: '0'
+    config.kubernetes.io/path: 'statefulset_deployment-foo.yaml'
 ---
 apiVersion: v1
 kind: Service
@@ -220,11 +221,12 @@ metadata:
   name: service-foo
   annotations:
     config.kubernetes.io/index: '1'
+    config.kubernetes.io/path: 'service_service-foo.yaml'
 `, b.String())
 }
 
 func TestFilter_Filter_noChange(t *testing.T) {
-	cfg, err := yaml.Parse(`apiversion: apps/v1
+	cfg, err := yaml.Parse(`apiVersion: apps/v1
 kind: Deployment
 metadata:
   name: foo
@@ -234,7 +236,7 @@ metadata:
 	}
 
 	input, err := (&kio.ByteReader{Reader: bytes.NewBufferString(`
-apiversion: apps/v1
+apiVersion: apps/v1
 kind: Deployment
 metadata:
   name: deployment-foo
@@ -258,7 +260,7 @@ metadata:
 			if !assert.Equal(t, `apiVersion: config.kubernetes.io/v1alpha1
 kind: ResourceList
 items:
-- apiversion: apps/v1
+- apiVersion: apps/v1
   kind: Deployment
   metadata:
     name: deployment-foo
@@ -270,7 +272,7 @@ items:
     name: service-foo
     annotations:
       config.kubernetes.io/index: '1'
-functionConfig: {apiversion: apps/v1, kind: Deployment, metadata: {name: foo}}
+functionConfig: {apiVersion: apps/v1, kind: Deployment, metadata: {name: foo}}
 `, s) {
 				t.FailNow()
 			}
@@ -289,12 +291,13 @@ functionConfig: {apiversion: apps/v1, kind: Deployment, metadata: {name: foo}}
 		return
 	}
 
-	assert.Equal(t, `apiversion: apps/v1
+	assert.Equal(t, `apiVersion: apps/v1
 kind: Deployment
 metadata:
   name: deployment-foo
   annotations:
     config.kubernetes.io/index: '0'
+    config.kubernetes.io/path: 'deployment_deployment-foo.yaml'
 ---
 apiVersion: v1
 kind: Service
@@ -302,6 +305,7 @@ metadata:
   name: service-foo
   annotations:
     config.kubernetes.io/index: '1'
+    config.kubernetes.io/path: 'service_service-foo.yaml'
 `, b.String())
 }
 
@@ -359,3 +363,418 @@ metadata:
 	c, _ = GetContainerName(n)
 	assert.Equal(t, "", c)
 }
+
+func TestFilter_Filter_defaultNaming(t *testing.T) {
+	cfg, err := yaml.Parse(`apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: foo
+  annotations:
+    config.kubernetes.io/path: 'foo/bar.yaml'
+`)
+	if !assert.NoError(t, err) {
+		return
+	}
+
+	input, err := (&kio.ByteReader{Reader: bytes.NewBufferString(``)}).Read()
+	if !assert.NoError(t, err) {
+		return
+	}
+
+	called := false
+	result, err := (&ContainerFilter{
+		Image:  "example.com:version",
+		Config: cfg,
+		args: []string{"echo", `apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: deployment-foo
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: service-foo
+`},
+		checkInput: func(s string) {
+			called = true
+			if !assert.Equal(t, `apiVersion: config.kubernetes.io/v1alpha1
+kind: ResourceList
+items: []
+functionConfig: {apiVersion: apps/v1, kind: Deployment, metadata: {name: foo, annotations: {
+      config.kubernetes.io/path: 'foo/bar.yaml'}}}
+`, s) {
+				t.FailNow()
+			}
+		},
+	}).Filter(input)
+	if !assert.NoError(t, err) {
+		return
+	}
+	if !assert.True(t, called) {
+		return
+	}
+
+	b := &bytes.Buffer{}
+	err = kio.ByteWriter{Writer: b, KeepReaderAnnotations: true}.Write(result)
+	if !assert.NoError(t, err) {
+		return
+	}
+
+	assert.Equal(t, `apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: deployment-foo
+  annotations:
+    config.kubernetes.io/index: '0'
+    config.kubernetes.io/path: 'foo/deployment_deployment-foo.yaml'
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: service-foo
+  annotations:
+    config.kubernetes.io/index: '1'
+    config.kubernetes.io/path: 'foo/service_service-foo.yaml'
+`, b.String())
+}
+
+func TestFilter_Filter_defaultNamingFunctions(t *testing.T) {
+	cfg, err := yaml.Parse(`apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: foo
+  annotations:
+    config.kubernetes.io/path: 'foo/functions/bar.yaml'
+`)
+	if !assert.NoError(t, err) {
+		return
+	}
+
+	input, err := (&kio.ByteReader{Reader: bytes.NewBufferString(``)}).Read()
+	if !assert.NoError(t, err) {
+		return
+	}
+
+	called := false
+	result, err := (&ContainerFilter{
+		Image:  "example.com:version",
+		Config: cfg,
+		args: []string{"echo", `apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: deployment-foo
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: service-foo
+`},
+		checkInput: func(s string) {
+			called = true
+			if !assert.Equal(t, `apiVersion: config.kubernetes.io/v1alpha1
+kind: ResourceList
+items: []
+functionConfig: {apiVersion: apps/v1, kind: Deployment, metadata: {name: foo, annotations: {
+      config.kubernetes.io/path: 'foo/functions/bar.yaml'}}}
+`, s) {
+				t.FailNow()
+			}
+		},
+	}).Filter(input)
+	if !assert.NoError(t, err) {
+		return
+	}
+	if !assert.True(t, called) {
+		return
+	}
+
+	b := &bytes.Buffer{}
+	err = kio.ByteWriter{Writer: b, KeepReaderAnnotations: true}.Write(result)
+	if !assert.NoError(t, err) {
+		return
+	}
+
+	assert.Equal(t, `apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: deployment-foo
+  annotations:
+    config.kubernetes.io/index: '0'
+    config.kubernetes.io/path: 'foo/deployment_deployment-foo.yaml'
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: service-foo
+  annotations:
+    config.kubernetes.io/index: '1'
+    config.kubernetes.io/path: 'foo/service_service-foo.yaml'
+`, b.String())
+}
+
+func TestFilter_Filter_scopeMissingFromResource(t *testing.T) {
+	cfg, err := yaml.Parse(`apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: foo
+  annotations:
+    config.kubernetes.io/path: 'foo/bar.yaml'
+`)
+	if !assert.NoError(t, err) {
+		return
+	}
+
+	input, err := (&kio.ByteReader{Reader: bytes.NewBufferString(`
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: deployment-foo
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: service-foo
+`)}).Read()
+	if !assert.NoError(t, err) {
+		return
+	}
+
+	// no resources match the scope
+	called := false
+	result, err := (&ContainerFilter{
+		Image:  "example.com:version",
+		Config: cfg,
+		args:   []string{"sed", "s/Deployment/StatefulSet/g"},
+		checkInput: func(s string) {
+			called = true
+			if !assert.Equal(t, `apiVersion: config.kubernetes.io/v1alpha1
+kind: ResourceList
+items: []
+functionConfig: {apiVersion: apps/v1, kind: Deployment, metadata: {name: foo, annotations: {
+      config.kubernetes.io/path: 'foo/bar.yaml'}}}
+`, s) {
+				t.FailNow()
+			}
+		},
+	}).Filter(input)
+	if !assert.NoError(t, err) {
+		return
+	}
+	if !assert.True(t, called) {
+		return
+	}
+
+	b := &bytes.Buffer{}
+	err = kio.ByteWriter{Writer: b, KeepReaderAnnotations: true}.Write(result)
+	if !assert.NoError(t, err) {
+		return
+	}
+
+	// Resources should be preserved -- paths shouldn't be set by container
+	assert.Equal(t, `apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: deployment-foo
+  annotations:
+    config.kubernetes.io/index: '0'
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: service-foo
+  annotations:
+    config.kubernetes.io/index: '1'
+`, b.String())
+}
+
+func TestFilter_Filter_scopeFunctionsDir(t *testing.T) {
+	// functions under "functions/" dir should be scoped to parent dir
+	cfg, err := yaml.Parse(`apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: foo
+  annotations:
+    config.kubernetes.io/path: 'foo/functions/bar.yaml'
+`)
+	if !assert.NoError(t, err) {
+		return
+	}
+
+	input, err := (&kio.ByteReader{Reader: bytes.NewBufferString(`
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: deployment-foo
+  annotations:
+    config.kubernetes.io/path: 'foo/bar/d.yaml'
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: service-foo
+  annotations:
+    config.kubernetes.io/path: 'foo/bar/s.yaml'
+`)}).Read()
+	if !assert.NoError(t, err) {
+		return
+	}
+
+	// no resources match the scope
+	called := false
+	result, err := (&ContainerFilter{
+		Image:  "example.com:version",
+		Config: cfg,
+		args:   []string{"sed", "s/Deployment/StatefulSet/g"},
+		checkInput: func(s string) {
+			called = true
+			if !assert.Equal(t, `apiVersion: config.kubernetes.io/v1alpha1
+kind: ResourceList
+items:
+- apiVersion: apps/v1
+  kind: Deployment
+  metadata:
+    name: deployment-foo
+    annotations:
+      config.kubernetes.io/path: 'foo/bar/d.yaml'
+      config.kubernetes.io/index: '0'
+- apiVersion: v1
+  kind: Service
+  metadata:
+    name: service-foo
+    annotations:
+      config.kubernetes.io/path: 'foo/bar/s.yaml'
+      config.kubernetes.io/index: '1'
+functionConfig: {apiVersion: apps/v1, kind: Deployment, metadata: {name: foo, annotations: {
+      config.kubernetes.io/path: 'foo/functions/bar.yaml'}}}
+`, s) {
+				t.FailNow()
+			}
+		},
+	}).Filter(input)
+	if !assert.NoError(t, err) {
+		return
+	}
+	if !assert.True(t, called) {
+		return
+	}
+
+	b := &bytes.Buffer{}
+	err = kio.ByteWriter{Writer: b, KeepReaderAnnotations: true}.Write(result)
+	if !assert.NoError(t, err) {
+		return
+	}
+
+	// Resources should be modified
+	assert.Equal(t, `apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: deployment-foo
+  annotations:
+    config.kubernetes.io/path: 'foo/bar/d.yaml'
+    config.kubernetes.io/index: '0'
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: service-foo
+  annotations:
+    config.kubernetes.io/path: 'foo/bar/s.yaml'
+    config.kubernetes.io/index: '1'
+`, b.String())
+}
+
+func TestFilter_Filter_scopeDir(t *testing.T) {
+	// functions under "functions/" dir should be scoped to parent dir
+	cfg, err := yaml.Parse(`apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: foo
+  annotations:
+    config.kubernetes.io/path: 'foo/bar.yaml'
+`)
+	if !assert.NoError(t, err) {
+		return
+	}
+
+	input, err := (&kio.ByteReader{Reader: bytes.NewBufferString(`
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: deployment-foo
+  annotations:
+    config.kubernetes.io/path: 'foo/bar/d.yaml'
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: service-foo
+  annotations:
+    config.kubernetes.io/path: 'foo/bar/s.yaml'
+`)}).Read()
+	if !assert.NoError(t, err) {
+		return
+	}
+
+	// no resources match the scope
+	called := false
+	result, err := (&ContainerFilter{
+		Image:  "example.com:version",
+		Config: cfg,
+		args:   []string{"sed", "s/Deployment/StatefulSet/g"},
+		checkInput: func(s string) {
+			called = true
+			if !assert.Equal(t, `apiVersion: config.kubernetes.io/v1alpha1
+kind: ResourceList
+items:
+- apiVersion: apps/v1
+  kind: Deployment
+  metadata:
+    name: deployment-foo
+    annotations:
+      config.kubernetes.io/path: 'foo/bar/d.yaml'
+      config.kubernetes.io/index: '0'
+- apiVersion: v1
+  kind: Service
+  metadata:
+    name: service-foo
+    annotations:
+      config.kubernetes.io/path: 'foo/bar/s.yaml'
+      config.kubernetes.io/index: '1'
+functionConfig: {apiVersion: apps/v1, kind: Deployment, metadata: {name: foo, annotations: {
+      config.kubernetes.io/path: 'foo/bar.yaml'}}}
+`, s) {
+				t.FailNow()
+			}
+		},
+	}).Filter(input)
+	if !assert.NoError(t, err) {
+		return
+	}
+	if !assert.True(t, called) {
+		return
+	}
+
+	b := &bytes.Buffer{}
+	err = kio.ByteWriter{Writer: b, KeepReaderAnnotations: true}.Write(result)
+	if !assert.NoError(t, err) {
+		return
+	}
+
+	// Resources should be preserved
+	assert.Equal(t, `apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: deployment-foo
+  annotations:
+    config.kubernetes.io/path: 'foo/bar/d.yaml'
+    config.kubernetes.io/index: '0'
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: service-foo
+  annotations:
+    config.kubernetes.io/path: 'foo/bar/s.yaml'
+    config.kubernetes.io/index: '1'
+`, b.String())
+}

kyaml/kio/kioutil/kioutil.go

@@ -5,8 +5,10 @@ package kioutil
 
 import (
 	"fmt"
+	"path"
 	"sort"
 	"strconv"
+	"strings"
 
 	"sigs.k8s.io/kustomize/kyaml/errors"
 	"sigs.k8s.io/kustomize/kyaml/yaml"
@@ -41,13 +43,101 @@ func ErrorIfMissingAnnotation(nodes []*yaml.RNode, keys ...AnnotationKey) error
 				return errors.Wrap(err)
 			}
 			if val == nil {
-				return errors.Errorf("missing package annotation %s", key)
+				return errors.Errorf("missing annotation %s", key)
 			}
 		}
 	}
 	return nil
 }
 
+// CreatePathAnnotationValue creates a default path annotation value for a Resource.
+// The path prefix will be dir.
+func CreatePathAnnotationValue(dir string, m yaml.ResourceMeta) string {
+	filename := fmt.Sprintf("%s_%s.yaml", strings.ToLower(m.Kind), m.Name)
+	return path.Join(dir, m.Namespace, filename)
+}
+
+// DefaultPathAndIndexAnnotation sets a default path or index value on any nodes missing the
+// annotation
+func DefaultPathAndIndexAnnotation(dir string, nodes []*yaml.RNode) error {
+	counts := map[string]int{}
+
+	// check each node for the path annotation
+	for i := range nodes {
+		m, err := nodes[i].GetMeta()
+		if err != nil {
+			return err
+		}
+
+		// calculate the max index in each file in case we are appending
+		if p, found := m.Annotations[PathAnnotation]; found {
+			// record the max indexes into each file
+			if i, found := m.Annotations[IndexAnnotation]; found {
+				index, _ := strconv.Atoi(i)
+				if index > counts[p] {
+					counts[p] = index
+				}
+			}
+
+			// has the path annotation already -- do nothing
+			continue
+		}
+
+		// set a path annotation on the Resource
+		path := CreatePathAnnotationValue(dir, m)
+		if err := nodes[i].PipeE(yaml.SetAnnotation(PathAnnotation, path)); err != nil {
+			return err
+		}
+	}
+
+	// set the index annotations
+	for i := range nodes {
+		m, err := nodes[i].GetMeta()
+		if err != nil {
+			return err
+		}
+
+		if _, found := m.Annotations[IndexAnnotation]; found {
+			continue
+		}
+
+		p := m.Annotations[PathAnnotation]
+
+		// set an index annotation on the Resource
+		c := counts[p]
+		counts[p] = c + 1
+		if err := nodes[i].PipeE(
+			yaml.SetAnnotation(IndexAnnotation, fmt.Sprintf("%d", c))); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// DefaultPathAnnotation sets a default path annotation on any Reources
+// missing it.
+func DefaultPathAnnotation(dir string, nodes []*yaml.RNode) error {
+	// check each node for the path annotation
+	for i := range nodes {
+		m, err := nodes[i].GetMeta()
+		if err != nil {
+			return err
+		}
+
+		if _, found := m.Annotations[PathAnnotation]; found {
+			// has the path annotation already -- do nothing
+			continue
+		}
+
+		// set a path annotation on the Resource
+		path := CreatePathAnnotationValue(dir, m)
+		if err := nodes[i].PipeE(yaml.SetAnnotation(PathAnnotation, path)); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 // Map invokes fn for each element in nodes.
 func Map(nodes []*yaml.RNode, fn func(*yaml.RNode) (*yaml.RNode, error)) ([]*yaml.RNode, error) {
 	var returnNodes []*yaml.RNode

kyaml/kio/kioutil/kioutil_test.go

@@ -13,6 +13,7 @@ import (
 	"github.com/stretchr/testify/assert"
 	"sigs.k8s.io/kustomize/kyaml/kio"
 	"sigs.k8s.io/kustomize/kyaml/kio/kioutil"
+	"sigs.k8s.io/kustomize/kyaml/yaml"
 )
 
 func TestSortNodes_moreThan10(t *testing.T) {
@@ -75,3 +76,257 @@ y: z
 
 	assert.Equal(t, strings.TrimSpace(input), strings.TrimSpace(actual.String()))
 }
+
+func TestDefaultPathAnnotation(t *testing.T) {
+	var tests = []struct {
+		dir      string
+		input    string // input
+		expected string // expected result
+		name     string
+	}{
+		{
+			`foo`,
+			`apiVersion: v1
+kind: Bar
+metadata:
+  name: a
+  namespace: b
+`,
+			`apiVersion: v1
+kind: Bar
+metadata:
+  name: a
+  namespace: b
+  annotations:
+    config.kubernetes.io/path: 'foo/b/bar_a.yaml'
+`, `with namespace`},
+		{
+			`foo`,
+			`apiVersion: v1
+kind: Bar
+metadata:
+  name: a
+`,
+			`apiVersion: v1
+kind: Bar
+metadata:
+  name: a
+  annotations:
+    config.kubernetes.io/path: 'foo/bar_a.yaml'
+`, `without namespace`},
+
+		{
+			``,
+			`apiVersion: v1
+kind: Bar
+metadata:
+  name: a
+  namespace: b
+`,
+			`apiVersion: v1
+kind: Bar
+metadata:
+  name: a
+  namespace: b
+  annotations:
+    config.kubernetes.io/path: 'b/bar_a.yaml'
+`, `without dir`},
+		{
+			``,
+			`apiVersion: v1
+kind: Bar
+metadata:
+  name: a
+  namespace: b
+  annotations:
+    config.kubernetes.io/path: 'a/b.yaml'
+`,
+			`apiVersion: v1
+kind: Bar
+metadata:
+  name: a
+  namespace: b
+  annotations:
+    config.kubernetes.io/path: 'a/b.yaml'
+`, `skip`},
+	}
+
+	for _, s := range tests {
+		n := yaml.MustParse(s.input)
+		err := kioutil.DefaultPathAnnotation(s.dir, []*yaml.RNode{n})
+		if !assert.NoError(t, err, s.name) {
+			t.FailNow()
+		}
+		if !assert.Equal(t, s.expected, n.MustString(), s.name) {
+			t.FailNow()
+		}
+	}
+}
+
+func TestDefaultPathAndIndexAnnotation(t *testing.T) {
+	var tests = []struct {
+		dir      string
+		input    string // input
+		expected string // expected result
+		name     string
+	}{
+		{
+			`foo`,
+			`apiVersion: v1
+kind: Bar
+metadata:
+  name: a
+  namespace: b
+`,
+			`apiVersion: v1
+kind: Bar
+metadata:
+  name: a
+  namespace: b
+  annotations:
+    config.kubernetes.io/path: 'foo/b/bar_a.yaml'
+    config.kubernetes.io/index: '0'
+`, `with namespace`},
+		{
+			`foo`,
+			`apiVersion: v1
+kind: Bar
+metadata:
+  name: a
+`,
+			`apiVersion: v1
+kind: Bar
+metadata:
+  name: a
+  annotations:
+    config.kubernetes.io/path: 'foo/bar_a.yaml'
+    config.kubernetes.io/index: '0'
+`, `without namespace`},
+
+		{
+			``,
+			`apiVersion: v1
+kind: Bar
+metadata:
+  name: a
+  namespace: b
+`,
+			`apiVersion: v1
+kind: Bar
+metadata:
+  name: a
+  namespace: b
+  annotations:
+    config.kubernetes.io/path: 'b/bar_a.yaml'
+    config.kubernetes.io/index: '0'
+`, `without dir`},
+		{
+			``,
+			`apiVersion: v1
+kind: Bar
+metadata:
+  name: a
+  namespace: b
+  annotations:
+    config.kubernetes.io/path: 'a/b.yaml'
+    config.kubernetes.io/index: '5'
+`,
+			`apiVersion: v1
+kind: Bar
+metadata:
+  name: a
+  namespace: b
+  annotations:
+    config.kubernetes.io/path: 'a/b.yaml'
+    config.kubernetes.io/index: '5'
+`, `skip`},
+	}
+
+	for _, s := range tests {
+		out := &bytes.Buffer{}
+		r := kio.ByteReadWriter{
+			Reader:                bytes.NewBufferString(s.input),
+			Writer:                out,
+			KeepReaderAnnotations: true,
+			OmitReaderAnnotations: true,
+		}
+		n, err := r.Read()
+		if !assert.NoError(t, err, s.name) {
+			t.FailNow()
+		}
+		if !assert.NoError(t, kioutil.DefaultPathAndIndexAnnotation(s.dir, n), s.name) {
+			t.FailNow()
+		}
+		if !assert.NoError(t, r.Write(n), s.name) {
+			t.FailNow()
+		}
+		if !assert.Equal(t, s.expected, out.String(), s.name) {
+			t.FailNow()
+		}
+	}
+}
+
+func TestCreatePathAnnotationValue(t *testing.T) {
+	var tests = []struct {
+		dir      string
+		meta     yaml.ResourceMeta // input
+		expected string            // expected result
+		name     string
+	}{
+		{
+			`dir`,
+			yaml.ResourceMeta{Kind: "foo",
+				APIVersion: "apps/v1",
+				ObjectMeta: yaml.ObjectMeta{Name: "bar", Namespace: "baz"},
+			},
+			`dir/baz/foo_bar.yaml`, `with namespace`,
+		},
+		{
+			``,
+			yaml.ResourceMeta{Kind: "foo",
+				APIVersion: "apps/v1",
+				ObjectMeta: yaml.ObjectMeta{Name: "bar", Namespace: "baz"},
+			},
+			`baz/foo_bar.yaml`, `without dir`,
+		},
+		{
+			`dir`,
+			yaml.ResourceMeta{Kind: "foo",
+				APIVersion: "apps/v1",
+				ObjectMeta: yaml.ObjectMeta{Name: "bar"},
+			},
+			`dir/foo_bar.yaml`, `without namespace`,
+		},
+		{
+			``,
+			yaml.ResourceMeta{Kind: "foo",
+				APIVersion: "apps/v1",
+				ObjectMeta: yaml.ObjectMeta{Name: "bar"},
+			},
+			`foo_bar.yaml`, `without namespace or dir`,
+		},
+		{
+			``,
+			yaml.ResourceMeta{Kind: "foo",
+				APIVersion: "apps/v1",
+				ObjectMeta: yaml.ObjectMeta{},
+			},
+			`foo_.yaml`, `without namespace, dir or name`,
+		},
+		{
+			``,
+			yaml.ResourceMeta{
+				APIVersion: "apps/v1",
+				ObjectMeta: yaml.ObjectMeta{},
+			},
+			`_.yaml`, `without any`,
+		},
+	}
+
+	for _, s := range tests {
+		p := kioutil.CreatePathAnnotationValue(s.dir, s.meta)
+		if !assert.Equal(t, s.expected, p, s.name) {
+			t.FailNow()
+		}
+	}
+}

kyaml/kio/pkgio_writer.go

@@ -31,7 +31,8 @@ type LocalPackageWriter struct {
 var _ Writer = LocalPackageWriter{}
 
 func (r LocalPackageWriter) Write(nodes []*yaml.RNode) error {
-	if err := kioutil.ErrorIfMissingAnnotation(nodes, requiredResourcePackageAnnotations...); err != nil {
+	// set the path and index annotations if they are missing
+	if err := kioutil.DefaultPathAndIndexAnnotation("", nodes); err != nil {
 		return err
 	}
 

kyaml/kio/pkgio_writer_test.go

@@ -207,55 +207,37 @@ metadata:
 	}
 }
 
-// TestLocalPackageWriter_Write_missingIndex tests:
+// TestLocalPackageWriter_Write_missingPath tests:
-// - If config.kubernetes.io/path is missing, fail
+// - If config.kubernetes.io/path or index are missing, then default them
-func TestLocalPackageWriter_Write_missingPath(t *testing.T) {
+func TestLocalPackageWriter_Write_missingAnnotations(t *testing.T) {
 	d, node1, node2, node3 := getWriterInputs(t)
 	defer os.RemoveAll(d)
 
-	node4, err := yaml.Parse(`e: f
+	node4String := `e: f
 g:
   h:
   - i # has a list
   - j
+kind: Foo
 metadata:
-  annotations:
+  name: bar
-    config.kubernetes.io/index: a
+`
-`)
+	node4, err := yaml.Parse(node4String)
 	if !assert.NoError(t, err) {
 		assert.FailNow(t, err.Error())
 	}
 
 	w := LocalPackageWriter{PackagePath: d}
 	err = w.Write([]*yaml.RNode{node2, node1, node3, node4})
-	if assert.Error(t, err) {
-		assert.Contains(t, err.Error(), "config.kubernetes.io/path")
-	}
-}
-
-// TestLocalPackageWriter_Write_missingIndex tests:
-// - If config.kubernetes.io/index is missing, fail
-func TestLocalPackageWriter_Write_missingIndex(t *testing.T) {
-	d, node1, node2, node3 := getWriterInputs(t)
-	defer os.RemoveAll(d)
-
-	node4, err := yaml.Parse(`e: f
-g:
-  h:
-  - i # has a list
-  - j
-metadata:
-  annotations:
-    config.kubernetes.io/path: a/a.yaml
-`)
 	if !assert.NoError(t, err) {
-		assert.FailNow(t, err.Error())
+		t.FailNow()
 	}
-
+	b, err := ioutil.ReadFile(filepath.Join(d, "foo_bar.yaml"))
-	w := LocalPackageWriter{PackagePath: d}
+	if !assert.NoError(t, err) {
-	err = w.Write([]*yaml.RNode{node2, node1, node3, node4})
+		t.FailNow()
-	if assert.Error(t, err) {
+	}
-		assert.Contains(t, err.Error(), "config.kubernetes.io/index")
+	if !assert.Equal(t, node4String, string(b)) {
+		t.FailNow()
 	}
 }
 

kyaml/runfn/runfn.go

@@ -55,7 +55,6 @@ func (r RunFns) Execute() error {
 		return err
 	}
 
-	// accept a
 	for i := range r.FunctionPaths {
 		err := kio.Pipeline{
 			Inputs:  []kio.Reader{kio.LocalPackageReader{PackagePath: r.FunctionPaths[i]}},