Merge pull request #2104 from pwittrock/master

kyaml/rnfn: support explicit fn list and reading from an io.Reader

kyaml/runfn/runfn.go

@@ -5,6 +5,7 @@ package runfn
 
 import (
 	"io"
+	"os"
 	"path"
 	"path/filepath"
 	"sort"
@@ -28,12 +29,21 @@ type RunFns struct {
 	// FunctionPaths Paths allows functions to be specified outside the configuration
 	// directory.
 	// Functions provided on FunctionPaths are globally scoped.
+	// If FunctionPaths length is > 0, then NoFunctionsFromInput defaults to true
 	FunctionPaths []string
 
+	// Functions is an explicit list of functions to run against the input.
+	// Functions provided on Functions are globally scoped.
+	// If Functions length is > 0, then NoFunctionsFromInput defaults to true
+	Functions []*yaml.RNode
+
 	// GlobalScope if true, functions read from input will be scoped globally rather
 	// than only to Resources under their subdirs.
 	GlobalScope bool
 
+	// Input can be set to read the Resources from Input rather than from a directory
+	Input io.Reader
+
 	// Output can be set to write the result to Output rather than back to the directory
 	Output io.Writer
 
@@ -56,51 +66,83 @@ func (r RunFns) Execute() error {
 
 	// default the containerFilterProvider if it hasn't been override.  Split out for testing.
 	(&r).init()
-
-	fltrs, err := r.getFilters()
+	nodes, fltrs, output, err := r.getNodesAndFilters()
 	if err != nil {
 		return err
 	}
-
-	return r.runFunctions(fltrs)
+	return r.runFunctions(nodes, output, fltrs)
 }
 
-func (r RunFns) getFilters() ([]kio.Filter, error) {
+func (r RunFns) getNodesAndFilters() (
+	*kio.PackageBuffer, []kio.Filter, *kio.LocalPackageReadWriter, error) {
+	// Read Resources from Directory or Input
+	buff := &kio.PackageBuffer{}
+	p := kio.Pipeline{Outputs: []kio.Writer{buff}}
+	// save the output dir because we will need it to write back
+	// the same one for reading must be used for writing if deleting Resources
+	var outputPkg *kio.LocalPackageReadWriter
+	if r.Path != "" {
+		outputPkg = &kio.LocalPackageReadWriter{PackagePath: r.Path}
+	}
+
+	if r.Input == nil {
+		p.Inputs = []kio.Reader{outputPkg}
+	} else {
+		p.Inputs = []kio.Reader{&kio.ByteReader{Reader: r.Input}}
+	}
+	if err := p.Execute(); err != nil {
+		return nil, nil, outputPkg, err
+	}
+
+	fltrs, err := r.getFilters(buff.Nodes)
+	if err != nil {
+		return nil, nil, outputPkg, err
+	}
+	return buff, fltrs, outputPkg, nil
+}
+
+func (r RunFns) getFilters(nodes []*yaml.RNode) ([]kio.Filter, error) {
 	var fltrs []kio.Filter
 
 	// implicit filters from the input Resources
-	f, err := r.getFunctionsFromInput()
+	f, err := r.getFunctionsFromInput(nodes)
 	if err != nil {
 		return nil, err
 	}
 	fltrs = append(fltrs, f...)
 
 	// explicit filters from a list of directories
-	f, err = r.getFunctionsFromDirList()
+	f, err = r.getFunctionsFromFunctionPaths()
 	if err != nil {
 		return nil, err
 	}
 	fltrs = append(fltrs, f...)
+
+	// explicit filters from a list of directories
+	f = r.getFunctionsFromFunctions()
+	fltrs = append(fltrs, f...)
+
 	return fltrs, nil
 }
 
-// runFunctions runs the fltrs against the input
-func (r RunFns) runFunctions(fltrs []kio.Filter) error {
-	pkgIO := &kio.LocalPackageReadWriter{PackagePath: r.Path}
-	inputs := []kio.Reader{pkgIO}
+// runFunctions runs the fltrs against the input and writes to either r.Output or output
+func (r RunFns) runFunctions(
+	input kio.Reader, output kio.Writer, fltrs []kio.Filter) error {
+	// use the previously read Resources as input
 	var outputs []kio.Writer
 	if r.Output == nil {
 		// write back to the package
-		outputs = append(outputs, pkgIO)
+		outputs = append(outputs, output)
 	} else {
-		// write to the output instead of the directory
+		// write to the output instead of the directory if r.Output is specified or
+		// the output is nil (reading from Input)
 		outputs = append(outputs, kio.ByteWriter{Writer: r.Output})
 	}
-	return kio.Pipeline{Inputs: inputs, Filters: fltrs, Outputs: outputs}.Execute()
+	return kio.Pipeline{Inputs: []kio.Reader{input}, Filters: fltrs, Outputs: outputs}.Execute()
 }
 
 // getFunctionsFromInput scans the input for functions and runs them
-func (r RunFns) getFunctionsFromInput() ([]kio.Filter, error) {
+func (r RunFns) getFunctionsFromInput(nodes []*yaml.RNode) ([]kio.Filter, error) {
 	if *r.NoFunctionsFromInput {
 		return nil, nil
 	}
@@ -108,7 +150,7 @@ func (r RunFns) getFunctionsFromInput() ([]kio.Filter, error) {
 	var fltrs []kio.Filter
 	buff := &kio.PackageBuffer{}
 	err := kio.Pipeline{
-		Inputs:  []kio.Reader{kio.LocalPackageReader{PackagePath: r.Path}},
+		Inputs:  []kio.Reader{&kio.PackageBuffer{Nodes: nodes}},
 		Filters: []kio.Filter{&filters.IsReconcilerFilter{}},
 		Outputs: []kio.Writer{buff},
 	}.Execute()
@@ -124,9 +166,9 @@ func (r RunFns) getFunctionsFromInput() ([]kio.Filter, error) {
 	return fltrs, nil
 }
 
-// getFunctionsFromDirList returns the set of functions read from r.FunctionPaths
+// getFunctionsFromFunctionPaths returns the set of functions read from r.FunctionPaths
 // as a slice of Filters
-func (r RunFns) getFunctionsFromDirList() ([]kio.Filter, error) {
+func (r RunFns) getFunctionsFromFunctionPaths() ([]kio.Filter, error) {
 	var fltrs []kio.Filter
 	buff := &kio.PackageBuffer{}
 	for i := range r.FunctionPaths {
@@ -144,7 +186,7 @@ func (r RunFns) getFunctionsFromDirList() ([]kio.Filter, error) {
 		c := r.containerFilterProvider(img, path, api)
 		cf, ok := c.(*filters.ContainerFilter)
 		if ok {
-			// functions provided on FunctionPaths are globally scoped
+			// functions provided by FunctionPaths are globally scoped
 			cf.GlobalScope = true
 		}
 		fltrs = append(fltrs, c)
@@ -152,6 +194,24 @@ func (r RunFns) getFunctionsFromDirList() ([]kio.Filter, error) {
 	return fltrs, nil
 }
 
+// getFunctionsFromFunctions returns the set of explicitly provided functions as
+// Filters
+func (r RunFns) getFunctionsFromFunctions() []kio.Filter {
+	var fltrs []kio.Filter
+	for i := range r.Functions {
+		api := r.Functions[i]
+		img, path := filters.GetContainerName(api)
+		c := r.containerFilterProvider(img, path, api)
+		cf, ok := c.(*filters.ContainerFilter)
+		if ok {
+			// functions provided by Functions are globally scoped
+			cf.GlobalScope = true
+		}
+		fltrs = append(fltrs, c)
+	}
+	return fltrs
+}
+
 // sortFns sorts functions so that functions with the longest paths come first
 func sortFns(buff *kio.PackageBuffer) {
 	// sort the nodes so that we traverse them depth first
@@ -201,10 +261,21 @@ func sortFns(buff *kio.PackageBuffer) {
 // init initializes the RunFns with a containerFilterProvider.
 func (r *RunFns) init() {
 	if r.NoFunctionsFromInput == nil {
-		nfn := len(r.FunctionPaths) > 0
+		// default no functions from input if any function sources are explicitly provided
+		nfn := len(r.FunctionPaths) > 0 || len(r.Functions) > 0
 		r.NoFunctionsFromInput = &nfn
 	}
 
+	// if no path is specified, default reading from stdin and writing to stdout
+	if r.Path == "" {
+		if r.Output == nil {
+			r.Output = os.Stdout
+		}
+		if r.Input == nil {
+			r.Input = os.Stdin
+		}
+	}
+
 	// if containerFilterProvider hasn't been set, use the default
 	if r.containerFilterProvider == nil {
 		r.containerFilterProvider = func(image, path string, api *yaml.RNode) kio.Filter {

kyaml/runfn/runfn_test.go

@@ -34,9 +34,16 @@ replace: StatefulSet
 `
 )
 
-func TestRunFns_Execute(t *testing.T) {
+func TestRunFns_init(t *testing.T) {
 	instance := RunFns{}
 	instance.init()
+	if !assert.Equal(t, instance.Input, os.Stdin) {
+		t.FailNow()
+	}
+	if !assert.Equal(t, instance.Output, os.Stdout) {
+		t.FailNow()
+	}
+
 	api, err := yaml.Parse(`apiVersion: apps/v1
 kind: 
 `)
@@ -47,9 +54,15 @@ kind:
 	assert.Equal(t, &filters.ContainerFilter{Image: "example.com:version", Config: api}, filter)
 }
 
-func TestRunFns_Execute_globalScope(t *testing.T) {
+func TestRunFns_Execute__initGlobalScope(t *testing.T) {
 	instance := RunFns{GlobalScope: true}
 	instance.init()
+	if !assert.Equal(t, instance.Input, os.Stdin) {
+		t.FailNow()
+	}
+	if !assert.Equal(t, instance.Output, os.Stdout) {
+		t.FailNow()
+	}
 	api, err := yaml.Parse(`apiVersion: apps/v1
 kind: 
 `)
@@ -61,8 +74,80 @@ kind:
 		Image: "example.com:version", Config: api, GlobalScope: true}, filter)
 }
 
-var tru = true
-var fls = false
+func TestRunFns_Execute__initDefault(t *testing.T) {
+	b := &bytes.Buffer{}
+	var tests = []struct {
+		instance RunFns
+		expected RunFns
+		name     string
+	}{
+		{
+			instance: RunFns{},
+			name:     "empty",
+			expected: RunFns{Output: os.Stdout, Input: os.Stdin, NoFunctionsFromInput: getFalse()},
+		},
+		{
+			name:     "explicit output",
+			instance: RunFns{Output: b},
+			expected: RunFns{Output: b, Input: os.Stdin, NoFunctionsFromInput: getFalse()},
+		},
+		{
+			name:     "explicit input",
+			instance: RunFns{Input: b},
+			expected: RunFns{Output: os.Stdout, Input: b, NoFunctionsFromInput: getFalse()},
+		},
+		{
+			name:     "explicit functions -- no functions from input",
+			instance: RunFns{Functions: []*yaml.RNode{{}}},
+			expected: RunFns{Output: os.Stdout, Input: os.Stdin, NoFunctionsFromInput: getTrue(), Functions: []*yaml.RNode{{}}},
+		},
+		{
+			name:     "explicit functions -- yes functions from input",
+			instance: RunFns{Functions: []*yaml.RNode{{}}, NoFunctionsFromInput: getFalse()},
+			expected: RunFns{Output: os.Stdout, Input: os.Stdin, NoFunctionsFromInput: getFalse(), Functions: []*yaml.RNode{{}}},
+		},
+		{
+			name:     "explicit functions in paths -- no functions from input",
+			instance: RunFns{FunctionPaths: []string{"foo"}},
+			expected: RunFns{
+				Output:               os.Stdout,
+				Input:                os.Stdin,
+				NoFunctionsFromInput: getTrue(),
+				FunctionPaths:        []string{"foo"},
+			},
+		},
+		{
+			name:     "functions in paths -- yes functions from input",
+			instance: RunFns{FunctionPaths: []string{"foo"}, NoFunctionsFromInput: getFalse()},
+			expected: RunFns{
+				Output:               os.Stdout,
+				Input:                os.Stdin,
+				NoFunctionsFromInput: getFalse(),
+				FunctionPaths:        []string{"foo"},
+			},
+		},
+	}
+	for i := range tests {
+		tt := tests[i]
+		t.Run(tt.name, func(t *testing.T) {
+			(&tt.instance).init()
+			(&tt.instance).containerFilterProvider = nil
+			if !assert.Equal(t, tt.expected, tt.instance) {
+				t.FailNow()
+			}
+		})
+	}
+}
+
+func getTrue() *bool {
+	t := true
+	return &t
+}
+
+func getFalse() *bool {
+	f := false
+	return &f
+}
 
 // TestRunFns_getFilters tests how filters are found and sorted
 func TestRunFns_getFilters(t *testing.T) {
@@ -72,6 +157,10 @@ func TestRunFns_getFilters(t *testing.T) {
 		// if true, create the function in a separate directory from
 		// the config, and provide it through FunctionPaths
 		outOfPackage bool
+
+		// if true, create the function as an explicit Functions input
+		explicitFunction bool
+
 		// if true and outOfPackage is true, create a new directory
 		// for this function separate from the previous one.  If
 		// false and outOfPackage is true, create the function in
@@ -173,7 +262,7 @@ metadata:
 		//
 		//
 		{name: "sort functions -- skip implicit",
-			noFunctionsFromInput: &tru,
+			noFunctionsFromInput: getTrue(),
 			in: []f{
 				{
 					path: filepath.Join("foo", "a.yaml"),
@@ -203,7 +292,7 @@ metadata:
 		//
 		//
 		{name: "sort functions -- include implicit",
-			noFunctionsFromInput: &fls,
+			noFunctionsFromInput: getFalse(),
 			in: []f{
 				{
 					path: filepath.Join("foo", "a.yaml"),
@@ -233,7 +322,7 @@ metadata:
 		//
 		//
 		{name: "sort functions -- implicit first",
-			noFunctionsFromInput: &fls,
+			noFunctionsFromInput: getFalse(),
 			in: []f{
 				{
 					path:         filepath.Join("foo", "a.yaml"),
@@ -259,6 +348,76 @@ metadata:
 			},
 			out: []string{"b", "a"},
 		},
+
+		// Test
+		//
+		//
+		{name: "explicit functions",
+			in: []f{
+				{
+					explicitFunction: true,
+					value: `
+metadata:
+  annotations:
+    config.kubernetes.io/function: |
+      container:
+        image: c
+`,
+				},
+				{
+					path: filepath.Join("b.yaml"),
+					value: `
+metadata:
+  annotations:
+    config.kubernetes.io/function: |
+      container:
+        image: b
+`,
+				},
+			},
+			out: []string{"c"},
+		},
+
+		// Test
+		//
+		//
+		{name: "sort functions -- implicit first",
+			noFunctionsFromInput: getFalse(),
+			in: []f{
+				{
+					explicitFunction: true,
+					value: `
+metadata:
+  annotations:
+    config.kubernetes.io/function: |
+      container:
+        image: c
+`,
+				},
+				{
+					path:         filepath.Join("foo", "a.yaml"),
+					outOfPackage: true, // out of package is run last
+					value: `
+metadata:
+  annotations:
+    config.kubernetes.io/function: |
+      container:
+        image: a
+`,
+				},
+				{
+					path: filepath.Join("b.yaml"),
+					value: `
+metadata:
+  annotations:
+    config.kubernetes.io/function: |
+      container:
+        image: b
+`,
+				},
+			},
+			out: []string{"b", "a", "c"},
+		},
 	}
 
 	for i := range tests {
@@ -270,12 +429,14 @@ metadata:
 
 			// write the functions to files
 			var fnPaths []string
+			var parsedFns []*yaml.RNode
 			var fnPath string
 			var err error
 			for _, f := range tt.in {
 				// get the location for the file
 				var dir string
-				if f.outOfPackage {
+				switch {
+				case f.outOfPackage:
 					// if out of package, write to a separate temp directory
 					if f.newFnPath || fnPath == "" {
 						// create a new fn directory
@@ -287,25 +448,30 @@ metadata:
 						fnPaths = append(fnPaths, fnPath)
 					}
 					dir = fnPath
-				} else {
+				case f.explicitFunction:
+					parsedFns = append(parsedFns, yaml.MustParse(f.value))
+				default:
 					// if in package, write to the dir containing the configs
 					dir = d
 				}
 
-				// create the parent dir and write the file
-				err = os.MkdirAll(filepath.Join(dir, filepath.Dir(f.path)), 0700)
-				if !assert.NoError(t, err) {
-					t.FailNow()
-				}
-				err := ioutil.WriteFile(filepath.Join(dir, f.path), []byte(f.value), 0600)
-				if !assert.NoError(t, err) {
-					t.FailNow()
+				if !f.explicitFunction {
+					// create the parent dir and write the file
+					err = os.MkdirAll(filepath.Join(dir, filepath.Dir(f.path)), 0700)
+					if !assert.NoError(t, err) {
+						t.FailNow()
+					}
+					err := ioutil.WriteFile(filepath.Join(dir, f.path), []byte(f.value), 0600)
+					if !assert.NoError(t, err) {
+						t.FailNow()
+					}
 				}
 			}
 
 			// init the instance
 			r := &RunFns{
 				FunctionPaths:        fnPaths,
+				Functions:            parsedFns,
 				Path:                 d,
 				NoFunctionsFromInput: tt.noFunctionsFromInput,
 			}
@@ -313,7 +479,7 @@ metadata:
 
 			// get the filters which would be run
 			var results []string
-			fltrs, err := r.getFilters()
+			_, fltrs, _, err := r.getNodesAndFilters()
 			if !assert.NoError(t, err) {
 				t.FailNow()
 			}
@@ -351,6 +517,7 @@ func TestCmd_Execute(t *testing.T) {
 	assert.Contains(t, string(b), "kind: StatefulSet")
 }
 
+// TestCmd_Execute_setOutput tests the execution of a filter reading and writing to a dir
 func TestCmd_Execute_setFunctionPaths(t *testing.T) {
 	dir := setupTest(t)
 	defer os.RemoveAll(dir)
@@ -371,6 +538,9 @@ func TestCmd_Execute_setFunctionPaths(t *testing.T) {
 		Path:                    dir,
 		containerFilterProvider: getFilterProvider(t),
 	}
+	// initialize the defaults
+	instance.init()
+
 	err = instance.Execute()
 	if !assert.NoError(t, err) {
 		return
@@ -383,6 +553,7 @@ func TestCmd_Execute_setFunctionPaths(t *testing.T) {
 	assert.Contains(t, string(b), "kind: StatefulSet")
 }
 
+// TestCmd_Execute_setOutput tests the execution of a filter using an io.Writer as output
 func TestCmd_Execute_setOutput(t *testing.T) {
 	dir := setupTest(t)
 	defer os.RemoveAll(dir)
@@ -399,6 +570,8 @@ func TestCmd_Execute_setOutput(t *testing.T) {
 		Path:                    dir,
 		containerFilterProvider: getFilterProvider(t),
 	}
+	// initialize the defaults
+	instance.init()
 
 	if !assert.NoError(t, instance.Execute()) {
 		return
@@ -412,6 +585,53 @@ func TestCmd_Execute_setOutput(t *testing.T) {
 	assert.Contains(t, out.String(), "kind: StatefulSet")
 }
 
+// TestCmd_Execute_setInput tests the execution of a filter using an io.Reader as input
+func TestCmd_Execute_setInput(t *testing.T) {
+	dir := setupTest(t)
+	defer os.RemoveAll(dir)
+	if !assert.NoError(t, ioutil.WriteFile(
+		filepath.Join(dir, "filter.yaml"), []byte(ValueReplacerYAMLData), 0600)) {
+		return
+	}
+
+	read, err := kio.LocalPackageReader{PackagePath: dir}.Read()
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
+	input := &bytes.Buffer{}
+	if !assert.NoError(t, kio.ByteWriter{Writer: input}.Write(read)) {
+		t.FailNow()
+	}
+
+	outDir, err := ioutil.TempDir("", "kustomize-test")
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
+
+	if !assert.NoError(t, ioutil.WriteFile(
+		filepath.Join(dir, "filter.yaml"), []byte(ValueReplacerYAMLData), 0600)) {
+		return
+	}
+
+	instance := RunFns{
+		Input:                   input, // read from input
+		Path:                    outDir,
+		containerFilterProvider: getFilterProvider(t),
+	}
+	// initialize the defaults
+	instance.init()
+
+	if !assert.NoError(t, instance.Execute()) {
+		return
+	}
+	b, err := ioutil.ReadFile(
+		filepath.Join(outDir, "java", "java-deployment.resource.yaml"))
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
+	assert.Contains(t, string(b), "kind: StatefulSet")
+}
+
 // setupTest initializes a temp test directory containing test data
 func setupTest(t *testing.T) string {
 	dir, err := ioutil.TempDir("", "kustomize-kyaml-test")