From 474dfc916bbbe223287b03a58d46aae3132d7ac4 Mon Sep 17 00:00:00 2001 From: Phillip Wittrock Date: Tue, 14 Jan 2020 18:16:59 -0800 Subject: [PATCH] kyaml/rnfn: support explicit fn list and reading from an io.Reader - Support specifying an io.Reader as Input. Use this instead of Path for reading Resources. - Default io.Writer to os.Stdout if no Path is specified - Default io.Reader to os.Stdin if no Path is specified - Support specifying an explicit list of Functions. If specified, use these in place of reading from the Input or Directory source by default. --- kyaml/runfn/runfn.go | 111 ++++++++++++++--- kyaml/runfn/runfn_test.go | 256 +++++++++++++++++++++++++++++++++++--- 2 files changed, 329 insertions(+), 38 deletions(-) diff --git a/kyaml/runfn/runfn.go b/kyaml/runfn/runfn.go index bc1c1c29c..3a19f1da5 100644 --- a/kyaml/runfn/runfn.go +++ b/kyaml/runfn/runfn.go @@ -5,6 +5,7 @@ package runfn import ( "io" + "os" "path" "path/filepath" "sort" @@ -28,12 +29,21 @@ type RunFns struct { // FunctionPaths Paths allows functions to be specified outside the configuration // directory. // Functions provided on FunctionPaths are globally scoped. + // If FunctionPaths length is > 0, then NoFunctionsFromInput defaults to true FunctionPaths []string + // Functions is an explicit list of functions to run against the input. + // Functions provided on Functions are globally scoped. + // If Functions length is > 0, then NoFunctionsFromInput defaults to true + Functions []*yaml.RNode + // GlobalScope if true, functions read from input will be scoped globally rather // than only to Resources under their subdirs. GlobalScope bool + // Input can be set to read the Resources from Input rather than from a directory + Input io.Reader + // Output can be set to write the result to Output rather than back to the directory Output io.Writer @@ -56,51 +66,83 @@ func (r RunFns) Execute() error { // default the containerFilterProvider if it hasn't been override. Split out for testing. (&r).init() - - fltrs, err := r.getFilters() + nodes, fltrs, output, err := r.getNodesAndFilters() if err != nil { return err } - - return r.runFunctions(fltrs) + return r.runFunctions(nodes, output, fltrs) } -func (r RunFns) getFilters() ([]kio.Filter, error) { +func (r RunFns) getNodesAndFilters() ( + *kio.PackageBuffer, []kio.Filter, *kio.LocalPackageReadWriter, error) { + // Read Resources from Directory or Input + buff := &kio.PackageBuffer{} + p := kio.Pipeline{Outputs: []kio.Writer{buff}} + // save the output dir because we will need it to write back + // the same one for reading must be used for writing if deleting Resources + var outputPkg *kio.LocalPackageReadWriter + if r.Path != "" { + outputPkg = &kio.LocalPackageReadWriter{PackagePath: r.Path} + } + + if r.Input == nil { + p.Inputs = []kio.Reader{outputPkg} + } else { + p.Inputs = []kio.Reader{&kio.ByteReader{Reader: r.Input}} + } + if err := p.Execute(); err != nil { + return nil, nil, outputPkg, err + } + + fltrs, err := r.getFilters(buff.Nodes) + if err != nil { + return nil, nil, outputPkg, err + } + return buff, fltrs, outputPkg, nil +} + +func (r RunFns) getFilters(nodes []*yaml.RNode) ([]kio.Filter, error) { var fltrs []kio.Filter // implicit filters from the input Resources - f, err := r.getFunctionsFromInput() + f, err := r.getFunctionsFromInput(nodes) if err != nil { return nil, err } fltrs = append(fltrs, f...) // explicit filters from a list of directories - f, err = r.getFunctionsFromDirList() + f, err = r.getFunctionsFromFunctionPaths() if err != nil { return nil, err } fltrs = append(fltrs, f...) + + // explicit filters from a list of directories + f = r.getFunctionsFromFunctions() + fltrs = append(fltrs, f...) + return fltrs, nil } -// runFunctions runs the fltrs against the input -func (r RunFns) runFunctions(fltrs []kio.Filter) error { - pkgIO := &kio.LocalPackageReadWriter{PackagePath: r.Path} - inputs := []kio.Reader{pkgIO} +// runFunctions runs the fltrs against the input and writes to either r.Output or output +func (r RunFns) runFunctions( + input kio.Reader, output kio.Writer, fltrs []kio.Filter) error { + // use the previously read Resources as input var outputs []kio.Writer if r.Output == nil { // write back to the package - outputs = append(outputs, pkgIO) + outputs = append(outputs, output) } else { - // write to the output instead of the directory + // write to the output instead of the directory if r.Output is specified or + // the output is nil (reading from Input) outputs = append(outputs, kio.ByteWriter{Writer: r.Output}) } - return kio.Pipeline{Inputs: inputs, Filters: fltrs, Outputs: outputs}.Execute() + return kio.Pipeline{Inputs: []kio.Reader{input}, Filters: fltrs, Outputs: outputs}.Execute() } // getFunctionsFromInput scans the input for functions and runs them -func (r RunFns) getFunctionsFromInput() ([]kio.Filter, error) { +func (r RunFns) getFunctionsFromInput(nodes []*yaml.RNode) ([]kio.Filter, error) { if *r.NoFunctionsFromInput { return nil, nil } @@ -108,7 +150,7 @@ func (r RunFns) getFunctionsFromInput() ([]kio.Filter, error) { var fltrs []kio.Filter buff := &kio.PackageBuffer{} err := kio.Pipeline{ - Inputs: []kio.Reader{kio.LocalPackageReader{PackagePath: r.Path}}, + Inputs: []kio.Reader{&kio.PackageBuffer{Nodes: nodes}}, Filters: []kio.Filter{&filters.IsReconcilerFilter{}}, Outputs: []kio.Writer{buff}, }.Execute() @@ -124,9 +166,9 @@ func (r RunFns) getFunctionsFromInput() ([]kio.Filter, error) { return fltrs, nil } -// getFunctionsFromDirList returns the set of functions read from r.FunctionPaths +// getFunctionsFromFunctionPaths returns the set of functions read from r.FunctionPaths // as a slice of Filters -func (r RunFns) getFunctionsFromDirList() ([]kio.Filter, error) { +func (r RunFns) getFunctionsFromFunctionPaths() ([]kio.Filter, error) { var fltrs []kio.Filter buff := &kio.PackageBuffer{} for i := range r.FunctionPaths { @@ -144,7 +186,7 @@ func (r RunFns) getFunctionsFromDirList() ([]kio.Filter, error) { c := r.containerFilterProvider(img, path, api) cf, ok := c.(*filters.ContainerFilter) if ok { - // functions provided on FunctionPaths are globally scoped + // functions provided by FunctionPaths are globally scoped cf.GlobalScope = true } fltrs = append(fltrs, c) @@ -152,6 +194,24 @@ func (r RunFns) getFunctionsFromDirList() ([]kio.Filter, error) { return fltrs, nil } +// getFunctionsFromFunctions returns the set of explicitly provided functions as +// Filters +func (r RunFns) getFunctionsFromFunctions() []kio.Filter { + var fltrs []kio.Filter + for i := range r.Functions { + api := r.Functions[i] + img, path := filters.GetContainerName(api) + c := r.containerFilterProvider(img, path, api) + cf, ok := c.(*filters.ContainerFilter) + if ok { + // functions provided by Functions are globally scoped + cf.GlobalScope = true + } + fltrs = append(fltrs, c) + } + return fltrs +} + // sortFns sorts functions so that functions with the longest paths come first func sortFns(buff *kio.PackageBuffer) { // sort the nodes so that we traverse them depth first @@ -201,10 +261,21 @@ func sortFns(buff *kio.PackageBuffer) { // init initializes the RunFns with a containerFilterProvider. func (r *RunFns) init() { if r.NoFunctionsFromInput == nil { - nfn := len(r.FunctionPaths) > 0 + // default no functions from input if any function sources are explicitly provided + nfn := len(r.FunctionPaths) > 0 || len(r.Functions) > 0 r.NoFunctionsFromInput = &nfn } + // if no path is specified, default reading from stdin and writing to stdout + if r.Path == "" { + if r.Output == nil { + r.Output = os.Stdout + } + if r.Input == nil { + r.Input = os.Stdin + } + } + // if containerFilterProvider hasn't been set, use the default if r.containerFilterProvider == nil { r.containerFilterProvider = func(image, path string, api *yaml.RNode) kio.Filter { diff --git a/kyaml/runfn/runfn_test.go b/kyaml/runfn/runfn_test.go index 1d3b0d490..49e805a73 100644 --- a/kyaml/runfn/runfn_test.go +++ b/kyaml/runfn/runfn_test.go @@ -34,9 +34,16 @@ replace: StatefulSet ` ) -func TestRunFns_Execute(t *testing.T) { +func TestRunFns_init(t *testing.T) { instance := RunFns{} instance.init() + if !assert.Equal(t, instance.Input, os.Stdin) { + t.FailNow() + } + if !assert.Equal(t, instance.Output, os.Stdout) { + t.FailNow() + } + api, err := yaml.Parse(`apiVersion: apps/v1 kind: `) @@ -47,9 +54,15 @@ kind: assert.Equal(t, &filters.ContainerFilter{Image: "example.com:version", Config: api}, filter) } -func TestRunFns_Execute_globalScope(t *testing.T) { +func TestRunFns_Execute__initGlobalScope(t *testing.T) { instance := RunFns{GlobalScope: true} instance.init() + if !assert.Equal(t, instance.Input, os.Stdin) { + t.FailNow() + } + if !assert.Equal(t, instance.Output, os.Stdout) { + t.FailNow() + } api, err := yaml.Parse(`apiVersion: apps/v1 kind: `) @@ -61,8 +74,80 @@ kind: Image: "example.com:version", Config: api, GlobalScope: true}, filter) } -var tru = true -var fls = false +func TestRunFns_Execute__initDefault(t *testing.T) { + b := &bytes.Buffer{} + var tests = []struct { + instance RunFns + expected RunFns + name string + }{ + { + instance: RunFns{}, + name: "empty", + expected: RunFns{Output: os.Stdout, Input: os.Stdin, NoFunctionsFromInput: getFalse()}, + }, + { + name: "explicit output", + instance: RunFns{Output: b}, + expected: RunFns{Output: b, Input: os.Stdin, NoFunctionsFromInput: getFalse()}, + }, + { + name: "explicit input", + instance: RunFns{Input: b}, + expected: RunFns{Output: os.Stdout, Input: b, NoFunctionsFromInput: getFalse()}, + }, + { + name: "explicit functions -- no functions from input", + instance: RunFns{Functions: []*yaml.RNode{{}}}, + expected: RunFns{Output: os.Stdout, Input: os.Stdin, NoFunctionsFromInput: getTrue(), Functions: []*yaml.RNode{{}}}, + }, + { + name: "explicit functions -- yes functions from input", + instance: RunFns{Functions: []*yaml.RNode{{}}, NoFunctionsFromInput: getFalse()}, + expected: RunFns{Output: os.Stdout, Input: os.Stdin, NoFunctionsFromInput: getFalse(), Functions: []*yaml.RNode{{}}}, + }, + { + name: "explicit functions in paths -- no functions from input", + instance: RunFns{FunctionPaths: []string{"foo"}}, + expected: RunFns{ + Output: os.Stdout, + Input: os.Stdin, + NoFunctionsFromInput: getTrue(), + FunctionPaths: []string{"foo"}, + }, + }, + { + name: "functions in paths -- yes functions from input", + instance: RunFns{FunctionPaths: []string{"foo"}, NoFunctionsFromInput: getFalse()}, + expected: RunFns{ + Output: os.Stdout, + Input: os.Stdin, + NoFunctionsFromInput: getFalse(), + FunctionPaths: []string{"foo"}, + }, + }, + } + for i := range tests { + tt := tests[i] + t.Run(tt.name, func(t *testing.T) { + (&tt.instance).init() + (&tt.instance).containerFilterProvider = nil + if !assert.Equal(t, tt.expected, tt.instance) { + t.FailNow() + } + }) + } +} + +func getTrue() *bool { + t := true + return &t +} + +func getFalse() *bool { + f := false + return &f +} // TestRunFns_getFilters tests how filters are found and sorted func TestRunFns_getFilters(t *testing.T) { @@ -72,6 +157,10 @@ func TestRunFns_getFilters(t *testing.T) { // if true, create the function in a separate directory from // the config, and provide it through FunctionPaths outOfPackage bool + + // if true, create the function as an explicit Functions input + explicitFunction bool + // if true and outOfPackage is true, create a new directory // for this function separate from the previous one. If // false and outOfPackage is true, create the function in @@ -173,7 +262,7 @@ metadata: // // {name: "sort functions -- skip implicit", - noFunctionsFromInput: &tru, + noFunctionsFromInput: getTrue(), in: []f{ { path: filepath.Join("foo", "a.yaml"), @@ -203,7 +292,7 @@ metadata: // // {name: "sort functions -- include implicit", - noFunctionsFromInput: &fls, + noFunctionsFromInput: getFalse(), in: []f{ { path: filepath.Join("foo", "a.yaml"), @@ -233,7 +322,7 @@ metadata: // // {name: "sort functions -- implicit first", - noFunctionsFromInput: &fls, + noFunctionsFromInput: getFalse(), in: []f{ { path: filepath.Join("foo", "a.yaml"), @@ -259,6 +348,76 @@ metadata: }, out: []string{"b", "a"}, }, + + // Test + // + // + {name: "explicit functions", + in: []f{ + { + explicitFunction: true, + value: ` +metadata: + annotations: + config.kubernetes.io/function: | + container: + image: c +`, + }, + { + path: filepath.Join("b.yaml"), + value: ` +metadata: + annotations: + config.kubernetes.io/function: | + container: + image: b +`, + }, + }, + out: []string{"c"}, + }, + + // Test + // + // + {name: "sort functions -- implicit first", + noFunctionsFromInput: getFalse(), + in: []f{ + { + explicitFunction: true, + value: ` +metadata: + annotations: + config.kubernetes.io/function: | + container: + image: c +`, + }, + { + path: filepath.Join("foo", "a.yaml"), + outOfPackage: true, // out of package is run last + value: ` +metadata: + annotations: + config.kubernetes.io/function: | + container: + image: a +`, + }, + { + path: filepath.Join("b.yaml"), + value: ` +metadata: + annotations: + config.kubernetes.io/function: | + container: + image: b +`, + }, + }, + out: []string{"b", "a", "c"}, + }, } for i := range tests { @@ -270,12 +429,14 @@ metadata: // write the functions to files var fnPaths []string + var parsedFns []*yaml.RNode var fnPath string var err error for _, f := range tt.in { // get the location for the file var dir string - if f.outOfPackage { + switch { + case f.outOfPackage: // if out of package, write to a separate temp directory if f.newFnPath || fnPath == "" { // create a new fn directory @@ -287,25 +448,30 @@ metadata: fnPaths = append(fnPaths, fnPath) } dir = fnPath - } else { + case f.explicitFunction: + parsedFns = append(parsedFns, yaml.MustParse(f.value)) + default: // if in package, write to the dir containing the configs dir = d } - // create the parent dir and write the file - err = os.MkdirAll(filepath.Join(dir, filepath.Dir(f.path)), 0700) - if !assert.NoError(t, err) { - t.FailNow() - } - err := ioutil.WriteFile(filepath.Join(dir, f.path), []byte(f.value), 0600) - if !assert.NoError(t, err) { - t.FailNow() + if !f.explicitFunction { + // create the parent dir and write the file + err = os.MkdirAll(filepath.Join(dir, filepath.Dir(f.path)), 0700) + if !assert.NoError(t, err) { + t.FailNow() + } + err := ioutil.WriteFile(filepath.Join(dir, f.path), []byte(f.value), 0600) + if !assert.NoError(t, err) { + t.FailNow() + } } } // init the instance r := &RunFns{ FunctionPaths: fnPaths, + Functions: parsedFns, Path: d, NoFunctionsFromInput: tt.noFunctionsFromInput, } @@ -313,7 +479,7 @@ metadata: // get the filters which would be run var results []string - fltrs, err := r.getFilters() + _, fltrs, _, err := r.getNodesAndFilters() if !assert.NoError(t, err) { t.FailNow() } @@ -351,6 +517,7 @@ func TestCmd_Execute(t *testing.T) { assert.Contains(t, string(b), "kind: StatefulSet") } +// TestCmd_Execute_setOutput tests the execution of a filter reading and writing to a dir func TestCmd_Execute_setFunctionPaths(t *testing.T) { dir := setupTest(t) defer os.RemoveAll(dir) @@ -371,6 +538,9 @@ func TestCmd_Execute_setFunctionPaths(t *testing.T) { Path: dir, containerFilterProvider: getFilterProvider(t), } + // initialize the defaults + instance.init() + err = instance.Execute() if !assert.NoError(t, err) { return @@ -383,6 +553,7 @@ func TestCmd_Execute_setFunctionPaths(t *testing.T) { assert.Contains(t, string(b), "kind: StatefulSet") } +// TestCmd_Execute_setOutput tests the execution of a filter using an io.Writer as output func TestCmd_Execute_setOutput(t *testing.T) { dir := setupTest(t) defer os.RemoveAll(dir) @@ -399,6 +570,8 @@ func TestCmd_Execute_setOutput(t *testing.T) { Path: dir, containerFilterProvider: getFilterProvider(t), } + // initialize the defaults + instance.init() if !assert.NoError(t, instance.Execute()) { return @@ -412,6 +585,53 @@ func TestCmd_Execute_setOutput(t *testing.T) { assert.Contains(t, out.String(), "kind: StatefulSet") } +// TestCmd_Execute_setInput tests the execution of a filter using an io.Reader as input +func TestCmd_Execute_setInput(t *testing.T) { + dir := setupTest(t) + defer os.RemoveAll(dir) + if !assert.NoError(t, ioutil.WriteFile( + filepath.Join(dir, "filter.yaml"), []byte(ValueReplacerYAMLData), 0600)) { + return + } + + read, err := kio.LocalPackageReader{PackagePath: dir}.Read() + if !assert.NoError(t, err) { + t.FailNow() + } + input := &bytes.Buffer{} + if !assert.NoError(t, kio.ByteWriter{Writer: input}.Write(read)) { + t.FailNow() + } + + outDir, err := ioutil.TempDir("", "kustomize-test") + if !assert.NoError(t, err) { + t.FailNow() + } + + if !assert.NoError(t, ioutil.WriteFile( + filepath.Join(dir, "filter.yaml"), []byte(ValueReplacerYAMLData), 0600)) { + return + } + + instance := RunFns{ + Input: input, // read from input + Path: outDir, + containerFilterProvider: getFilterProvider(t), + } + // initialize the defaults + instance.init() + + if !assert.NoError(t, instance.Execute()) { + return + } + b, err := ioutil.ReadFile( + filepath.Join(outDir, "java", "java-deployment.resource.yaml")) + if !assert.NoError(t, err) { + t.FailNow() + } + assert.Contains(t, string(b), "kind: StatefulSet") +} + // setupTest initializes a temp test directory containing test data func setupTest(t *testing.T) string { dir, err := ioutil.TempDir("", "kustomize-kyaml-test")