Support standalone mode for framework functions

kyaml/errors/errors.go

@@ -30,3 +30,11 @@ func WrapPrefixf(err interface{}, msg string, args ...interface{}) error {
 func Errorf(msg string, args ...interface{}) error {
 	return goerrors.Wrap(fmt.Errorf(msg, args...), 1)
 }
+
+// GetStack returns a stack trace for the error if it has one
+func GetStack(err error) string {
+	if e, ok := err.(*goerrors.Error); ok {
+		return string(e.Stack())
+	}
+	return ""
+}

kyaml/fn/framework/doc.go

@@ -4,7 +4,72 @@
 // Package framework contains a framework for writing functions in go.  The function spec
 // is defined at: https://github.com/kubernetes-sigs/kustomize/blob/master/cmd/config/docs/api-conventions/functions-spec.md
 //
-// Examples
+// Functions are executables which generate, modify, delete or validate Kubernetes resources.
+// They are often used used to implement abstractions ("kind: JavaSpringBoot") and
+// cross-cutting logic ("kind: SidecarInjector").
+//
+// Functions may be run as standalone executables or invoked as part of an orchestrated
+// pipeline (e.g. kustomize).
+//
+// Example standalone usage
+//
+// Function template input:
+//
+//    # config.yaml -- this is the input to the template
+//    apiVersion: example.com/v1alpha1
+//    kind: Example
+//    Key: a
+//    Value: b
+//
+// Additional function inputs:
+//
+//    # patch.yaml -- this will be applied as a patch
+//    apiVersion: apps/v1
+//    kind: Deployment
+//    metadata:
+//      name: foo
+//      namespace: default
+//      annotations:
+//        patch-key: patch-value
+//
+// Manually run the function:
+//
+//    # build the function
+//    $ go build example-fn/
+//
+//    # run the function using the
+//    $ ./example-fn config.yaml patch.yaml
+//
+// Go implementation
+//
+//   // example-fn/main.go
+//   func main() {
+//
+//     // Define the template used to generate resources
+//     tc := framework.TemplateCommand{
+//       Merge: true, // apply inputs as patches to the template output
+//       API: &struct {
+//         Key   string `json:"key" yaml:"key"`
+//         Value string `json:"value" yaml:"value"`
+//       }{},
+//       Template: template.Must(template.New("example").Parse(`
+//   apiVersion: apps/v1
+//   kind: Deployment
+//   metadata:
+//     name: foo
+//     namespace: default
+//     annotations:
+//       {{ .Key }}: {{ .Value }}
+//   `))}
+//
+//     // Run the command
+//     if err := tc.GetCommand().Execute(); err != nil {
+//       fmt.Fprintf(cmd.ErrOrStderr(), "%v\n", err)
+//       os.Exit(1)
+//     }
+//  }
+//
+// More Examples
 //
 // Example function implementation using framework.Command with flag input
 //
@@ -113,7 +178,7 @@
 //
 // Building a container image for the function
 //
-// The go program must be built into a container to be run as a function.  The framework
+// The go program may be built into a container and run as a function.  The framework
 // can be used to generate a Dockerfile to build the function container.
 //
 //   # create the ./Dockerfile for the container

kyaml/fn/framework/example2/Dockerfile

@@ -0,0 +1,12 @@
+# Copyright 2019 The Kubernetes Authors.
+# SPDX-License-Identifier: Apache-2.0
+
+FROM golang:1.13-stretch
+ENV CGO_ENABLED=0
+WORKDIR /go/src/
+COPY . .
+RUN go build -v -o /usr/local/bin/function ./
+
+FROM alpine:latest
+COPY --from=0 /usr/local/bin/function /usr/local/bin/function
+CMD ["function"]

kyaml/fn/framework/example2/config.yaml

@@ -0,0 +1,7 @@
+# Copyright 2019 The Kubernetes Authors.
+# SPDX-License-Identifier: Apache-2.0
+
+apiVersion: example.com/v1alpha1
+kind: Example
+key: key
+value: value

kyaml/fn/framework/example2/main.go

@@ -0,0 +1,35 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package main
+
+import (
+	"fmt"
+	"os"
+	"text/template"
+
+	"sigs.k8s.io/kustomize/kyaml/fn/framework"
+)
+
+func main() {
+	type api struct {
+		Key   string `json:"key" yaml:"key"`
+		Value string `json:"value" yaml:"value"`
+	}
+	cmd := framework.TemplateCommand{
+		API: &api{},
+		Template: template.Must(template.New("example").Parse(`
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: foo
+  namespace: default
+  annotations:
+    {{ .Key }}: {{ .Value }}
+`)),
+	}.GetCommand()
+	if err := cmd.Execute(); err != nil {
+		fmt.Fprintln(cmd.OutOrStderr(), err)
+		os.Exit(1)
+	}
+}

kyaml/fn/framework/example_test.go

@@ -6,6 +6,8 @@ package framework_test
 import (
 	"bytes"
 	"fmt"
+	"path/filepath"
+	"text/template"
 
 	"github.com/spf13/pflag"
 	"sigs.k8s.io/kustomize/kyaml/fn/framework"
@@ -537,3 +539,67 @@ items:
 	//       path: spec.field
 	//       suggestedValue: "1"
 }
+
+// ExampleTemplateCommand provides an example for using the TemplateCommand
+func ExampleTemplateCommand() {
+	// create the template
+	cmd := framework.TemplateCommand{
+		// Template input
+		API: &struct {
+			Key   string `json:"key" yaml:"key"`
+			Value string `json:"value" yaml:"value"`
+		}{},
+		// Template
+		Template: template.Must(template.New("example").Parse(`
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: foo
+  namespace: default
+  annotations:
+    {{ .Key }}: {{ .Value }}
+`)),
+	}.GetCommand()
+
+	cmd.SetArgs([]string{filepath.Join("testdata", "template", "config.yaml")})
+	if err := cmd.Execute(); err != nil {
+		fmt.Fprintf(cmd.ErrOrStderr(), "%v\n", err)
+	}
+
+	// Output:
+	// apiVersion: apps/v1
+	// kind: Deployment
+	// metadata:
+	//   name: foo
+	//   namespace: default
+	//   annotations:
+	//     a: b
+}
+
+// ExampleTemplateCommand_files provides an example for using the TemplateCommand
+func ExampleTemplateCommand_files() {
+	// create the template
+	cmd := framework.TemplateCommand{
+		// Template input
+		API: &struct {
+			Key   string `json:"key" yaml:"key"`
+			Value string `json:"value" yaml:"value"`
+		}{},
+		// Template
+		TemplatesFiles: []string{filepath.Join("testdata", "templatefiles", "deployment.template")},
+	}.GetCommand()
+
+	cmd.SetArgs([]string{filepath.Join("testdata", "templatefiles", "config.yaml")})
+	if err := cmd.Execute(); err != nil {
+		fmt.Fprintf(cmd.ErrOrStderr(), "%v\n", err)
+	}
+
+	// Output:
+	// apiVersion: apps/v1
+	// kind: Deployment
+	// metadata:
+	//   name: foo
+	//   namespace: default
+	//   annotations:
+	//     a: b
+}

kyaml/fn/framework/framework.go

@@ -4,16 +4,20 @@
 package framework
 
 import (
+	"bytes"
 	"fmt"
 	"io"
 	"io/ioutil"
 	"os"
 	"path/filepath"
+	"strings"
+	"text/template"
 
 	"github.com/spf13/cobra"
 	"github.com/spf13/pflag"
 	"sigs.k8s.io/kustomize/kyaml/errors"
 	"sigs.k8s.io/kustomize/kyaml/kio"
+	"sigs.k8s.io/kustomize/kyaml/kio/filters"
 	"sigs.k8s.io/kustomize/kyaml/yaml"
 )
 
@@ -57,6 +61,12 @@ type ResourceList struct {
 	// ResourceList.Write()
 	Result *Result
 
+	// DisableStandalone if set will not support standalone mode
+	DisableStandalone bool
+
+	// Args are the command args used for standalone mode
+	Args []string
+
 	// Flags are an optional set of flags to parse the ResourceList.functionConfig.data.
 	// If non-nil, ResourceList.Read() will set the flag value for each flag name matching
 	// a ResourceList.functionConfig.data map entry.
@@ -82,10 +92,33 @@ type ResourceList struct {
 
 	// rw reads function input and writes function output
 	rw *kio.ByteReadWriter
+
+	// NoPrintError if set will prevent the error from being printed
+	NoPrintError bool
 }
 
 // Read reads the ResourceList
 func (r *ResourceList) Read() error {
+	// parse the inputs from the args
+	if len(r.Args) > 0 && !r.DisableStandalone {
+		// write the files as input
+		var buf bytes.Buffer
+		for i := range r.Args {
+			// the first argument is the resourceList.FunctionConfig and will be parsed
+			// separately later, the rest of the arguments are the resourceList.items
+			if i == 0 {
+				continue
+			}
+			b, err := ioutil.ReadFile(r.Args[i])
+			if err != nil {
+				return errors.WrapPrefixf(err, "unable to read input file %s", r.Args[i])
+			}
+			buf.WriteString(string(b))
+			buf.WriteString("\n---\n")
+		}
+		r.Reader = &buf
+	}
+
 	if r.Reader == nil {
 		r.Reader = os.Stdin
 	}
@@ -98,6 +131,26 @@ func (r *ResourceList) Read() error {
 		KeepReaderAnnotations: true,
 	}
 
+	// parse the resourceList.FunctionConfig from the first arg
+	if len(r.Args) > 0 && !r.DisableStandalone {
+		// Don't keep the reader annotations if we are in standalone mode
+		r.rw.KeepReaderAnnotations = false
+		// Don't wrap the resources in a resourceList -- we are in
+		// standalone mode and writing to stdout to be applied
+		r.rw.NoWrap = true
+
+		b, err := ioutil.ReadFile(r.Args[0])
+		if err != nil {
+			return errors.WrapPrefixf(err, "unable to read configuration file %s", r.Args[0])
+		}
+		fc, err := yaml.Parse(string(b))
+		if err != nil {
+			return errors.WrapPrefixf(err, "unable to parse configuration file %s", r.Args[0])
+		}
+		// use this as the function config used to configure the function
+		r.rw.FunctionConfig = fc
+	}
+
 	var err error
 	r.Items, err = r.rw.Read()
 	if err != nil {
@@ -173,18 +226,142 @@ func (r *ResourceList) Write() error {
 func Command(resourceList *ResourceList, function Function) cobra.Command {
 	cmd := cobra.Command{}
 	AddGenerateDockerfile(&cmd)
+	var printStack bool
 	cmd.RunE = func(cmd *cobra.Command, args []string) error {
-		err := execute(resourceList, function, cmd)
+		err := execute(resourceList, function, cmd, args)
-		if err != nil {
+		if err != nil && !resourceList.NoPrintError {
 			fmt.Fprintf(cmd.ErrOrStderr(), "%v", err)
 		}
+		// print the stack if requested
+		if s := errors.GetStack(err); printStack && s != "" {
+			fmt.Fprintln(cmd.ErrOrStderr(), s)
+		}
 		return err
 	}
+	cmd.Flags().BoolVar(&printStack, "stack", false, "print the stack trace on failure")
+	cmd.Args = cobra.MinimumNArgs(0)
 	cmd.SilenceErrors = true
 	cmd.SilenceUsage = true
 	return cmd
 }
 
+// TemplateCommand provides a cobra command to invoke a template
+type TemplateCommand struct {
+	// API is the function API provide to the template as input
+	API interface{}
+
+	// Template is a go template to render and is appended to Templates.
+	Template *template.Template
+
+	// Templates is a list of templates to render.
+	Templates []*template.Template
+
+	// TemplateFiles list of templates to read from disk which are appended
+	// to Templates.
+	TemplatesFiles []string
+
+	// MergeResources if set to true will apply input resources
+	// as patches to the templates
+	MergeResources bool
+
+	// PreProcess is run on the ResourceList before the template is invoked
+	PreProcess func(*ResourceList) error
+
+	// PostProcess is run on the ResourceList after the template is invoked
+	PostProcess func(*ResourceList) error
+}
+
+func (tc TemplateCommand) doTemplate(t *template.Template, rl *ResourceList) error {
+	// invoke the template
+	var b bytes.Buffer
+	err := t.Execute(&b, tc.API)
+	if err != nil {
+		return errors.WrapPrefixf(err, "failed to render template %v", t.DefinedTemplates())
+	}
+	// split the resources so the error messaging is better
+	for _, s := range strings.Split(b.String(), "\n---\n") {
+		s = strings.TrimSpace(s)
+		if s == "" {
+			continue
+		}
+		nodes, err := (&kio.ByteReader{Reader: bytes.NewBufferString(s)}).Read()
+		if err != nil {
+			// create the debug string
+			lines := strings.Split(s, "\n")
+			for j := range lines {
+				lines[j] = fmt.Sprintf("%03d %s", j+1, lines[j])
+			}
+			s = strings.Join(lines, "\n")
+			return errors.WrapPrefixf(err, "failed to parse rendered template into a resource:\n%s\n", s)
+		}
+
+		if tc.MergeResources {
+			// apply inputs as patches -- add the
+			rl.Items = append(nodes, rl.Items...)
+		} else {
+			// add to the inputs list
+			rl.Items = append(rl.Items, nodes...)
+		}
+	}
+	return nil
+}
+
+// GetCommand returns a new cobra command
+func (tc TemplateCommand) GetCommand() cobra.Command {
+	rl := ResourceList{
+		FunctionConfig: tc.API,
+		NoPrintError:   true,
+	}
+	c := Command(&rl, func() error {
+		// do any preprocessing
+		if tc.PreProcess != nil {
+			if err := tc.PreProcess(&rl); err != nil {
+				return err
+			}
+		}
+
+		if tc.Template != nil {
+			tc.Templates = append(tc.Templates, tc.Template)
+		}
+
+		for i := range tc.TemplatesFiles {
+			tbytes, err := ioutil.ReadFile(tc.TemplatesFiles[i])
+			if err != nil {
+				return errors.WrapPrefixf(err, "unable to read template file")
+			}
+			t, err := template.New("files").Parse(string(tbytes))
+			if err != nil {
+				return errors.WrapPrefixf(err, "unable to parse template files %v", tc.TemplatesFiles)
+			}
+			tc.Templates = append(tc.Templates, t)
+		}
+
+		for i := range tc.Templates {
+			if err := tc.doTemplate(tc.Templates[i], &rl); err != nil {
+				return err
+			}
+		}
+
+		var err error
+		if tc.MergeResources {
+			rl.Items, err = filters.MergeFilter{}.Filter(rl.Items)
+			if err != nil {
+				return err
+			}
+		}
+
+		// finish up
+		if tc.PostProcess != nil {
+			if err := tc.PostProcess(&rl); err != nil {
+				return err
+			}
+		}
+
+		return nil
+	})
+	return c
+}
+
 // AddGenerateDockerfile adds a "gen" subcommand to create a Dockerfile for building
 // the function as a container.
 func AddGenerateDockerfile(cmd *cobra.Command) {
@@ -207,10 +384,11 @@ CMD ["function"]
 	cmd.AddCommand(gen)
 }
 
-func execute(rl *ResourceList, function Function, cmd *cobra.Command) error {
+func execute(rl *ResourceList, function Function, cmd *cobra.Command, args []string) error {
 	rl.Reader = cmd.InOrStdin()
 	rl.Writer = cmd.OutOrStdout()
 	rl.Flags = cmd.Flags()
+	rl.Args = args
 
 	if err := rl.Read(); err != nil {
 		return err

kyaml/fn/framework/framework_test.go

@@ -4,6 +4,7 @@
 package framework_test
 
 import (
+	"bytes"
 	"io/ioutil"
 	"os"
 	"path/filepath"
@@ -11,6 +12,8 @@ import (
 
 	"github.com/stretchr/testify/assert"
 	"sigs.k8s.io/kustomize/kyaml/fn/framework"
+	"sigs.k8s.io/kustomize/kyaml/testutil"
+	"sigs.k8s.io/kustomize/kyaml/yaml"
 )
 
 func TestCommand_dockerfile(t *testing.T) {
@@ -50,3 +53,53 @@ CMD ["function"]
 		t.FailNow()
 	}
 }
+
+// TestCommand_standalone tests the framework works in standalone mode
+func TestCommand_standalone(t *testing.T) {
+	// TODO: make this test pass on windows -- currently failure seems spurious
+	testutil.SkipWindows(t)
+
+	type api = struct {
+		A string `json:"a" yaml:"a"`
+	}
+	var config api
+
+	resourceList := &framework.ResourceList{FunctionConfig: &config}
+	cmd := framework.Command(resourceList, func() error {
+		resourceList.Items = append(resourceList.Items, yaml.MustParse(`
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: bar1
+  namespace: default
+  annotations:
+    foo: bar1
+`))
+
+		for i := range resourceList.Items {
+			err := resourceList.Items[i].PipeE(yaml.SetAnnotation("a", config.A))
+			if err != nil {
+				return err
+			}
+		}
+
+		return nil
+	})
+	cmd.SetArgs([]string{
+		filepath.Join("testdata", "command", "config.yaml"),
+		filepath.Join("testdata", "command", "input.yaml"),
+	})
+	var out bytes.Buffer
+	cmd.SetOutput(&out)
+	if !assert.NoError(t, cmd.Execute()) {
+		t.FailNow()
+	}
+
+	expected, err := ioutil.ReadFile(filepath.Join("testdata", "command", "expected.yaml"))
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
+	if !assert.Equal(t, string(expected), out.String()) {
+		t.FailNow()
+	}
+}

kyaml/fn/framework/testdata/command/config.yaml

@@ -0,0 +1,6 @@
+# Copyright 2019 The Kubernetes Authors.
+# SPDX-License-Identifier: Apache-2.0
+
+apiVersion: example.com/v1alpha1
+kind: Foo
+a: b

kyaml/fn/framework/testdata/command/expected.yaml

@@ -0,0 +1,20 @@
+# Copyright 2019 The Kubernetes Authors.
+# SPDX-License-Identifier: Apache-2.0
+
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: bar2
+  namespace: default
+  annotations:
+    foo: bar2
+    a: 'b'
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: bar1
+  namespace: default
+  annotations:
+    foo: bar1
+    a: 'b'

kyaml/fn/framework/testdata/command/input.yaml

@@ -0,0 +1,10 @@
+# Copyright 2019 The Kubernetes Authors.
+# SPDX-License-Identifier: Apache-2.0
+
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: bar2
+  namespace: default
+  annotations:
+    foo: bar2

kyaml/fn/framework/testdata/template/config.yaml

@@ -0,0 +1,7 @@
+# Copyright 2019 The Kubernetes Authors.
+# SPDX-License-Identifier: Apache-2.0
+
+apiVersion: example.com/v1alpha1
+kind: Example
+key: a
+value: b

kyaml/fn/framework/testdata/templatefiles/config.yaml

@@ -0,0 +1,7 @@
+# Copyright 2019 The Kubernetes Authors.
+# SPDX-License-Identifier: Apache-2.0
+
+apiVersion: example.com/v1alpha1
+kind: Example
+key: a
+value: b

kyaml/fn/framework/testdata/templatefiles/deployment.template

@@ -0,0 +1,7 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: foo
+  namespace: default
+  annotations:
+    {{ .Key }}: {{ .Value }}