From 8b9d8a266db2aad2fc4b168ecbdcee313b0232a1 Mon Sep 17 00:00:00 2001 From: Phillip Wittrock Date: Thu, 12 Nov 2020 08:53:14 -0800 Subject: [PATCH] Support standalone mode for framework functions --- kyaml/errors/errors.go | 8 + kyaml/fn/framework/doc.go | 69 ++++++- kyaml/fn/framework/example2/Dockerfile | 12 ++ kyaml/fn/framework/example2/config.yaml | 7 + kyaml/fn/framework/example2/main.go | 35 ++++ kyaml/fn/framework/example_test.go | 66 +++++++ kyaml/fn/framework/framework.go | 184 +++++++++++++++++- kyaml/fn/framework/framework_test.go | 53 +++++ .../fn/framework/testdata/command/config.yaml | 6 + .../framework/testdata/command/expected.yaml | 20 ++ .../fn/framework/testdata/command/input.yaml | 10 + .../framework/testdata/template/config.yaml | 7 + .../testdata/templatefiles/config.yaml | 7 + .../templatefiles/deployment.template | 7 + 14 files changed, 486 insertions(+), 5 deletions(-) create mode 100644 kyaml/fn/framework/example2/Dockerfile create mode 100644 kyaml/fn/framework/example2/config.yaml create mode 100644 kyaml/fn/framework/example2/main.go create mode 100644 kyaml/fn/framework/testdata/command/config.yaml create mode 100644 kyaml/fn/framework/testdata/command/expected.yaml create mode 100644 kyaml/fn/framework/testdata/command/input.yaml create mode 100644 kyaml/fn/framework/testdata/template/config.yaml create mode 100644 kyaml/fn/framework/testdata/templatefiles/config.yaml create mode 100644 kyaml/fn/framework/testdata/templatefiles/deployment.template diff --git a/kyaml/errors/errors.go b/kyaml/errors/errors.go index 0129b752b..f072c3c97 100644 --- a/kyaml/errors/errors.go +++ b/kyaml/errors/errors.go @@ -30,3 +30,11 @@ func WrapPrefixf(err interface{}, msg string, args ...interface{}) error { func Errorf(msg string, args ...interface{}) error { return goerrors.Wrap(fmt.Errorf(msg, args...), 1) } + +// GetStack returns a stack trace for the error if it has one +func GetStack(err error) string { + if e, ok := err.(*goerrors.Error); ok { + return string(e.Stack()) + } + return "" +} diff --git a/kyaml/fn/framework/doc.go b/kyaml/fn/framework/doc.go index 2ef5336d9..8a1b62d3d 100644 --- a/kyaml/fn/framework/doc.go +++ b/kyaml/fn/framework/doc.go @@ -4,7 +4,72 @@ // Package framework contains a framework for writing functions in go. The function spec // is defined at: https://github.com/kubernetes-sigs/kustomize/blob/master/cmd/config/docs/api-conventions/functions-spec.md // -// Examples +// Functions are executables which generate, modify, delete or validate Kubernetes resources. +// They are often used used to implement abstractions ("kind: JavaSpringBoot") and +// cross-cutting logic ("kind: SidecarInjector"). +// +// Functions may be run as standalone executables or invoked as part of an orchestrated +// pipeline (e.g. kustomize). +// +// Example standalone usage +// +// Function template input: +// +// # config.yaml -- this is the input to the template +// apiVersion: example.com/v1alpha1 +// kind: Example +// Key: a +// Value: b +// +// Additional function inputs: +// +// # patch.yaml -- this will be applied as a patch +// apiVersion: apps/v1 +// kind: Deployment +// metadata: +// name: foo +// namespace: default +// annotations: +// patch-key: patch-value +// +// Manually run the function: +// +// # build the function +// $ go build example-fn/ +// +// # run the function using the +// $ ./example-fn config.yaml patch.yaml +// +// Go implementation +// +// // example-fn/main.go +// func main() { +// +// // Define the template used to generate resources +// tc := framework.TemplateCommand{ +// Merge: true, // apply inputs as patches to the template output +// API: &struct { +// Key string `json:"key" yaml:"key"` +// Value string `json:"value" yaml:"value"` +// }{}, +// Template: template.Must(template.New("example").Parse(` +// apiVersion: apps/v1 +// kind: Deployment +// metadata: +// name: foo +// namespace: default +// annotations: +// {{ .Key }}: {{ .Value }} +// `))} +// +// // Run the command +// if err := tc.GetCommand().Execute(); err != nil { +// fmt.Fprintf(cmd.ErrOrStderr(), "%v\n", err) +// os.Exit(1) +// } +// } +// +// More Examples // // Example function implementation using framework.Command with flag input // @@ -113,7 +178,7 @@ // // Building a container image for the function // -// The go program must be built into a container to be run as a function. The framework +// The go program may be built into a container and run as a function. The framework // can be used to generate a Dockerfile to build the function container. // // # create the ./Dockerfile for the container diff --git a/kyaml/fn/framework/example2/Dockerfile b/kyaml/fn/framework/example2/Dockerfile new file mode 100644 index 000000000..d177d7df1 --- /dev/null +++ b/kyaml/fn/framework/example2/Dockerfile @@ -0,0 +1,12 @@ +# Copyright 2019 The Kubernetes Authors. +# SPDX-License-Identifier: Apache-2.0 + +FROM golang:1.13-stretch +ENV CGO_ENABLED=0 +WORKDIR /go/src/ +COPY . . +RUN go build -v -o /usr/local/bin/function ./ + +FROM alpine:latest +COPY --from=0 /usr/local/bin/function /usr/local/bin/function +CMD ["function"] diff --git a/kyaml/fn/framework/example2/config.yaml b/kyaml/fn/framework/example2/config.yaml new file mode 100644 index 000000000..9be110fa0 --- /dev/null +++ b/kyaml/fn/framework/example2/config.yaml @@ -0,0 +1,7 @@ +# Copyright 2019 The Kubernetes Authors. +# SPDX-License-Identifier: Apache-2.0 + +apiVersion: example.com/v1alpha1 +kind: Example +key: key +value: value diff --git a/kyaml/fn/framework/example2/main.go b/kyaml/fn/framework/example2/main.go new file mode 100644 index 000000000..8098f660d --- /dev/null +++ b/kyaml/fn/framework/example2/main.go @@ -0,0 +1,35 @@ +// Copyright 2019 The Kubernetes Authors. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "fmt" + "os" + "text/template" + + "sigs.k8s.io/kustomize/kyaml/fn/framework" +) + +func main() { + type api struct { + Key string `json:"key" yaml:"key"` + Value string `json:"value" yaml:"value"` + } + cmd := framework.TemplateCommand{ + API: &api{}, + Template: template.Must(template.New("example").Parse(` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: foo + namespace: default + annotations: + {{ .Key }}: {{ .Value }} +`)), + }.GetCommand() + if err := cmd.Execute(); err != nil { + fmt.Fprintln(cmd.OutOrStderr(), err) + os.Exit(1) + } +} diff --git a/kyaml/fn/framework/example_test.go b/kyaml/fn/framework/example_test.go index 3d5b37393..477c45c16 100644 --- a/kyaml/fn/framework/example_test.go +++ b/kyaml/fn/framework/example_test.go @@ -6,6 +6,8 @@ package framework_test import ( "bytes" "fmt" + "path/filepath" + "text/template" "github.com/spf13/pflag" "sigs.k8s.io/kustomize/kyaml/fn/framework" @@ -537,3 +539,67 @@ items: // path: spec.field // suggestedValue: "1" } + +// ExampleTemplateCommand provides an example for using the TemplateCommand +func ExampleTemplateCommand() { + // create the template + cmd := framework.TemplateCommand{ + // Template input + API: &struct { + Key string `json:"key" yaml:"key"` + Value string `json:"value" yaml:"value"` + }{}, + // Template + Template: template.Must(template.New("example").Parse(` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: foo + namespace: default + annotations: + {{ .Key }}: {{ .Value }} +`)), + }.GetCommand() + + cmd.SetArgs([]string{filepath.Join("testdata", "template", "config.yaml")}) + if err := cmd.Execute(); err != nil { + fmt.Fprintf(cmd.ErrOrStderr(), "%v\n", err) + } + + // Output: + // apiVersion: apps/v1 + // kind: Deployment + // metadata: + // name: foo + // namespace: default + // annotations: + // a: b +} + +// ExampleTemplateCommand_files provides an example for using the TemplateCommand +func ExampleTemplateCommand_files() { + // create the template + cmd := framework.TemplateCommand{ + // Template input + API: &struct { + Key string `json:"key" yaml:"key"` + Value string `json:"value" yaml:"value"` + }{}, + // Template + TemplatesFiles: []string{filepath.Join("testdata", "templatefiles", "deployment.template")}, + }.GetCommand() + + cmd.SetArgs([]string{filepath.Join("testdata", "templatefiles", "config.yaml")}) + if err := cmd.Execute(); err != nil { + fmt.Fprintf(cmd.ErrOrStderr(), "%v\n", err) + } + + // Output: + // apiVersion: apps/v1 + // kind: Deployment + // metadata: + // name: foo + // namespace: default + // annotations: + // a: b +} diff --git a/kyaml/fn/framework/framework.go b/kyaml/fn/framework/framework.go index 5f1dc65b8..e45f78b02 100644 --- a/kyaml/fn/framework/framework.go +++ b/kyaml/fn/framework/framework.go @@ -4,16 +4,20 @@ package framework import ( + "bytes" "fmt" "io" "io/ioutil" "os" "path/filepath" + "strings" + "text/template" "github.com/spf13/cobra" "github.com/spf13/pflag" "sigs.k8s.io/kustomize/kyaml/errors" "sigs.k8s.io/kustomize/kyaml/kio" + "sigs.k8s.io/kustomize/kyaml/kio/filters" "sigs.k8s.io/kustomize/kyaml/yaml" ) @@ -57,6 +61,12 @@ type ResourceList struct { // ResourceList.Write() Result *Result + // DisableStandalone if set will not support standalone mode + DisableStandalone bool + + // Args are the command args used for standalone mode + Args []string + // Flags are an optional set of flags to parse the ResourceList.functionConfig.data. // If non-nil, ResourceList.Read() will set the flag value for each flag name matching // a ResourceList.functionConfig.data map entry. @@ -82,10 +92,33 @@ type ResourceList struct { // rw reads function input and writes function output rw *kio.ByteReadWriter + + // NoPrintError if set will prevent the error from being printed + NoPrintError bool } // Read reads the ResourceList func (r *ResourceList) Read() error { + // parse the inputs from the args + if len(r.Args) > 0 && !r.DisableStandalone { + // write the files as input + var buf bytes.Buffer + for i := range r.Args { + // the first argument is the resourceList.FunctionConfig and will be parsed + // separately later, the rest of the arguments are the resourceList.items + if i == 0 { + continue + } + b, err := ioutil.ReadFile(r.Args[i]) + if err != nil { + return errors.WrapPrefixf(err, "unable to read input file %s", r.Args[i]) + } + buf.WriteString(string(b)) + buf.WriteString("\n---\n") + } + r.Reader = &buf + } + if r.Reader == nil { r.Reader = os.Stdin } @@ -98,6 +131,26 @@ func (r *ResourceList) Read() error { KeepReaderAnnotations: true, } + // parse the resourceList.FunctionConfig from the first arg + if len(r.Args) > 0 && !r.DisableStandalone { + // Don't keep the reader annotations if we are in standalone mode + r.rw.KeepReaderAnnotations = false + // Don't wrap the resources in a resourceList -- we are in + // standalone mode and writing to stdout to be applied + r.rw.NoWrap = true + + b, err := ioutil.ReadFile(r.Args[0]) + if err != nil { + return errors.WrapPrefixf(err, "unable to read configuration file %s", r.Args[0]) + } + fc, err := yaml.Parse(string(b)) + if err != nil { + return errors.WrapPrefixf(err, "unable to parse configuration file %s", r.Args[0]) + } + // use this as the function config used to configure the function + r.rw.FunctionConfig = fc + } + var err error r.Items, err = r.rw.Read() if err != nil { @@ -173,18 +226,142 @@ func (r *ResourceList) Write() error { func Command(resourceList *ResourceList, function Function) cobra.Command { cmd := cobra.Command{} AddGenerateDockerfile(&cmd) + var printStack bool cmd.RunE = func(cmd *cobra.Command, args []string) error { - err := execute(resourceList, function, cmd) - if err != nil { + err := execute(resourceList, function, cmd, args) + if err != nil && !resourceList.NoPrintError { fmt.Fprintf(cmd.ErrOrStderr(), "%v", err) } + // print the stack if requested + if s := errors.GetStack(err); printStack && s != "" { + fmt.Fprintln(cmd.ErrOrStderr(), s) + } return err } + cmd.Flags().BoolVar(&printStack, "stack", false, "print the stack trace on failure") + cmd.Args = cobra.MinimumNArgs(0) cmd.SilenceErrors = true cmd.SilenceUsage = true return cmd } +// TemplateCommand provides a cobra command to invoke a template +type TemplateCommand struct { + // API is the function API provide to the template as input + API interface{} + + // Template is a go template to render and is appended to Templates. + Template *template.Template + + // Templates is a list of templates to render. + Templates []*template.Template + + // TemplateFiles list of templates to read from disk which are appended + // to Templates. + TemplatesFiles []string + + // MergeResources if set to true will apply input resources + // as patches to the templates + MergeResources bool + + // PreProcess is run on the ResourceList before the template is invoked + PreProcess func(*ResourceList) error + + // PostProcess is run on the ResourceList after the template is invoked + PostProcess func(*ResourceList) error +} + +func (tc TemplateCommand) doTemplate(t *template.Template, rl *ResourceList) error { + // invoke the template + var b bytes.Buffer + err := t.Execute(&b, tc.API) + if err != nil { + return errors.WrapPrefixf(err, "failed to render template %v", t.DefinedTemplates()) + } + // split the resources so the error messaging is better + for _, s := range strings.Split(b.String(), "\n---\n") { + s = strings.TrimSpace(s) + if s == "" { + continue + } + nodes, err := (&kio.ByteReader{Reader: bytes.NewBufferString(s)}).Read() + if err != nil { + // create the debug string + lines := strings.Split(s, "\n") + for j := range lines { + lines[j] = fmt.Sprintf("%03d %s", j+1, lines[j]) + } + s = strings.Join(lines, "\n") + return errors.WrapPrefixf(err, "failed to parse rendered template into a resource:\n%s\n", s) + } + + if tc.MergeResources { + // apply inputs as patches -- add the + rl.Items = append(nodes, rl.Items...) + } else { + // add to the inputs list + rl.Items = append(rl.Items, nodes...) + } + } + return nil +} + +// GetCommand returns a new cobra command +func (tc TemplateCommand) GetCommand() cobra.Command { + rl := ResourceList{ + FunctionConfig: tc.API, + NoPrintError: true, + } + c := Command(&rl, func() error { + // do any preprocessing + if tc.PreProcess != nil { + if err := tc.PreProcess(&rl); err != nil { + return err + } + } + + if tc.Template != nil { + tc.Templates = append(tc.Templates, tc.Template) + } + + for i := range tc.TemplatesFiles { + tbytes, err := ioutil.ReadFile(tc.TemplatesFiles[i]) + if err != nil { + return errors.WrapPrefixf(err, "unable to read template file") + } + t, err := template.New("files").Parse(string(tbytes)) + if err != nil { + return errors.WrapPrefixf(err, "unable to parse template files %v", tc.TemplatesFiles) + } + tc.Templates = append(tc.Templates, t) + } + + for i := range tc.Templates { + if err := tc.doTemplate(tc.Templates[i], &rl); err != nil { + return err + } + } + + var err error + if tc.MergeResources { + rl.Items, err = filters.MergeFilter{}.Filter(rl.Items) + if err != nil { + return err + } + } + + // finish up + if tc.PostProcess != nil { + if err := tc.PostProcess(&rl); err != nil { + return err + } + } + + return nil + }) + return c +} + // AddGenerateDockerfile adds a "gen" subcommand to create a Dockerfile for building // the function as a container. func AddGenerateDockerfile(cmd *cobra.Command) { @@ -207,10 +384,11 @@ CMD ["function"] cmd.AddCommand(gen) } -func execute(rl *ResourceList, function Function, cmd *cobra.Command) error { +func execute(rl *ResourceList, function Function, cmd *cobra.Command, args []string) error { rl.Reader = cmd.InOrStdin() rl.Writer = cmd.OutOrStdout() rl.Flags = cmd.Flags() + rl.Args = args if err := rl.Read(); err != nil { return err diff --git a/kyaml/fn/framework/framework_test.go b/kyaml/fn/framework/framework_test.go index fabb7a258..05f993891 100644 --- a/kyaml/fn/framework/framework_test.go +++ b/kyaml/fn/framework/framework_test.go @@ -4,6 +4,7 @@ package framework_test import ( + "bytes" "io/ioutil" "os" "path/filepath" @@ -11,6 +12,8 @@ import ( "github.com/stretchr/testify/assert" "sigs.k8s.io/kustomize/kyaml/fn/framework" + "sigs.k8s.io/kustomize/kyaml/testutil" + "sigs.k8s.io/kustomize/kyaml/yaml" ) func TestCommand_dockerfile(t *testing.T) { @@ -50,3 +53,53 @@ CMD ["function"] t.FailNow() } } + +// TestCommand_standalone tests the framework works in standalone mode +func TestCommand_standalone(t *testing.T) { + // TODO: make this test pass on windows -- currently failure seems spurious + testutil.SkipWindows(t) + + type api = struct { + A string `json:"a" yaml:"a"` + } + var config api + + resourceList := &framework.ResourceList{FunctionConfig: &config} + cmd := framework.Command(resourceList, func() error { + resourceList.Items = append(resourceList.Items, yaml.MustParse(` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: bar1 + namespace: default + annotations: + foo: bar1 +`)) + + for i := range resourceList.Items { + err := resourceList.Items[i].PipeE(yaml.SetAnnotation("a", config.A)) + if err != nil { + return err + } + } + + return nil + }) + cmd.SetArgs([]string{ + filepath.Join("testdata", "command", "config.yaml"), + filepath.Join("testdata", "command", "input.yaml"), + }) + var out bytes.Buffer + cmd.SetOutput(&out) + if !assert.NoError(t, cmd.Execute()) { + t.FailNow() + } + + expected, err := ioutil.ReadFile(filepath.Join("testdata", "command", "expected.yaml")) + if !assert.NoError(t, err) { + t.FailNow() + } + if !assert.Equal(t, string(expected), out.String()) { + t.FailNow() + } +} diff --git a/kyaml/fn/framework/testdata/command/config.yaml b/kyaml/fn/framework/testdata/command/config.yaml new file mode 100644 index 000000000..81f93e066 --- /dev/null +++ b/kyaml/fn/framework/testdata/command/config.yaml @@ -0,0 +1,6 @@ +# Copyright 2019 The Kubernetes Authors. +# SPDX-License-Identifier: Apache-2.0 + +apiVersion: example.com/v1alpha1 +kind: Foo +a: b \ No newline at end of file diff --git a/kyaml/fn/framework/testdata/command/expected.yaml b/kyaml/fn/framework/testdata/command/expected.yaml new file mode 100644 index 000000000..381ccf4c9 --- /dev/null +++ b/kyaml/fn/framework/testdata/command/expected.yaml @@ -0,0 +1,20 @@ +# Copyright 2019 The Kubernetes Authors. +# SPDX-License-Identifier: Apache-2.0 + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: bar2 + namespace: default + annotations: + foo: bar2 + a: 'b' +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: bar1 + namespace: default + annotations: + foo: bar1 + a: 'b' diff --git a/kyaml/fn/framework/testdata/command/input.yaml b/kyaml/fn/framework/testdata/command/input.yaml new file mode 100644 index 000000000..a604d4ebe --- /dev/null +++ b/kyaml/fn/framework/testdata/command/input.yaml @@ -0,0 +1,10 @@ +# Copyright 2019 The Kubernetes Authors. +# SPDX-License-Identifier: Apache-2.0 + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: bar2 + namespace: default + annotations: + foo: bar2 \ No newline at end of file diff --git a/kyaml/fn/framework/testdata/template/config.yaml b/kyaml/fn/framework/testdata/template/config.yaml new file mode 100644 index 000000000..51460d7bb --- /dev/null +++ b/kyaml/fn/framework/testdata/template/config.yaml @@ -0,0 +1,7 @@ +# Copyright 2019 The Kubernetes Authors. +# SPDX-License-Identifier: Apache-2.0 + +apiVersion: example.com/v1alpha1 +kind: Example +key: a +value: b \ No newline at end of file diff --git a/kyaml/fn/framework/testdata/templatefiles/config.yaml b/kyaml/fn/framework/testdata/templatefiles/config.yaml new file mode 100644 index 000000000..51460d7bb --- /dev/null +++ b/kyaml/fn/framework/testdata/templatefiles/config.yaml @@ -0,0 +1,7 @@ +# Copyright 2019 The Kubernetes Authors. +# SPDX-License-Identifier: Apache-2.0 + +apiVersion: example.com/v1alpha1 +kind: Example +key: a +value: b \ No newline at end of file diff --git a/kyaml/fn/framework/testdata/templatefiles/deployment.template b/kyaml/fn/framework/testdata/templatefiles/deployment.template new file mode 100644 index 000000000..cdf9df435 --- /dev/null +++ b/kyaml/fn/framework/testdata/templatefiles/deployment.template @@ -0,0 +1,7 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: foo + namespace: default + annotations: + {{ .Key }}: {{ .Value }}