Merge pull request #3221 from pwittrock/framework

Function framework standalone feature
This commit is contained in:
Kubernetes Prow Robot
2020-11-12 13:52:25 -08:00
committed by GitHub
18 changed files with 679 additions and 9 deletions

View File

@@ -30,3 +30,11 @@ func WrapPrefixf(err interface{}, msg string, args ...interface{}) error {
func Errorf(msg string, args ...interface{}) error {
return goerrors.Wrap(fmt.Errorf(msg, args...), 1)
}
// GetStack returns a stack trace for the error if it has one
func GetStack(err error) string {
if e, ok := err.(*goerrors.Error); ok {
return string(e.Stack())
}
return ""
}

View File

@@ -4,7 +4,72 @@
// Package framework contains a framework for writing functions in go. The function spec
// is defined at: https://github.com/kubernetes-sigs/kustomize/blob/master/cmd/config/docs/api-conventions/functions-spec.md
//
// Examples
// Functions are executables which generate, modify, delete or validate Kubernetes resources.
// They are often used used to implement abstractions ("kind: JavaSpringBoot") and
// cross-cutting logic ("kind: SidecarInjector").
//
// Functions may be run as standalone executables or invoked as part of an orchestrated
// pipeline (e.g. kustomize).
//
// Example standalone usage
//
// Function template input:
//
// # config.yaml -- this is the input to the template
// apiVersion: example.com/v1alpha1
// kind: Example
// Key: a
// Value: b
//
// Additional function inputs:
//
// # patch.yaml -- this will be applied as a patch
// apiVersion: apps/v1
// kind: Deployment
// metadata:
// name: foo
// namespace: default
// annotations:
// patch-key: patch-value
//
// Manually run the function:
//
// # build the function
// $ go build example-fn/
//
// # run the function using the
// $ ./example-fn config.yaml patch.yaml
//
// Go implementation
//
// // example-fn/main.go
// func main() {
//
// // Define the template used to generate resources
// tc := framework.TemplateCommand{
// Merge: true, // apply inputs as patches to the template output
// API: &struct {
// Key string `json:"key" yaml:"key"`
// Value string `json:"value" yaml:"value"`
// }{},
// Template: template.Must(template.New("example").Parse(`
// apiVersion: apps/v1
// kind: Deployment
// metadata:
// name: foo
// namespace: default
// annotations:
// {{ .Key }}: {{ .Value }}
// `))}
//
// // Run the command
// if err := tc.GetCommand().Execute(); err != nil {
// fmt.Fprintf(cmd.ErrOrStderr(), "%v\n", err)
// os.Exit(1)
// }
// }
//
// More Examples
//
// Example function implementation using framework.Command with flag input
//
@@ -113,7 +178,7 @@
//
// Building a container image for the function
//
// The go program must be built into a container to be run as a function. The framework
// The go program may be built into a container and run as a function. The framework
// can be used to generate a Dockerfile to build the function container.
//
// # create the ./Dockerfile for the container

View File

@@ -0,0 +1,12 @@
# Copyright 2019 The Kubernetes Authors.
# SPDX-License-Identifier: Apache-2.0
FROM golang:1.13-stretch
ENV CGO_ENABLED=0
WORKDIR /go/src/
COPY . .
RUN go build -v -o /usr/local/bin/function ./
FROM alpine:latest
COPY --from=0 /usr/local/bin/function /usr/local/bin/function
CMD ["function"]

View File

@@ -0,0 +1,7 @@
# Copyright 2019 The Kubernetes Authors.
# SPDX-License-Identifier: Apache-2.0
apiVersion: example.com/v1alpha1
kind: Example
key: key
value: value

View File

@@ -0,0 +1,35 @@
// Copyright 2019 The Kubernetes Authors.
// SPDX-License-Identifier: Apache-2.0
package main
import (
"fmt"
"os"
"text/template"
"sigs.k8s.io/kustomize/kyaml/fn/framework"
)
func main() {
type api struct {
Key string `json:"key" yaml:"key"`
Value string `json:"value" yaml:"value"`
}
cmd := framework.TemplateCommand{
API: &api{},
Template: template.Must(template.New("example").Parse(`
apiVersion: apps/v1
kind: Deployment
metadata:
name: foo
namespace: default
annotations:
{{ .Key }}: {{ .Value }}
`)),
}.GetCommand()
if err := cmd.Execute(); err != nil {
fmt.Fprintln(cmd.OutOrStderr(), err)
os.Exit(1)
}
}

View File

@@ -6,6 +6,8 @@ package framework_test
import (
"bytes"
"fmt"
"path/filepath"
"text/template"
"github.com/spf13/pflag"
"sigs.k8s.io/kustomize/kyaml/fn/framework"
@@ -537,3 +539,67 @@ items:
// path: spec.field
// suggestedValue: "1"
}
// ExampleTemplateCommand provides an example for using the TemplateCommand
func ExampleTemplateCommand() {
// create the template
cmd := framework.TemplateCommand{
// Template input
API: &struct {
Key string `json:"key" yaml:"key"`
Value string `json:"value" yaml:"value"`
}{},
// Template
Template: template.Must(template.New("example").Parse(`
apiVersion: apps/v1
kind: Deployment
metadata:
name: foo
namespace: default
annotations:
{{ .Key }}: {{ .Value }}
`)),
}.GetCommand()
cmd.SetArgs([]string{filepath.Join("testdata", "template", "config.yaml")})
if err := cmd.Execute(); err != nil {
fmt.Fprintf(cmd.ErrOrStderr(), "%v\n", err)
}
// Output:
// apiVersion: apps/v1
// kind: Deployment
// metadata:
// name: foo
// namespace: default
// annotations:
// a: b
}
// ExampleTemplateCommand_files provides an example for using the TemplateCommand
func ExampleTemplateCommand_files() {
// create the template
cmd := framework.TemplateCommand{
// Template input
API: &struct {
Key string `json:"key" yaml:"key"`
Value string `json:"value" yaml:"value"`
}{},
// Template
TemplatesFiles: []string{filepath.Join("testdata", "templatefiles", "deployment.template")},
}.GetCommand()
cmd.SetArgs([]string{filepath.Join("testdata", "templatefiles", "config.yaml")})
if err := cmd.Execute(); err != nil {
fmt.Fprintf(cmd.ErrOrStderr(), "%v\n", err)
}
// Output:
// apiVersion: apps/v1
// kind: Deployment
// metadata:
// name: foo
// namespace: default
// annotations:
// a: b
}

View File

@@ -4,16 +4,20 @@
package framework
import (
"bytes"
"fmt"
"io"
"io/ioutil"
"os"
"path/filepath"
"strings"
"text/template"
"github.com/spf13/cobra"
"github.com/spf13/pflag"
"sigs.k8s.io/kustomize/kyaml/errors"
"sigs.k8s.io/kustomize/kyaml/kio"
"sigs.k8s.io/kustomize/kyaml/kio/filters"
"sigs.k8s.io/kustomize/kyaml/yaml"
)
@@ -57,6 +61,12 @@ type ResourceList struct {
// ResourceList.Write()
Result *Result
// DisableStandalone if set will not support standalone mode
DisableStandalone bool
// Args are the command args used for standalone mode
Args []string
// Flags are an optional set of flags to parse the ResourceList.functionConfig.data.
// If non-nil, ResourceList.Read() will set the flag value for each flag name matching
// a ResourceList.functionConfig.data map entry.
@@ -82,10 +92,33 @@ type ResourceList struct {
// rw reads function input and writes function output
rw *kio.ByteReadWriter
// NoPrintError if set will prevent the error from being printed
NoPrintError bool
}
// Read reads the ResourceList
func (r *ResourceList) Read() error {
// parse the inputs from the args
if len(r.Args) > 0 && !r.DisableStandalone {
// write the files as input
var buf bytes.Buffer
for i := range r.Args {
// the first argument is the resourceList.FunctionConfig and will be parsed
// separately later, the rest of the arguments are the resourceList.items
if i == 0 {
continue
}
b, err := ioutil.ReadFile(r.Args[i])
if err != nil {
return errors.WrapPrefixf(err, "unable to read input file %s", r.Args[i])
}
buf.WriteString(string(b))
buf.WriteString("\n---\n")
}
r.Reader = &buf
}
if r.Reader == nil {
r.Reader = os.Stdin
}
@@ -98,6 +131,26 @@ func (r *ResourceList) Read() error {
KeepReaderAnnotations: true,
}
// parse the resourceList.FunctionConfig from the first arg
if len(r.Args) > 0 && !r.DisableStandalone {
// Don't keep the reader annotations if we are in standalone mode
r.rw.KeepReaderAnnotations = false
// Don't wrap the resources in a resourceList -- we are in
// standalone mode and writing to stdout to be applied
r.rw.NoWrap = true
b, err := ioutil.ReadFile(r.Args[0])
if err != nil {
return errors.WrapPrefixf(err, "unable to read configuration file %s", r.Args[0])
}
fc, err := yaml.Parse(string(b))
if err != nil {
return errors.WrapPrefixf(err, "unable to parse configuration file %s", r.Args[0])
}
// use this as the function config used to configure the function
r.rw.FunctionConfig = fc
}
var err error
r.Items, err = r.rw.Read()
if err != nil {
@@ -173,18 +226,142 @@ func (r *ResourceList) Write() error {
func Command(resourceList *ResourceList, function Function) cobra.Command {
cmd := cobra.Command{}
AddGenerateDockerfile(&cmd)
var printStack bool
cmd.RunE = func(cmd *cobra.Command, args []string) error {
err := execute(resourceList, function, cmd)
if err != nil {
err := execute(resourceList, function, cmd, args)
if err != nil && !resourceList.NoPrintError {
fmt.Fprintf(cmd.ErrOrStderr(), "%v", err)
}
// print the stack if requested
if s := errors.GetStack(err); printStack && s != "" {
fmt.Fprintln(cmd.ErrOrStderr(), s)
}
return err
}
cmd.Flags().BoolVar(&printStack, "stack", false, "print the stack trace on failure")
cmd.Args = cobra.MinimumNArgs(0)
cmd.SilenceErrors = true
cmd.SilenceUsage = true
return cmd
}
// TemplateCommand provides a cobra command to invoke a template
type TemplateCommand struct {
// API is the function API provide to the template as input
API interface{}
// Template is a go template to render and is appended to Templates.
Template *template.Template
// Templates is a list of templates to render.
Templates []*template.Template
// TemplateFiles list of templates to read from disk which are appended
// to Templates.
TemplatesFiles []string
// MergeResources if set to true will apply input resources
// as patches to the templates
MergeResources bool
// PreProcess is run on the ResourceList before the template is invoked
PreProcess func(*ResourceList) error
// PostProcess is run on the ResourceList after the template is invoked
PostProcess func(*ResourceList) error
}
func (tc TemplateCommand) doTemplate(t *template.Template, rl *ResourceList) error {
// invoke the template
var b bytes.Buffer
err := t.Execute(&b, tc.API)
if err != nil {
return errors.WrapPrefixf(err, "failed to render template %v", t.DefinedTemplates())
}
// split the resources so the error messaging is better
for _, s := range strings.Split(b.String(), "\n---\n") {
s = strings.TrimSpace(s)
if s == "" {
continue
}
nodes, err := (&kio.ByteReader{Reader: bytes.NewBufferString(s)}).Read()
if err != nil {
// create the debug string
lines := strings.Split(s, "\n")
for j := range lines {
lines[j] = fmt.Sprintf("%03d %s", j+1, lines[j])
}
s = strings.Join(lines, "\n")
return errors.WrapPrefixf(err, "failed to parse rendered template into a resource:\n%s\n", s)
}
if tc.MergeResources {
// apply inputs as patches -- add the
rl.Items = append(nodes, rl.Items...)
} else {
// add to the inputs list
rl.Items = append(rl.Items, nodes...)
}
}
return nil
}
// GetCommand returns a new cobra command
func (tc TemplateCommand) GetCommand() cobra.Command {
rl := ResourceList{
FunctionConfig: tc.API,
NoPrintError: true,
}
c := Command(&rl, func() error {
// do any preprocessing
if tc.PreProcess != nil {
if err := tc.PreProcess(&rl); err != nil {
return err
}
}
if tc.Template != nil {
tc.Templates = append(tc.Templates, tc.Template)
}
for i := range tc.TemplatesFiles {
tbytes, err := ioutil.ReadFile(tc.TemplatesFiles[i])
if err != nil {
return errors.WrapPrefixf(err, "unable to read template file")
}
t, err := template.New("files").Parse(string(tbytes))
if err != nil {
return errors.WrapPrefixf(err, "unable to parse template files %v", tc.TemplatesFiles)
}
tc.Templates = append(tc.Templates, t)
}
for i := range tc.Templates {
if err := tc.doTemplate(tc.Templates[i], &rl); err != nil {
return err
}
}
var err error
if tc.MergeResources {
rl.Items, err = filters.MergeFilter{}.Filter(rl.Items)
if err != nil {
return err
}
}
// finish up
if tc.PostProcess != nil {
if err := tc.PostProcess(&rl); err != nil {
return err
}
}
return nil
})
return c
}
// AddGenerateDockerfile adds a "gen" subcommand to create a Dockerfile for building
// the function as a container.
func AddGenerateDockerfile(cmd *cobra.Command) {
@@ -207,10 +384,11 @@ CMD ["function"]
cmd.AddCommand(gen)
}
func execute(rl *ResourceList, function Function, cmd *cobra.Command) error {
func execute(rl *ResourceList, function Function, cmd *cobra.Command, args []string) error {
rl.Reader = cmd.InOrStdin()
rl.Writer = cmd.OutOrStdout()
rl.Flags = cmd.Flags()
rl.Args = args
if err := rl.Read(); err != nil {
return err

View File

@@ -4,6 +4,7 @@
package framework_test
import (
"bytes"
"io/ioutil"
"os"
"path/filepath"
@@ -11,6 +12,8 @@ import (
"github.com/stretchr/testify/assert"
"sigs.k8s.io/kustomize/kyaml/fn/framework"
"sigs.k8s.io/kustomize/kyaml/testutil"
"sigs.k8s.io/kustomize/kyaml/yaml"
)
func TestCommand_dockerfile(t *testing.T) {
@@ -50,3 +53,53 @@ CMD ["function"]
t.FailNow()
}
}
// TestCommand_standalone tests the framework works in standalone mode
func TestCommand_standalone(t *testing.T) {
// TODO: make this test pass on windows -- currently failure seems spurious
testutil.SkipWindows(t)
type api = struct {
A string `json:"a" yaml:"a"`
}
var config api
resourceList := &framework.ResourceList{FunctionConfig: &config}
cmd := framework.Command(resourceList, func() error {
resourceList.Items = append(resourceList.Items, yaml.MustParse(`
apiVersion: apps/v1
kind: Deployment
metadata:
name: bar1
namespace: default
annotations:
foo: bar1
`))
for i := range resourceList.Items {
err := resourceList.Items[i].PipeE(yaml.SetAnnotation("a", config.A))
if err != nil {
return err
}
}
return nil
})
cmd.SetArgs([]string{
filepath.Join("testdata", "command", "config.yaml"),
filepath.Join("testdata", "command", "input.yaml"),
})
var out bytes.Buffer
cmd.SetOutput(&out)
if !assert.NoError(t, cmd.Execute()) {
t.FailNow()
}
expected, err := ioutil.ReadFile(filepath.Join("testdata", "command", "expected.yaml"))
if !assert.NoError(t, err) {
t.FailNow()
}
if !assert.Equal(t, string(expected), out.String()) {
t.FailNow()
}
}

View File

@@ -0,0 +1,6 @@
# Copyright 2019 The Kubernetes Authors.
# SPDX-License-Identifier: Apache-2.0
apiVersion: example.com/v1alpha1
kind: Foo
a: b

View File

@@ -0,0 +1,20 @@
# Copyright 2019 The Kubernetes Authors.
# SPDX-License-Identifier: Apache-2.0
apiVersion: apps/v1
kind: Deployment
metadata:
name: bar2
namespace: default
annotations:
foo: bar2
a: 'b'
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: bar1
namespace: default
annotations:
foo: bar1
a: 'b'

View File

@@ -0,0 +1,10 @@
# Copyright 2019 The Kubernetes Authors.
# SPDX-License-Identifier: Apache-2.0
apiVersion: apps/v1
kind: Deployment
metadata:
name: bar2
namespace: default
annotations:
foo: bar2

View File

@@ -0,0 +1,7 @@
# Copyright 2019 The Kubernetes Authors.
# SPDX-License-Identifier: Apache-2.0
apiVersion: example.com/v1alpha1
kind: Example
key: a
value: b

View File

@@ -0,0 +1,7 @@
# Copyright 2019 The Kubernetes Authors.
# SPDX-License-Identifier: Apache-2.0
apiVersion: example.com/v1alpha1
kind: Example
key: a
value: b

View File

@@ -0,0 +1,7 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: foo
namespace: default
annotations:
{{ .Key }}: {{ .Value }}

View File

@@ -43,6 +43,7 @@ type ByteReadWriter struct {
Results *yaml.RNode
NoWrap bool
WrappingAPIVersion string
WrappingKind string
}
@@ -53,10 +54,15 @@ func (rw *ByteReadWriter) Read() ([]*yaml.RNode, error) {
OmitReaderAnnotations: rw.OmitReaderAnnotations,
}
val, err := b.Read()
rw.FunctionConfig = b.FunctionConfig
if rw.FunctionConfig == nil {
rw.FunctionConfig = b.FunctionConfig
}
rw.Results = b.Results
rw.WrappingAPIVersion = b.WrappingAPIVersion
rw.WrappingKind = b.WrappingKind
if !rw.NoWrap {
rw.WrappingAPIVersion = b.WrappingAPIVersion
rw.WrappingKind = b.WrappingKind
}
return val, errors.Wrap(err)
}

View File

@@ -10,6 +10,7 @@ import (
"github.com/stretchr/testify/assert"
"sigs.k8s.io/kustomize/kyaml/kio"
"sigs.k8s.io/kustomize/kyaml/yaml"
)
func TestByteReadWriter(t *testing.T) {
@@ -226,6 +227,68 @@ metadata:
`,
instance: kio.ByteReadWriter{KeepReaderAnnotations: true},
},
{
name: "manual_override_wrap",
input: `
apiVersion: config.kubernetes.io/v1alpha1
kind: ResourceList
items:
- kind: Deployment
spec:
replicas: 1
- kind: Service
spec:
selectors:
foo: bar
functionConfig:
a: b # something
`,
expectedOutput: `
kind: Deployment
spec:
replicas: 1
---
kind: Service
spec:
selectors:
foo: bar
`,
instance: kio.ByteReadWriter{NoWrap: true},
},
{
name: "manual_override_function_config",
input: `
apiVersion: config.kubernetes.io/v1alpha1
kind: ResourceList
items:
- kind: Deployment
spec:
replicas: 1
- kind: Service
spec:
selectors:
foo: bar
functionConfig:
a: b # something
`,
expectedOutput: `
apiVersion: config.kubernetes.io/v1alpha1
kind: ResourceList
items:
- kind: Deployment
spec:
replicas: 1
- kind: Service
spec:
selectors:
foo: bar
functionConfig:
c: d
`,
instance: kio.ByteReadWriter{FunctionConfig: yaml.MustParse(`c: d`)},
},
}
for i := range testCases {

View File

@@ -41,6 +41,8 @@ func (c MergeFilter) Filter(input []*yaml.RNode) ([]*yaml.RNode, error) {
// index the Resources by G/V/K/NS/N
index := map[mergeKey][]*yaml.RNode{}
// retain the original ordering
var order []mergeKey
for i := range input {
meta, err := input[i].GetMeta()
if err != nil {
@@ -52,13 +54,16 @@ func (c MergeFilter) Filter(input []*yaml.RNode) ([]*yaml.RNode, error) {
namespace: meta.Namespace,
name: meta.Name,
}
if _, found := index[key]; !found {
order = append(order, key)
}
index[key] = append(index[key], input[i])
}
// merge each of the G/V/K/NS/N lists
var output []*yaml.RNode
var err error
for k := range index {
for _, k := range order {
var merged *yaml.RNode
resources := index[k]
for i := range resources {

View File

@@ -0,0 +1,115 @@
// Copyright 2019 The Kubernetes Authors.
// SPDX-License-Identifier: Apache-2.0
package filters_test
import (
"bytes"
"strings"
"testing"
"github.com/stretchr/testify/assert"
"sigs.k8s.io/kustomize/kyaml/kio"
"sigs.k8s.io/kustomize/kyaml/kio/filters"
"sigs.k8s.io/kustomize/kyaml/yaml"
)
// TestMerge_Merge_order tests that the original order of elements
// retained after merge
func TestMerge_Merge_order(t *testing.T) {
r1, err := yaml.Parse(`
apiVersion: apps/v1
kind: Deployment
metadata:
name: foo-1
namespace: bar-1
spec:
template:
spec: {}
`)
if !assert.NoError(t, err) {
t.FailNow()
}
r2, err := yaml.Parse(`
apiVersion: apps/v1
kind: Deployment
metadata:
name: foo-2
namespace: bar-2
spec:
template:
spec: {}
`)
if !assert.NoError(t, err) {
t.FailNow()
}
var b bytes.Buffer
err = kio.Pipeline{
Inputs: []kio.Reader{&kio.PackageBuffer{Nodes: []*yaml.RNode{r1, r2}}},
Filters: []kio.Filter{filters.MatchFilter{}},
Outputs: []kio.Writer{&kio.ByteWriter{Writer: &b}},
}.Execute()
if !assert.NoError(t, err) {
t.FailNow()
}
expected := strings.TrimSpace(`
apiVersion: apps/v1
kind: Deployment
metadata:
name: foo-1
namespace: bar-1
spec:
template:
spec: {}
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: foo-2
namespace: bar-2
spec:
template:
spec: {}
`)
if !assert.Equal(t, expected, strings.TrimSpace(b.String())) {
t.FailNow()
}
b.Reset()
err = kio.Pipeline{
Inputs: []kio.Reader{&kio.PackageBuffer{Nodes: []*yaml.RNode{r2, r1}}},
Filters: []kio.Filter{filters.MatchFilter{}},
Outputs: []kio.Writer{&kio.ByteWriter{Writer: &b}},
}.Execute()
if !assert.NoError(t, err) {
t.FailNow()
}
expected = strings.TrimSpace(`
apiVersion: apps/v1
kind: Deployment
metadata:
name: foo-2
namespace: bar-2
spec:
template:
spec: {}
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: foo-1
namespace: bar-1
spec:
template:
spec: {}
`)
if !assert.Equal(t, expected, strings.TrimSpace(b.String())) {
t.FailNow()
}
}