Better support for writing functions in go

kyaml/fn/framework/doc.go

@@ -0,0 +1,61 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+// Package framework contains a framework for writing functions in go.
+//
+// Example
+//
+// Example function implementation to set an annotation on each resource.
+//
+//      cmd := framework.Command(nil, func(items []*yaml.RNode) ([]*yaml.RNode, error) {
+//        for i := range items {
+//          if err := items[i].PipeE(yaml.SetAnnotation("value", value)); err != nil {
+//            return nil, err
+//          }
+//        }
+//        return items, nil
+//      })
+//      cmd.Flags().StringVar(&value, "value", "", "annotation value")
+//      if err := cmd.Execute(); err != nil {
+//        panic(err)
+//      }
+//
+// Architecture
+//
+// Functions are implemented as a go function which accept a slice of resources (items)
+// and returns a modified slice of resources (items).
+//
+// Mutator and Generator Functions
+//
+// Functions may add, delete or modify resources for the returned slice.
+//
+// Validator Functions
+//
+// Functions may validate resources, returning Results as go errors.  Results may contain
+// different items for different validation failures.
+//
+// Configuring Functions
+//
+// Functions may be configured through a functionConfig (i.e. a client side custom resource),
+// or through flags (which the framework parses from a ConfigMap provided as input).
+// Any flags registered on the cobra.Command will be parsed from the functionConfig input
+// if they are defined as functionConfig.data entries.
+//
+// Functions may also access environment variables set by the caller.
+//
+// Function Input
+//
+// The framework parses the function ResourceList.items into a slice of yaml.RNodes, and
+// parses the ResourceList.functionConfig into a passed in struct (optional).
+//
+// Building the Container
+//
+// The go program must be built into a container to be run as a function.  The framework
+// can be used to generate a Dockerfile to build the function container.
+//
+//   # create the ./Dockerfile for the container
+//   $ go run ./main.go gen ./
+//
+//   # build the function's container
+//   $ docker build . -t gcr.io/my-project/my-image:my-version
+package framework

kyaml/fn/framework/example/Dockerfile

@@ -0,0 +1,12 @@
+# Copyright 2019 The Kubernetes Authors.
+# SPDX-License-Identifier: Apache-2.0
+
+FROM golang:1.13-stretch
+ENV CGO_ENABLED=0
+WORKDIR /go/src/
+COPY . .
+RUN go build -v -o /usr/local/bin/function ./
+
+FROM alpine:latest
+COPY --from=0 /usr/local/bin/function /usr/local/bin/function
+CMD ["function"]

kyaml/fn/framework/example/doc.go

@@ -0,0 +1,9 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+// Package main contains an example using the the framework.
+//
+// To generate the Dockerfile for the function image run:
+//
+//   $ go run ./main.go .
+package main

kyaml/fn/framework/example/main.go

@@ -0,0 +1,29 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package main
+
+import (
+	"os"
+
+	"sigs.k8s.io/kustomize/kyaml/fn/framework"
+	"sigs.k8s.io/kustomize/kyaml/yaml"
+)
+
+func main() {
+	var value string
+	cmd := framework.Command(nil, func(items []*yaml.RNode) ([]*yaml.RNode, error) {
+		for i := range items {
+			// set the annotation on each resource item
+			if err := items[i].PipeE(yaml.SetAnnotation("value", value)); err != nil {
+				return nil, err
+			}
+		}
+		return items, nil
+	})
+	cmd.Flags().StringVar(&value, "value", "", "annotation value")
+
+	if err := cmd.Execute(); err != nil {
+		os.Exit(1)
+	}
+}

kyaml/fn/framework/example_test.go

@@ -0,0 +1,368 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package framework_test
+
+import (
+	"bytes"
+	"fmt"
+
+	"sigs.k8s.io/kustomize/kyaml/fn/framework"
+	"sigs.k8s.io/kustomize/kyaml/yaml"
+)
+
+// ExampleCommand_modify implements a function that sets an annotation on each resource.
+// The annotation value is configured via a flag value parsed from
+// ResourceList.functionConfig.data
+func ExampleCommand_modify() {
+	// configure the annotation value using a flag parsed from
+	// ResourceList.functionConfig.data.value
+	var value string
+	cmd := framework.Command(nil, func(items []*yaml.RNode) ([]*yaml.RNode, error) {
+		for i := range items {
+			// set the annotation on each resource item
+			if err := items[i].PipeE(yaml.SetAnnotation("value", value)); err != nil {
+				return nil, err
+			}
+		}
+		return items, nil
+	})
+	cmd.Flags().StringVar(&value, "value", "", "annotation value")
+
+	// for testing purposes only -- normally read from stdin when Executing
+	cmd.SetIn(bytes.NewBufferString(`
+apiVersion: config.kubernetes.io/v1alpha1
+kind: ResourceList
+# items are provided as nodes
+items:
+- apiVersion: apps/v1
+  kind: Deployment
+  metadata:
+    name: foo
+- apiVersion: v1
+  kind: Service
+  metadata:
+    name: foo
+# functionConfig is parsed into flags by framework.Command
+functionConfig:
+  apiVersion: v1
+  kind: ConfigMap
+  data:
+    value: baz
+`))
+	// run the command
+	if err := cmd.Execute(); err != nil {
+		panic(err)
+	}
+
+	// Output:
+	// apiVersion: config.kubernetes.io/v1alpha1
+	// kind: ResourceList
+	// items:
+	// - apiVersion: apps/v1
+	//   kind: Deployment
+	//   metadata:
+	//     name: foo
+	//     annotations:
+	//       value: 'baz'
+	// - apiVersion: v1
+	//   kind: Service
+	//   metadata:
+	//     name: foo
+	//     annotations:
+	//       value: 'baz'
+	// functionConfig:
+	//   apiVersion: v1
+	//   kind: ConfigMap
+	//   data:
+	//     value: baz
+}
+
+// ExampleCommand_generateReplace generates a resource from a functionConfig.
+// If the resource already exist s, it replaces the resource with a new copy.
+func ExampleCommand_generateReplace() {
+	// function API definition which will be parsed from the ResourceList.functionConfig
+	// read from stdin
+	type Spec struct {
+		Name string `yaml:"name,omitempty"`
+	}
+	type ExampleServiceGenerator struct {
+		Spec Spec `yaml:"spec,omitempty"`
+	}
+	functionConfig := &ExampleServiceGenerator{}
+
+	// function implementation -- generate a Service resource
+	cmd := framework.Command(functionConfig, func(nodes []*yaml.RNode) ([]*yaml.RNode, error) {
+		var newNodes []*yaml.RNode
+		for i := range nodes {
+			meta, err := nodes[i].GetMeta()
+			if err != nil {
+				return nil, err
+			}
+
+			// something we already generated, remove it from the list so we regenerate it
+			if meta.Name == functionConfig.Spec.Name &&
+				meta.Kind == "Service" &&
+				meta.APIVersion == "v1" {
+				continue
+			}
+			newNodes = append(newNodes, nodes[i])
+		}
+
+		// generate the resource
+		n, err := yaml.Parse(fmt.Sprintf(`apiVersion: v1
+kind: Service
+metadata:
+  name: %s
+`, functionConfig.Spec.Name))
+		if err != nil {
+			return nil, err
+		}
+		return append(newNodes, n), nil
+	})
+
+	// for testing purposes only -- normally read from stdin when Executing
+	cmd.SetIn(bytes.NewBufferString(`
+apiVersion: config.kubernetes.io/v1alpha1
+kind: ResourceList
+# items are provided as nodes
+items:
+- apiVersion: apps/v1
+  kind: Deployment
+  metadata:
+    name: foo
+# functionConfig is parsed into flags by framework.Command
+functionConfig:
+  apiVersion: example.com/v1alpha1
+  kind: ExampleServiceGenerator
+  spec:
+    name: bar
+`))
+
+	// run the command
+	if err := cmd.Execute(); err != nil {
+		panic(err)
+	}
+
+	// Output:
+	// apiVersion: config.kubernetes.io/v1alpha1
+	// kind: ResourceList
+	// items:
+	// - apiVersion: apps/v1
+	//   kind: Deployment
+	//   metadata:
+	//     name: foo
+	// - apiVersion: v1
+	//   kind: Service
+	//   metadata:
+	//     name: bar
+	// functionConfig:
+	//   apiVersion: example.com/v1alpha1
+	//   kind: ExampleServiceGenerator
+	//   spec:
+	//     name: bar
+}
+
+// ExampleCommand_generateUpdate generates a resource, updating the previously generated
+// copy rather than replacing it.
+//
+// Note: This will keep manual edits to the previously generated copy.
+func ExampleCommand_generateUpdate() {
+	// function API definition which will be parsed from the ResourceList.functionConfig
+	// read from stdin
+	type Spec struct {
+		Name        string            `yaml:"name,omitempty"`
+		Annotations map[string]string `yaml:"annotations,omitempty"`
+	}
+	type ExampleServiceGenerator struct {
+		Spec Spec `yaml:"spec,omitempty"`
+	}
+	functionConfig := &ExampleServiceGenerator{}
+
+	// function implementation -- generate or update a Service resource
+	cmd := framework.Command(functionConfig, func(nodes []*yaml.RNode) ([]*yaml.RNode, error) {
+		var found bool
+		for i := range nodes {
+			meta, err := nodes[i].GetMeta()
+			if err != nil {
+				return nil, err
+			}
+
+			// something we already generated, reconcile it to make sure it matches what
+			// is specified by the functionConfig
+			if meta.Name == functionConfig.Spec.Name &&
+				meta.Kind == "Service" &&
+				meta.APIVersion == "v1" {
+				// set some values
+				for k, v := range functionConfig.Spec.Annotations {
+					err := nodes[i].PipeE(yaml.SetAnnotation(k, v))
+					if err != nil {
+						return nil, err
+					}
+				}
+				found = true
+				break
+			}
+		}
+		if found {
+			return nodes, nil
+		}
+
+		// generate the resource if not found
+		n, err := yaml.Parse(fmt.Sprintf(`apiVersion: v1
+kind: Service
+metadata:
+  name: %s
+`, functionConfig.Spec.Name))
+		for k, v := range functionConfig.Spec.Annotations {
+			err := n.PipeE(yaml.SetAnnotation(k, v))
+			if err != nil {
+				return nil, err
+			}
+		}
+		nodes = append(nodes, n)
+		if err != nil {
+			return nil, err
+		}
+
+		return nodes, nil
+	})
+
+	// for testing purposes only -- normally read from stdin when Executing
+	cmd.SetIn(bytes.NewBufferString(`
+apiVersion: config.kubernetes.io/v1alpha1
+kind: ResourceList
+# items are provided as nodes
+items:
+- apiVersion: apps/v1
+  kind: Deployment
+  metadata:
+    name: foo
+- apiVersion: v1
+  kind: Service
+  metadata:
+    name: bar
+# functionConfig is parsed into flags by framework.Command
+functionConfig:
+  apiVersion: example.com/v1alpha1
+  kind: ExampleServiceGenerator
+  spec:
+    name: bar
+    annotations:
+      a: b
+`))
+
+	// run the command
+	if err := cmd.Execute(); err != nil {
+		panic(err)
+	}
+
+	// Output:
+	// apiVersion: config.kubernetes.io/v1alpha1
+	// kind: ResourceList
+	// items:
+	// - apiVersion: apps/v1
+	//   kind: Deployment
+	//   metadata:
+	//     name: foo
+	// - apiVersion: v1
+	//   kind: Service
+	//   metadata:
+	//     name: bar
+	//     annotations:
+	//       a: 'b'
+	// functionConfig:
+	//   apiVersion: example.com/v1alpha1
+	//   kind: ExampleServiceGenerator
+	//   spec:
+	//     name: bar
+	//     annotations:
+	//       a: b
+}
+
+// ExampleCommand_validate validates that all Deployment resources have the replicas field set.
+// If any Deployments do not contain spec.replicas, then the function will return Results
+// which will be set on ResourceList.results
+func ExampleCommand_validate() {
+	cmd := framework.Command(nil, func(nodes []*yaml.RNode) ([]*yaml.RNode, error) {
+		// validation results
+		var validationResults []framework.Item
+
+		// validate that each Deployment resource has spec.replicas set
+		for i := range nodes {
+			// only check Deployment resources
+			meta, err := nodes[i].GetMeta()
+			if err != nil {
+				return nil, err
+			}
+			if meta.Kind != "Deployment" {
+				continue
+			}
+
+			// lookup replicas field
+			r, err := nodes[i].Pipe(yaml.Lookup("spec", "replicas"))
+			if err != nil {
+				return nil, err
+			}
+
+			// check replicas not specified
+			if r != nil {
+				continue
+			}
+			validationResults = append(validationResults, framework.Item{
+				Severity:    framework.Error,
+				Message:     "missing replicas",
+				ResourceRef: meta,
+				Field: framework.Field{
+					Path:           "spec.field",
+					SuggestedValue: "1",
+				},
+			})
+		}
+
+		// framework will only consider Results an error if it has at least 1 item
+		return nodes, framework.Result{
+			Name:  "replicas-validator",
+			Items: validationResults,
+		}
+	})
+
+	// for testing purposes only -- normally read from stdin when Executing
+	cmd.SetIn(bytes.NewBufferString(`
+apiVersion: config.kubernetes.io/v1alpha1
+kind: ResourceList
+# items are provided as nodes
+items:
+- apiVersion: apps/v1
+  kind: Deployment
+  metadata:
+    name: foo
+`))
+
+	// run the command
+	if err := cmd.Execute(); err != nil {
+		// normally exit 1 here
+	}
+
+	// Output:
+	// apiVersion: config.kubernetes.io/v1alpha1
+	// kind: ResourceList
+	// items:
+	// - apiVersion: apps/v1
+	//   kind: Deployment
+	//   metadata:
+	//     name: foo
+	// results:
+	//   name: replicas-validator
+	//   items:
+	//   - message: missing replicas
+	//     severity: error
+	//     resourceRef:
+	//       apiVersion: apps/v1
+	//       kind: Deployment
+	//       metadata:
+	//         name: foo
+	//     field:
+	//       path: spec.field
+	//       suggestedValue: "1"
+}

kyaml/fn/framework/framework.go

@@ -0,0 +1,146 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package framework
+
+import (
+	"fmt"
+	"io/ioutil"
+	"path/filepath"
+
+	"github.com/spf13/cobra"
+	"sigs.k8s.io/kustomize/kyaml/errors"
+	"sigs.k8s.io/kustomize/kyaml/kio"
+	"sigs.k8s.io/kustomize/kyaml/yaml"
+)
+
+// Command provides a cobra.Command for running the function.
+//
+// If functionConfig is nil, the function may be configured with flags parsed from
+// the ResourceList.functionConfig by creating flags on the returned command.
+func Command(functionConfig interface{}, function Function) cobra.Command {
+	cmd := cobra.Command{}
+	addGenerate(&cmd)
+	cmd.RunE = func(cmd *cobra.Command, args []string) error {
+		err := execute(function, functionConfig, cmd)
+		if err != nil {
+			fmt.Fprintf(cmd.ErrOrStderr(), "%v", err)
+		}
+		return err
+	}
+	cmd.SilenceErrors = true
+	cmd.SilenceUsage = true
+	return cmd
+}
+
+func addGenerate(cmd *cobra.Command) {
+	gen := &cobra.Command{
+		Use:  "gen",
+		Args: cobra.ExactArgs(1),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			return ioutil.WriteFile(filepath.Join(args[0], "Dockerfile"), []byte(`FROM golang:1.13-stretch
+ENV CGO_ENABLED=0
+WORKDIR /go/src/
+COPY . .
+RUN go build -v -o /usr/local/bin/function ./
+
+FROM alpine:latest
+COPY --from=0 /usr/local/bin/function /usr/local/bin/function
+CMD ["function"]
+`), 0600)
+		},
+	}
+	cmd.AddCommand(gen)
+}
+
+func execute(function Function, functionConfig interface{}, cmd *cobra.Command) error {
+	rw := &kio.ByteReadWriter{
+		Reader:                cmd.InOrStdin(),
+		Writer:                cmd.OutOrStdout(),
+		KeepReaderAnnotations: true,
+	}
+	nodes, err := rw.Read()
+	if err != nil {
+		return errors.Wrap(err)
+	}
+
+	// parse the functionConfig
+	if rw.FunctionConfig != nil {
+		if functionConfig == nil {
+			functionConfig = map[string]interface{}{}
+		}
+
+		// unmarshal into the provided structure
+		err := yaml.Unmarshal([]byte(rw.FunctionConfig.MustString()), functionConfig)
+		if err != nil {
+			return errors.Wrap(err)
+		}
+
+		// set the functionConfig values as flags so they are easy to access
+		err = func() error {
+			if !cmd.HasFlags() {
+				return nil
+			}
+			// kpt serializes function arguments as a ConfigMap, read them from
+			// the data field.
+			fc, ok := functionConfig.(map[string]interface{})
+			if !ok {
+				// serialized as something else
+				return nil
+			}
+			if fc["data"] == nil {
+				return nil
+			}
+			data := fc["data"].(map[string]interface{})
+			// set the value of each flag from the ResourceList.function config input
+			// values
+			for k, v := range data {
+				s, ok := v.(string)
+				if !ok {
+					continue
+				}
+				if err = cmd.Flag(k).Value.Set(s); err != nil {
+					return errors.Wrap(err)
+				}
+			}
+			return nil
+		}()
+		if err != nil {
+			return err
+		}
+	}
+
+	// run the function implementation
+	nodes, err = function(nodes)
+
+	// set the ResourceList.results for validating functions
+	var result *Result
+	if err != nil {
+		if val, ok := err.(Result); ok {
+			if len(val.Items) > 0 {
+				result = &val
+				b, err := yaml.Marshal(val)
+				if err != nil {
+					return errors.Wrap(err)
+				}
+				y, err := yaml.Parse(string(b))
+				if err != nil {
+					return errors.Wrap(err)
+				}
+				rw.Results = y
+			}
+		} else {
+			return errors.Wrap(err)
+		}
+	}
+
+	// write the results
+	if err := rw.Write(nodes); err != nil {
+		return errors.Wrap(err)
+	}
+
+	if result != nil && result.ExitCode() != 0 {
+		return errors.Wrap(err)
+	}
+	return nil
+}

kyaml/fn/framework/framework_test.go

@@ -0,0 +1,53 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package framework_test
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"sigs.k8s.io/kustomize/kyaml/fn/framework"
+	"sigs.k8s.io/kustomize/kyaml/yaml"
+)
+
+func TestCommand_dockerfile(t *testing.T) {
+	d, err := ioutil.TempDir("", "kustomize")
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
+	defer os.RemoveAll(d)
+
+	// create a function
+	cmd := framework.Command(nil, func(nodes []*yaml.RNode) ([]*yaml.RNode, error) {
+		return nil, nil
+	})
+
+	// generate the Dockerfile
+	cmd.SetArgs([]string{"gen", d})
+	if !assert.NoError(t, cmd.Execute()) {
+		t.FailNow()
+	}
+
+	b, err := ioutil.ReadFile(filepath.Join(d, "Dockerfile"))
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
+
+	expected := `FROM golang:1.13-stretch
+ENV CGO_ENABLED=0
+WORKDIR /go/src/
+COPY . .
+RUN go build -v -o /usr/local/bin/function ./
+
+FROM alpine:latest
+COPY --from=0 /usr/local/bin/function /usr/local/bin/function
+CMD ["function"]
+`
+	if !assert.Equal(t, expected, string(b)) {
+		t.FailNow()
+	}
+}

kyaml/fn/framework/types.go

@@ -0,0 +1,92 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package framework
+
+import (
+	"strings"
+
+	"sigs.k8s.io/kustomize/kyaml/yaml"
+)
+
+// Function defines a function which mutates or validates a collection of configuration
+// To create a structured validation result, return a Result as the error.
+type Function func(nodes []*yaml.RNode) ([]*yaml.RNode, error)
+
+// Result defines a function result which will be set on the emitted ResourceList
+type Result struct {
+	// Name is the name of the function creating the result
+	Name string `yaml:"name,omitempty"`
+
+	// Items are the individual results
+	Items []Item `yaml:"items,omitempty"`
+}
+
+// Severity indicates the severity of the result
+type Severity string
+
+const (
+	// Error indicates the result is an error.  Will cause the function to exit non-0.
+	Error Severity = "error"
+	// Warning indicates the result is a warning
+	Warning Severity = "warning"
+	// Info indicates the result is an informative message
+	Info Severity = "info"
+)
+
+// Item defines a validation result
+type Item struct {
+	// Message is a human readable message
+	Message string `yaml:"message,omitempty"`
+
+	// Severity is the severity of the
+	Severity Severity `yaml:"severity,omitempty"`
+
+	// ResourceRef is a reference to a resource
+	ResourceRef yaml.ResourceMeta `yaml:"resourceRef,omitempty"`
+
+	Field Field `yaml:"field,omitempty"`
+
+	File File `yaml:"file,omitempty"`
+}
+
+// File references a file containing a resource
+type File struct {
+	// Path is relative path to the file containing the resource
+	Path string `yaml:"path,omitempty"`
+
+	// Index is the index into the file containing the resource
+	// (i.e. if there are multiple resources in a single file)
+	Index int `yaml:"index,omitempty"`
+}
+
+// Field references a field in a resource
+type Field struct {
+	// Path is the field path
+	Path string `yaml:"path,omitempty"`
+
+	// CurrentValue is the current field value
+	CurrentValue string `yaml:"currentValue,omitempty"`
+
+	// SuggestedValue is the suggested field value
+	SuggestedValue string `yaml:"suggestedValue,omitempty"`
+}
+
+// Error implement error
+func (e Result) Error() string {
+	var msgs []string
+	for _, i := range e.Items {
+		msgs = append(msgs, i.Message)
+	}
+	return strings.Join(msgs, "\n\n")
+}
+
+// ExitCode provides the exit code based on the result
+func (e Result) ExitCode() int {
+	for _, i := range e.Items {
+		if i.Severity == Error {
+			return 1
+		}
+	}
+	return 0
+}