Merge pull request #2296 from pwittrock/slfn

Refactor function code and allow for using starlark based filters

cmd/config/go.sum

@@ -109,6 +109,7 @@ github.com/posener/complete/v2 v2.0.1-alpha.12 h1:0wvkuDfHb5vSZlNBYgpEH4XQHpF46M
 github.com/posener/complete/v2 v2.0.1-alpha.12/go.mod h1://JlL91cS2JV7rOl6LVHrRqBXoBUecJu3ILQPgbJiMQ=
 github.com/posener/script v1.0.4 h1:nSuXW5ZdmFnQIueLB2s0qvs4oNsUloM1Zydzh75v42w=
 github.com/posener/script v1.0.4/go.mod h1:Rg3ijooqulo05aGLyGsHoLmIOUzHUVK19WVgrYBPU/E=
+github.com/qri-io/starlib v0.4.2-0.20200213133954-ff2e8cd5ef8d h1:K6eOUihrFLdZjZnA4XlRp864fmWXv9YTIk7VPLhRacA=
 github.com/qri-io/starlib v0.4.2-0.20200213133954-ff2e8cd5ef8d/go.mod h1:7DPO4domFU579Ga6E61sB9VFNaniPVwJP5C4bBCu3wA=
 github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
 github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0=
@@ -135,6 +136,7 @@ github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca h1:1CFlNzQhALwjS9mB
 github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg=
 github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
 go.starlark.net v0.0.0-20190528202925-30ae18b8564f/go.mod h1:c1/X6cHgvdXj6pUlmWKMkuqRnW4K8x2vwt6JAaaircg=
+go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5 h1:+FNtrFTmVw0YZGpBGX56XDee331t6JAXeK2bcyhLOOc=
 go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5/go.mod h1:nmDLcffg48OtT/PSW0Hg7FvpRQsQh5OSqIylirxKC7o=
 golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=

cmd/config/internal/commands/run-fns.go

@@ -40,6 +40,16 @@ func GetRunFnRunner(name string) *RunFnRunner {
 	r.Command.Flags().StringVar(
 		&r.Image, "image", "",
 		"run this image as a function instead of discovering them.")
+	r.Command.Flags().BoolVar(
+		&r.EnableStar, "enable-star", false, "enable support for starlark functions.")
+	r.Command.Flags().MarkHidden("enable-star")
+	r.Command.Flags().StringVar(
+		&r.StarPath, "star-path", "", "run a starlark script as a function.")
+	r.Command.Flags().MarkHidden("star-path")
+	r.Command.Flags().StringVar(
+		&r.StarName, "star-name", "", "name of starlark program.")
+	r.Command.Flags().MarkHidden("star-name")
+
 	r.Command.Flags().BoolVar(
 		&r.Network, "network", false, "enable network access for functions that declare it")
 	r.Command.Flags().StringVar(
@@ -59,6 +69,9 @@ type RunFnRunner struct {
 	GlobalScope        bool
 	FnPaths            []string
 	Image              string
+	EnableStar         bool
+	StarPath           string
+	StarName           string
 	RunFns             runfn.RunFns
 	Network            bool
 	NetworkName        string
@@ -68,34 +81,61 @@ func (r *RunFnRunner) runE(c *cobra.Command, args []string) error {
 	return handleError(c, r.RunFns.Execute())
 }
 
-// getFunctions parses the commandline flags and arguments into explicit
+// getContainerFunctions parses the commandline flags and arguments into explicit
 // Functions to run.
-func (r *RunFnRunner) getFunctions(c *cobra.Command, args, dataItems []string) (
+func (r *RunFnRunner) getContainerFunctions(c *cobra.Command, args, dataItems []string) (
 	[]*yaml.RNode, error) {
-	// if image isn't specified, then Functions is empty
+	if r.Image == "" && r.StarPath == "" {
-	if r.Image == "" {
 		return nil, nil
 	}
 
-	// create the function spec to set as an annotation
+	var fn *yaml.RNode
-	fn, err := yaml.Parse(`container: {}`)
+	var err error
-	if err != nil {
+
-		return nil, err
+	// if image isn't specified, then Functions is empty
-	}
+	if r.Image != "" {
-	// TODO: add support network, volumes, etc based on flag values
+		// create the function spec to set as an annotation
-	err = fn.PipeE(
+		fn, err = yaml.Parse(`container: {}`)
-		yaml.Lookup("container"),
-		yaml.SetField("image", yaml.NewScalarRNode(r.Image)))
-	if err != nil {
-		return nil, err
-	}
-	if r.Network {
-		err = fn.PipeE(
-			yaml.LookupCreate(yaml.MappingNode, "container", "network"),
-			yaml.SetField("required", yaml.NewScalarRNode("true")))
 		if err != nil {
 			return nil, err
 		}
+		// TODO: add support network, volumes, etc based on flag values
+		err = fn.PipeE(
+			yaml.Lookup("container"),
+			yaml.SetField("image", yaml.NewScalarRNode(r.Image)))
+		if err != nil {
+			return nil, err
+		}
+		if r.Network {
+			err = fn.PipeE(
+				yaml.LookupCreate(yaml.MappingNode, "container", "network"),
+				yaml.SetField("required", yaml.NewScalarRNode("true")))
+			if err != nil {
+				return nil, err
+			}
+		}
+	} else if r.EnableStar && r.StarPath != "" {
+		// create the function spec to set as an annotation
+		fn, err = yaml.Parse(`starlark: {}`)
+		if err != nil {
+			return nil, err
+		}
+
+		err = fn.PipeE(
+			yaml.Lookup("starlark"),
+			yaml.SetField("path", yaml.NewScalarRNode(r.StarPath)))
+		if err != nil {
+			return nil, err
+		}
+
+		err = fn.PipeE(
+			yaml.Lookup("starlark"),
+			yaml.SetField("name", yaml.NewScalarRNode(r.StarName)))
+		if err != nil {
+			return nil, err
+		}
+	} else {
+		return nil, nil
 	}
 
 	// create the function config
@@ -160,7 +200,12 @@ data: {}
 }
 
 func (r *RunFnRunner) preRunE(c *cobra.Command, args []string) error {
-	if c.ArgsLenAtDash() >= 0 && r.Image == "" {
+	if r.EnableStar != (r.StarPath != "") {
+		return errors.Errorf("must specify --star-path with --enable-star")
+	}
+
+	if c.ArgsLenAtDash() >= 0 && r.Image == "" &&
+		!(r.EnableStar && r.StarPath != "") {
 		return errors.Errorf("must specify --image")
 	}
 
@@ -173,7 +218,7 @@ func (r *RunFnRunner) preRunE(c *cobra.Command, args []string) error {
 		return errors.Errorf("0 or 1 arguments supported, function arguments go after '--'")
 	}
 
-	fns, err := r.getFunctions(c, args, dataItems)
+	fns, err := r.getContainerFunctions(c, args, dataItems)
 	if err != nil {
 		return err
 	}
@@ -196,14 +241,15 @@ func (r *RunFnRunner) preRunE(c *cobra.Command, args []string) error {
 	}
 
 	r.RunFns = runfn.RunFns{
-		FunctionPaths: r.FnPaths,
+		FunctionPaths:  r.FnPaths,
-		GlobalScope:   r.GlobalScope,
+		GlobalScope:    r.GlobalScope,
-		Functions:     fns,
+		Functions:      fns,
-		Output:        output,
+		Output:         output,
-		Input:         input,
+		Input:          input,
-		Path:          path,
+		Path:           path,
-		Network:       r.Network,
+		Network:        r.Network,
-		NetworkName:   r.NetworkName,
+		NetworkName:    r.NetworkName,
+		EnableStarlark: r.EnableStar,
 	}
 
 	// don't consider args for the function

cmd/config/internal/commands/run_test.go

@@ -154,6 +154,44 @@ kind: Foo
 apiVersion: v1
 `,
 		},
+		{
+			name: "star",
+			args: []string{"run", "dir",
+				"--enable-star",
+				"--star-path", "a/b/c",
+				"--star-name", "foo",
+				"--", "Foo", "g=h"},
+			path: "dir",
+			expected: `
+metadata:
+  name: function-input
+  annotations:
+    config.kubernetes.io/function: |
+      starlark: {path: a/b/c, name: foo}
+data: {g: h}
+kind: Foo
+apiVersion: v1
+`,
+		},
+		{
+			name: "star-not-enabled",
+			args: []string{"run", "dir",
+				"--star-path", "a/b/c",
+				"--star-name", "foo",
+				"--", "Foo", "g=h"},
+			path: "dir",
+			err:  "must specify --star-path with --enable-star",
+		},
+		{
+			name: "image-star-not-enabled",
+			args: []string{"run", "dir",
+				"--image", "some_image",
+				"--star-path", "a/b/c",
+				"--star-name", "foo",
+				"--", "Foo", "g=h"},
+			path: "dir",
+			err:  "must specify --star-path with --enable-star",
+		},
 		{
 			name:          "function paths",
 			args:          []string{"run", "dir", "--fn-path", "path1", "--fn-path", "path2"},

kyaml/kio/filters/container.go

@@ -389,85 +389,13 @@ type IsReconcilerFilter struct {
 func (c *IsReconcilerFilter) Filter(inputs []*yaml.RNode) ([]*yaml.RNode, error) {
 	var out []*yaml.RNode
 	for i := range inputs {
-		isContainerResource := GetFunctionSpec(inputs[i]) != nil
+		isFnResource := GetFunctionSpec(inputs[i]) != nil
-		if isContainerResource && !c.ExcludeReconcilers {
+		if isFnResource && !c.ExcludeReconcilers {
 			out = append(out, inputs[i])
 		}
-		if !isContainerResource && c.IncludeNonReconcilers {
+		if !isFnResource && c.IncludeNonReconcilers {
 			out = append(out, inputs[i])
 		}
 	}
 	return out, nil
 }
-
-const (
-	FunctionAnnotationKey    = "config.kubernetes.io/function"
-	oldFunctionAnnotationKey = "config.k8s.io/function"
-)
-
-var functionAnnotationKeys = []string{FunctionAnnotationKey, oldFunctionAnnotationKey}
-
-// getFunction parses the config function from the object if it is found
-func getFunction(n *yaml.RNode, meta yaml.ResourceMeta) *FunctionSpec {
-	var fs FunctionSpec
-	for _, s := range functionAnnotationKeys {
-		fn := meta.Annotations[s]
-		if fn != "" {
-			_ = yaml.Unmarshal([]byte(fn), &fs)
-			return &fs
-		}
-	}
-	n, err := n.Pipe(yaml.Lookup("metadata", "configFn"))
-	if err != nil || yaml.IsEmpty(n) {
-		return nil
-	}
-	s, err := n.String()
-	if err != nil {
-		return nil
-	}
-	_ = yaml.Unmarshal([]byte(s), &fs)
-	return &fs
-}
-
-type ContainerSpec struct {
-	Image   string           `json:"image,omitempty" yaml:"image,omitempty"`
-	Network ContainerNetwork `json:"network,omitempty" yaml:"network,omitempty"`
-}
-
-type FunctionSpec struct {
-	Path      string        `json:"path,omitempty" yaml:"path,omitempty"`
-	Network   string        `json:"network,omitempty" yaml:"network,omitempty"`
-	Container ContainerSpec `json:"container,omitempty" yaml:"container,omitempty"`
-}
-
-type ContainerNetwork struct {
-	Required bool `json:"required,omitempty" yaml:"required,omitempty"`
-}
-
-// GetFunctionSpec returns the FunctionSpec for a resource.  Returns
-// nil if the resource does not have a FunctionSpec.
-//
-// The FunctionSpec is read from the resource metadata.annotation
-// "config.kubernetes.io/function"
-func GetFunctionSpec(n *yaml.RNode) *FunctionSpec {
-	meta, err := n.GetMeta()
-	if err != nil {
-		return nil
-	}
-
-	// path to the function, this will be mounted into the container
-	path := meta.Annotations[kioutil.PathAnnotation]
-	if fn := getFunction(n, meta); fn != nil {
-		fn.Network = ""
-		fn.Path = path
-		return fn
-	}
-
-	// legacy function specification for backwards compatibility
-	container := meta.Annotations["config.kubernetes.io/container"]
-	if container != "" {
-		return &FunctionSpec{
-			Path: path, Container: ContainerSpec{Image: container}}
-	}
-	return nil
-}

kyaml/kio/filters/container_test.go

@@ -511,9 +511,7 @@ metadata:
 		if !assert.NoError(t, err) {
 			return
 		}
-
+		fn := GetFunctionSpec(cfg)
-		meta, _ := cfg.GetMeta()
-		fn := getFunction(cfg, meta)
 		assert.Equal(t, tc.required, fn.Container.Network.Required)
 	}
 }

kyaml/kio/filters/functiontypes.go

@@ -0,0 +1,105 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package filters
+
+import (
+	"sigs.k8s.io/kustomize/kyaml/kio/kioutil"
+	"sigs.k8s.io/kustomize/kyaml/yaml"
+)
+
+const (
+	FunctionAnnotationKey    = "config.kubernetes.io/function"
+	oldFunctionAnnotationKey = "config.k8s.io/function"
+)
+
+var functionAnnotationKeys = []string{FunctionAnnotationKey, oldFunctionAnnotationKey}
+
+// FunctionSpec defines a spec for running a function
+type FunctionSpec struct {
+	// Path defines the path for scoped functions
+	Path string `json:"path,omitempty" yaml:"path,omitempty"`
+
+	// Network is the name of the network to use from a container
+	Network string `json:"network,omitempty" yaml:"network,omitempty"`
+
+	// Container is the spec for running a function as a container
+	Container ContainerSpec `json:"container,omitempty" yaml:"container,omitempty"`
+
+	// Starlark is the spec for running a function as a starlark script
+	Starlark StarlarkSpec `json:"starlark,omitempty" yaml:"starlark,omitempty"`
+}
+
+// ContainerSpec defines a spec for running a function as a container
+type ContainerSpec struct {
+	// Image is the container image to run
+	Image string `json:"image,omitempty" yaml:"image,omitempty"`
+
+	// Network defines network specific configuration
+	Network ContainerNetwork `json:"network,omitempty" yaml:"network,omitempty"`
+}
+
+// ContainerNetwork
+type ContainerNetwork struct {
+	// Required specifies that function requires a network
+	Required bool `json:"required,omitempty" yaml:"required,omitempty"`
+}
+
+// StarlarkSpec defines how to run a function as a starlark program
+type StarlarkSpec struct {
+	Name string `json:"name,omitempty" yaml:"name,omitempty"`
+
+	// Path specifies a path to a starlark script
+	Path string `json:"path,omitempty" yaml:"path,omitempty"`
+}
+
+// GetFunctionSpec returns the FunctionSpec for a resource.  Returns
+// nil if the resource does not have a FunctionSpec.
+//
+// The FunctionSpec is read from the resource metadata.annotation
+// "config.kubernetes.io/function"
+func GetFunctionSpec(n *yaml.RNode) *FunctionSpec {
+	meta, err := n.GetMeta()
+	if err != nil {
+		return nil
+	}
+
+	// path to the function, this will be mounted into the container
+	path := meta.Annotations[kioutil.PathAnnotation]
+	if fn := getFunctionSpecFromAnnotation(n, meta); fn != nil {
+		fn.Network = ""
+		fn.Path = path
+		return fn
+	}
+
+	// legacy function specification for backwards compatibility
+	container := meta.Annotations["config.kubernetes.io/container"]
+	if container != "" {
+		return &FunctionSpec{
+			Path: path, Container: ContainerSpec{Image: container}}
+	}
+	return nil
+}
+
+// getFunctionSpecFromAnnotation parses the config function from an annotation
+// if it is found
+func getFunctionSpecFromAnnotation(n *yaml.RNode, meta yaml.ResourceMeta) *FunctionSpec {
+	var fs FunctionSpec
+	for _, s := range functionAnnotationKeys {
+		fn := meta.Annotations[s]
+		if fn != "" {
+			_ = yaml.Unmarshal([]byte(fn), &fs)
+			return &fs
+		}
+	}
+	n, err := n.Pipe(yaml.Lookup("metadata", "configFn"))
+	if err != nil || yaml.IsEmpty(n) {
+		return nil
+	}
+	s, err := n.String()
+	if err != nil {
+		return nil
+	}
+	_ = yaml.Unmarshal([]byte(s), &fs)
+	return &fs
+}

kyaml/runfn/runfn.go

@@ -15,6 +15,7 @@ import (
 	"sigs.k8s.io/kustomize/kyaml/kio"
 	"sigs.k8s.io/kustomize/kyaml/kio/filters"
 	"sigs.k8s.io/kustomize/kyaml/kio/kioutil"
+	"sigs.k8s.io/kustomize/kyaml/starlark"
 	"sigs.k8s.io/kustomize/kyaml/yaml"
 )
 
@@ -57,6 +58,12 @@ type RunFns struct {
 	// and only use explicit sources
 	NoFunctionsFromInput *bool
 
+	// EnableStarlark will enable functions run as starlark scripts
+	EnableStarlark bool
+
+	// DisableContainers will disable functions run as containers
+	DisableContainers bool
+
 	// functionFilterProvider provides a filter to perform the function.
 	// this is a variable so it can be mocked in tests
 	functionFilterProvider func(
@@ -208,8 +215,10 @@ func (r RunFns) getFunctionFilters(global bool, fns ...*yaml.RNode) (
 			}
 			spec.Network = r.NetworkName
 		}
-
 		c := r.functionFilterProvider(*spec, api)
+		if c == nil {
+			continue
+		}
 		cf, ok := c.(*filters.ContainerFilter)
 		if global && ok {
 			cf.GlobalScope = true
@@ -291,7 +300,7 @@ func (r *RunFns) init() {
 
 // ffp provides function filters
 func (r *RunFns) ffp(spec filters.FunctionSpec, api *yaml.RNode) kio.Filter {
-	if spec.Container.Image != "" {
+	if !r.DisableContainers && spec.Container.Image != "" {
 		return &filters.ContainerFilter{
 			Image:         spec.Container.Image,
 			Config:        api,
@@ -300,9 +309,12 @@ func (r *RunFns) ffp(spec filters.FunctionSpec, api *yaml.RNode) kio.Filter {
 			GlobalScope:   r.GlobalScope,
 		}
 	}
-	return noOpFilter
+	if r.EnableStarlark && spec.Starlark.Path != "" {
+		return &starlark.Filter{
+			Name:           spec.Starlark.Name,
+			Path:           spec.Starlark.Path,
+			FunctionConfig: api,
+		}
+	}
+	return nil
 }
-
-var noOpFilter = kio.FilterFunc(func(in []*yaml.RNode) ([]*yaml.RNode, error) {
-	return in, nil
-})

kyaml/runfn/runfn_test.go

@@ -190,6 +190,10 @@ func TestRunFns_getFilters(t *testing.T) {
 		name string
 		// value to set for NoFunctionsFromInput
 		noFunctionsFromInput *bool
+
+		enableStarlark bool
+
+		disableContainers bool
 	}{
 		// Test
 		//
@@ -213,6 +217,26 @@ metadata:
 			out: []string{"gcr.io/example.com/image:v1.0.0"},
 		},
 
+		{name: "disable containers",
+			in: []f{
+				{
+					path: filepath.Join("foo", "bar.yaml"),
+					value: `
+apiVersion: example.com/v1alpha1
+kind: ExampleFunction
+metadata:
+  annotations:
+    config.kubernetes.io/function: |
+      container:
+        image: gcr.io/example.com/image:v1.0.0
+    config.kubernetes.io/local-config: "true"
+`,
+				},
+			},
+			out:               nil,
+			disableContainers: true,
+		},
+
 		// Test
 		//
 		//
@@ -432,6 +456,45 @@ metadata:
 			},
 			out: []string{"b", "a", "c"},
 		},
+
+		// Test
+		//
+		//
+		{name: "starlark-function",
+			in: []f{
+				{
+					path: filepath.Join("foo", "bar.yaml"),
+					value: `
+apiVersion: example.com/v1alpha1
+kind: ExampleFunction
+metadata:
+  annotations:
+    config.kubernetes.io/function: |
+      starlark:
+        path: a/b/c
+`,
+				},
+			},
+			enableStarlark: true,
+			out:            []string{"name:  path: a/b/c url:  program:"},
+		},
+
+		{name: "starlark-function-disabled",
+			in: []f{
+				{
+					path: filepath.Join("foo", "bar.yaml"),
+					value: `
+apiVersion: example.com/v1alpha1
+kind: ExampleFunction
+metadata:
+  annotations:
+    config.kubernetes.io/function: |
+      starlark:
+        path: a/b/c
+`,
+				},
+			},
+		},
 	}
 
 	for i := range tests {
@@ -484,6 +547,8 @@ metadata:
 
 			// init the instance
 			r := &RunFns{
+				EnableStarlark:       tt.enableStarlark,
+				DisableContainers:    tt.disableContainers,
 				FunctionPaths:        fnPaths,
 				Functions:            parsedFns,
 				Path:                 d,

kyaml/starlark/starlark.go

@@ -35,6 +35,10 @@ type Filter struct {
 	FunctionConfig *yaml.RNode
 }
 
+func (sf *Filter) String() string {
+	return fmt.Sprintf("name: %v path: %v url: %v program: %v", sf.Name, sf.Path, sf.URL, sf.Program)
+}
+
 func (sf *Filter) Filter(input []*yaml.RNode) ([]*yaml.RNode, error) {
 	if sf.URL != "" && sf.Path != "" ||
 		sf.URL != "" && sf.Program != "" ||