Refactor starlark runtime ontop of runtimeutil

kyaml/fn/runtime/starlark/context.go

@@ -60,22 +60,6 @@ func env() (starlark.Value, error) {
 	return value, nil
 }
 
-func nodeToValue(node *yaml.RNode) (starlark.Value, error) {
-	s, err := node.String()
-	if err != nil {
-		return nil, errors.Wrap(err)
-	}
-	var in map[string]interface{}
-	if err := yaml.Unmarshal([]byte(s), &in); err != nil {
-		return nil, errors.Wrap(err)
-	}
-	value, err := util.Marshal(in)
-	if err != nil {
-		return nil, errors.Wrap(err)
-	}
-	return value, nil
-}
-
 func interfaceToValue(i interface{}) (starlark.Value, error) {
 	b, err := json.Marshal(i)
 	if err != nil {

kyaml/fn/runtime/starlark/example_test.go

@@ -11,8 +11,9 @@ import (
 	"os"
 	"path/filepath"
 
+	"sigs.k8s.io/kustomize/kyaml/fn/runtime/runtimeutil"
+	"sigs.k8s.io/kustomize/kyaml/fn/runtime/starlark"
 	"sigs.k8s.io/kustomize/kyaml/kio"
-	"sigs.k8s.io/kustomize/kyaml/starlark"
 	"sigs.k8s.io/kustomize/kyaml/yaml"
 )
 
@@ -75,6 +76,7 @@ run(ctx.resource_list["items"])
 	//   name: deployment-1
 	//   annotations:
 	//     foo: bar
+	//     config.kubernetes.io/path: 'deployment_deployment-1.yaml'
 	// spec:
 	//   template:
 	//     spec:
@@ -88,6 +90,7 @@ run(ctx.resource_list["items"])
 	//   name: deployment-2
 	//   annotations:
 	//     foo: bar
+	//     config.kubernetes.io/path: 'deployment_deployment-2.yaml'
 	// spec:
 	//   template:
 	//     spec:
@@ -141,7 +144,7 @@ def run(items, value):
 
 run(ctx.resource_list["items"], ctx.resource_list["functionConfig"]["spec"]["value"])
 `,
-		FunctionConfig: fc,
+		FunctionFilter: runtimeutil.FunctionFilter{FunctionConfig: fc},
 	}
 
 	// output contains the transformed resources
@@ -165,6 +168,7 @@ run(ctx.resource_list["items"], ctx.resource_list["functionConfig"]["spec"]["val
 	//   name: deployment-1
 	//   annotations:
 	//     foo: hello world
+	//     config.kubernetes.io/path: 'deployment_deployment-1.yaml'
 	// spec:
 	//   template:
 	//     spec:
@@ -178,6 +182,7 @@ run(ctx.resource_list["items"], ctx.resource_list["functionConfig"]["spec"]["val
 	//   name: deployment-2
 	//   annotations:
 	//     foo: hello world
+	//     config.kubernetes.io/path: 'deployment_deployment-2.yaml'
 	// spec:
 	//   template:
 	//     spec:

kyaml/fn/runtime/starlark/starlark.go

@@ -0,0 +1,224 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package starlark
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/http"
+
+	"github.com/qri-io/starlib/util"
+	"go.starlark.net/starlark"
+	"sigs.k8s.io/kustomize/kyaml/comments"
+	"sigs.k8s.io/kustomize/kyaml/errors"
+	"sigs.k8s.io/kustomize/kyaml/fn/runtime/runtimeutil"
+	"sigs.k8s.io/kustomize/kyaml/kio/filters"
+	"sigs.k8s.io/kustomize/kyaml/yaml"
+)
+
+// Filter transforms a set of resources through the provided program
+type Filter struct {
+	Name string
+
+	// Program is a starlark script which will be run against the resources
+	Program string
+
+	// URL is the url of a starlark program to fetch and run
+	URL string
+
+	// Path is the path to a starlark program to read and run
+	Path string
+
+	runtimeutil.FunctionFilter
+
+	ids map[string]*yaml.RNode
+}
+
+func (sf *Filter) String() string {
+	return fmt.Sprintf(
+		"name: %v path: %v url: %v program: %v", sf.Name, sf.Path, sf.URL, sf.Program)
+}
+
+func (sf *Filter) Filter(nodes []*yaml.RNode) ([]*yaml.RNode, error) {
+	err := sf.setup()
+	if err != nil {
+		return nil, err
+	}
+	sf.FunctionFilter.Run = sf.Run
+
+	return sf.FunctionFilter.Filter(nodes)
+}
+
+func (sf *Filter) setup() error {
+	if sf.URL != "" && sf.Path != "" ||
+		sf.URL != "" && sf.Program != "" ||
+		sf.Path != "" && sf.Program != "" {
+		return errors.Errorf("Filter Path, Program and URL are mutually exclusive")
+	}
+
+	// read the program from a file
+	if sf.Path != "" {
+		b, err := ioutil.ReadFile(sf.Path)
+		if err != nil {
+			return err
+		}
+		sf.Program = string(b)
+	}
+
+	// read the program from a URL
+	if sf.URL != "" {
+		err := func() error {
+			resp, err := http.Get(sf.URL)
+			if err != nil {
+				return err
+			}
+			defer resp.Body.Close()
+			b, err := ioutil.ReadAll(resp.Body)
+			if err != nil {
+				return err
+			}
+			sf.Program = string(b)
+			return nil
+		}()
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (sf *Filter) Run(reader io.Reader, writer io.Writer) error {
+	// retain map of inputs to outputs by id so if the name is changed by the
+	// starlark program, we are able to match the same resources
+	value, err := sf.readResourceList(reader)
+	if err != nil {
+		return errors.Wrap(err)
+	}
+
+	// run the starlark as program as transformation function
+	thread := &starlark.Thread{Name: sf.Name}
+
+	ctx := &Context{resourceList: value}
+	pd, err := ctx.predeclared()
+	if err != nil {
+		return errors.Wrap(err)
+	}
+	_, err = starlark.ExecFile(thread, sf.Name, sf.Program, pd)
+	if err != nil {
+		return errors.Wrap(err)
+	}
+
+	return sf.writeResourceList(value, writer)
+}
+
+// inputToResourceList transforms input into a starlark.Value
+func (sf *Filter) readResourceList(reader io.Reader) (starlark.Value, error) {
+	// read and parse the inputs
+	rl := bytes.Buffer{}
+	_, err := rl.ReadFrom(reader)
+	if err != nil {
+		return nil, errors.Wrap(err)
+	}
+	rn, err := yaml.Parse(rl.String())
+	if err != nil {
+		return nil, errors.Wrap(err)
+	}
+
+	// set the id on each node to map inputs to outputs
+	var id int
+	sf.ids = map[string]*yaml.RNode{}
+	items, err := rn.Pipe(yaml.Lookup("items"))
+	if err != nil {
+		return nil, errors.Wrap(err)
+	}
+	err = items.VisitElements(func(node *yaml.RNode) error {
+		id++
+		idStr := fmt.Sprintf("%v", id)
+		sf.ids[idStr] = node
+		return node.PipeE(yaml.SetAnnotation("config.k8s.io/id", idStr))
+	})
+	if err != nil {
+		return nil, errors.Wrap(err)
+	}
+
+	// convert to a starlark value
+	b, err := yaml.Marshal(rn.Document()) // convert to bytes
+	if err != nil {
+		return nil, errors.Wrap(err)
+	}
+	var in map[string]interface{}
+	err = yaml.Unmarshal(b, &in) // convert to map[string]interface{}
+	if err != nil {
+		return nil, errors.Wrap(err)
+	}
+	return util.Marshal(in) // convert to starlark value
+}
+
+// resourceListToOutput converts the output of the starlark program to the filter output
+func (sf *Filter) writeResourceList(value starlark.Value, writer io.Writer) error {
+	// convert the modified resourceList back into a slice of RNodes
+	// by first converting to a map[string]interface{}
+	out, err := util.Unmarshal(value)
+	if err != nil {
+		return errors.Wrap(err)
+	}
+	b, err := yaml.Marshal(out)
+	if err != nil {
+		return errors.Wrap(err)
+	}
+
+	rl, err := yaml.Parse(string(b))
+	if err != nil {
+		return errors.Wrap(err)
+	}
+
+	// preserve the comments from the input
+	items, err := rl.Pipe(yaml.Lookup("items"))
+	if err != nil {
+		return errors.Wrap(err)
+	}
+	err = items.VisitElements(func(node *yaml.RNode) error {
+		anID, err := node.Pipe(yaml.GetAnnotation("config.k8s.io/id"))
+		if err != nil {
+			return errors.Wrap(err)
+		}
+		if anID == nil {
+			return nil
+		}
+
+		var in *yaml.RNode
+		var found bool
+		if in, found = sf.ids[anID.YNode().Value]; !found {
+			return nil
+		}
+		if err := node.PipeE(yaml.ClearAnnotation("config.k8s.io/id")); err != nil {
+			return errors.Wrap(err)
+		}
+		if err := comments.CopyComments(in, node); err != nil {
+			return errors.Wrap(err)
+		}
+
+		// starlark will serialize the resources sorting the fields alphabetically,
+		// format them to have a better ordering
+		fmtFltr := filters.FormatFilter{}
+		if _, err := fmtFltr.Filter([]*yaml.RNode{node}); err != nil {
+			return errors.Wrap(err)
+		}
+		return nil
+	})
+	if err != nil {
+		return errors.Wrap(err)
+	}
+
+	s, err := rl.String()
+	if err != nil {
+		return errors.Wrap(err)
+	}
+
+	_, err = writer.Write([]byte(s))
+	return err
+}

kyaml/fn/runtime/starlark/starlark_test.go

@@ -56,6 +56,7 @@ metadata:
   name: nginx-deployment
   annotations:
     foo: bar
+    config.kubernetes.io/path: 'deployment_nginx-deployment.yaml'
 spec:
   template:
     spec:
@@ -95,6 +96,7 @@ metadata:
   name: nginx-deployment
   annotations:
     foo: annotation-value
+    config.kubernetes.io/path: 'deployment_nginx-deployment.yaml'
 spec:
   template:
     spec:
@@ -133,6 +135,7 @@ metadata:
   name: nginx-deployment
   annotations:
     foo: Deployment enables declarative updates for Pods and ReplicaSets.
+    config.kubernetes.io/path: 'deployment_nginx-deployment.yaml'
 spec:
   template:
     spec:
@@ -174,6 +177,7 @@ metadata:
   name: nginx-deployment
   annotations:
     foo: bar
+    config.kubernetes.io/path: 'deployment_nginx-deployment.yaml'
 spec:
   template:
     spec:
@@ -213,6 +217,8 @@ apiVersion: apps/v1
 kind: Deployment
 metadata:
   name: nginx-deployment
+  annotations:
+    config.kubernetes.io/path: 'deployment_nginx-deployment.yaml'
 spec:
   template:
     spec:
@@ -266,6 +272,7 @@ metadata:
   name: nginx-deployment-1
   annotations:
     foo: bar
+    config.kubernetes.io/path: 'deployment_nginx-deployment-1.yaml'
 spec:
   template:
     spec:
@@ -280,6 +287,7 @@ metadata:
   name: nginx-deployment-2
   annotations:
     foo: bar
+    config.kubernetes.io/path: 'deployment_nginx-deployment-2.yaml'
 spec:
   template:
     spec:
@@ -318,13 +326,10 @@ run(ctx.resource_list["items"])
 			expected: `
 apiVersion: apps/v1
 kind: Deployment
-metadata:
-  name: nginx-deployment-2
----
-apiVersion: apps/v1
-kind: Deployment
 metadata:
   name: nginx-deployment-1
+  annotations:
+    config.kubernetes.io/path: 'deployment_nginx-deployment-1.yaml'
 spec:
   template:
     spec:
@@ -332,6 +337,13 @@ spec:
       - name: nginx
         # head comment
         image: nginx:1.8.1 # {"$ref": "#/definitions/io.k8s.cli.substitutions.image"}
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: nginx-deployment-2
+  annotations:
+    config.kubernetes.io/path: 'deployment_nginx-deployment-2.yaml'
 `,
 		},
 		{
@@ -357,6 +369,8 @@ apiVersion: apps/v1
 kind: Deployment
 metadata:
   name: nginx-deployment-1
+  annotations:
+    config.kubernetes.io/path: 'deployment_nginx-deployment-1.yaml'
 `,
 		},
 		{
@@ -395,6 +409,7 @@ metadata:
   name: nginx-deployment
   annotations:
     foo: hello world
+    config.kubernetes.io/path: 'deployment_nginx-deployment.yaml'
 spec:
   template:
     spec:
@@ -406,7 +421,7 @@ spec:
 			expectedFunctionConfig: `
 kind: Script
 spec:
-  value: hello world
+  value: "hello world"
 `,
 		},
 
@@ -447,6 +462,7 @@ metadata:
   name: nginx-deployment
   annotations:
     foo: hello world
+    config.kubernetes.io/path: 'deployment_nginx-deployment.yaml'
 spec:
   template:
     spec:
@@ -458,7 +474,7 @@ spec:
 			expectedFunctionConfig: `
 kind: Script
 spec:
-  value: updated
+  value: "hello world"
 `,
 		},
 	}

kyaml/runfn/runfn.go

@@ -16,9 +16,9 @@ import (
 	"sigs.k8s.io/kustomize/kyaml/errors"
 	"sigs.k8s.io/kustomize/kyaml/fn/runtime/container"
 	"sigs.k8s.io/kustomize/kyaml/fn/runtime/runtimeutil"
+	"sigs.k8s.io/kustomize/kyaml/fn/runtime/starlark"
 	"sigs.k8s.io/kustomize/kyaml/kio"
 	"sigs.k8s.io/kustomize/kyaml/kio/kioutil"
-	"sigs.k8s.io/kustomize/kyaml/starlark"
 	"sigs.k8s.io/kustomize/kyaml/yaml"
 )
 
@@ -375,7 +375,7 @@ func (r *RunFns) ffp(spec runtimeutil.FunctionSpec, api *yaml.RNode) (kio.Filter
 		return &starlark.Filter{
 			Name:           spec.Starlark.Name,
 			Path:           p,
-			FunctionConfig: api,
+			FunctionFilter: runtimeutil.FunctionFilter{FunctionConfig: api},
 		}, nil
 	}
 	return nil, nil

kyaml/starlark/starlark.go

@@ -1,251 +0,0 @@
-// Copyright 2019 The Kubernetes Authors.
-// SPDX-License-Identifier: Apache-2.0
-
-package starlark
-
-import (
-	"fmt"
-	"io/ioutil"
-	"net/http"
-	"strconv"
-
-	"github.com/qri-io/starlib/util"
-	"go.starlark.net/starlark"
-	"sigs.k8s.io/kustomize/kyaml/comments"
-	"sigs.k8s.io/kustomize/kyaml/errors"
-	"sigs.k8s.io/kustomize/kyaml/kio/filters"
-	"sigs.k8s.io/kustomize/kyaml/yaml"
-)
-
-// Filter transforms a set of resources through the provided program
-type Filter struct {
-	Name string
-
-	// Program is a starlark script which will be run against the resources
-	Program string
-
-	// URL is the url of a starlark program to fetch and run
-	URL string
-
-	// Path is the path to a starlark program to read and run
-	Path string
-
-	// FunctionConfig is the value to be provided for resourceList.functionConfig as specified by
-	// https://github.com/kubernetes-sigs/kustomize/blob/master/cmd/config/docs/api-conventions/functions-spec.md.
-	FunctionConfig *yaml.RNode
-}
-
-func (sf *Filter) String() string {
-	return fmt.Sprintf("name: %v path: %v url: %v program: %v", sf.Name, sf.Path, sf.URL, sf.Program)
-}
-
-func (sf *Filter) Filter(input []*yaml.RNode) ([]*yaml.RNode, error) {
-	if sf.URL != "" && sf.Path != "" ||
-		sf.URL != "" && sf.Program != "" ||
-		sf.Path != "" && sf.Program != "" {
-		return nil, errors.Errorf("Filter Path, Program and URL are mutually exclusive")
-	}
-
-	// read the program from a file
-	if sf.Path != "" {
-		b, err := ioutil.ReadFile(sf.Path)
-		if err != nil {
-			return nil, err
-		}
-		sf.Program = string(b)
-	}
-
-	// read the program from a URL
-	if sf.URL != "" {
-		err := func() error {
-			resp, err := http.Get(sf.URL)
-			if err != nil {
-				return err
-			}
-			defer resp.Body.Close()
-			b, err := ioutil.ReadAll(resp.Body)
-			if err != nil {
-				return err
-			}
-			sf.Program = string(b)
-			return nil
-		}()
-		if err != nil {
-			return nil, err
-		}
-	}
-
-	// retain map of inputs to outputs by id so if the name is changed by the
-	// starlark program, we are able to match the same resources
-	value, ids, err := sf.inputToResourceList(input)
-	if err != nil {
-		return nil, errors.Wrap(err)
-	}
-
-	// run the starlark as program as transformation function
-	thread := &starlark.Thread{Name: sf.Name}
-
-	ctx := &Context{
-		resourceList: value,
-	}
-	pd, err := ctx.predeclared()
-	if err != nil {
-		return nil, errors.Wrap(err)
-	}
-	_, err = starlark.ExecFile(thread, sf.Name, sf.Program, pd)
-	if err != nil {
-		return nil, errors.Wrap(err)
-	}
-
-	results, err := sf.resourceListToOutput(value, ids)
-	if err != nil {
-		return nil, errors.Wrap(err)
-	}
-
-	// starlark will serialize the resources sorting the fields alphabetically,
-	// format them to have a better ordering
-	return filters.FormatFilter{}.Filter(results)
-}
-
-// tuple maps an input resource to the output resource
-type tuple struct {
-	// in is the RNode provided to the starlark program
-	in *yaml.RNode
-	// out is the RNode emitted by the starlark program with the id matching in
-	out *yaml.RNode
-}
-
-// inputToResourceList transforms input into a starlark.Value
-func (sf *Filter) inputToResourceList(
-	input []*yaml.RNode) (starlark.Value, map[int]*tuple, error) {
-	var id int
-	ids := map[int]*tuple{}
-
-	// convert into a ResourceList which will be converted to a starlark dictionary
-	// create the ResourceList
-	resourceList, err := yaml.Parse(`kind: ResourceList`)
-	if err != nil {
-		return nil, nil, errors.Wrap(err)
-	}
-
-	// set the functionConfig
-	if sf.FunctionConfig != nil {
-		if err := resourceList.PipeE(
-			yaml.FieldSetter{Name: "functionConfig", Value: sf.FunctionConfig}); err != nil {
-			return nil, nil, err
-		}
-	}
-
-	// the inputs should be provided as the list "items"
-	items, err := resourceList.Pipe(yaml.LookupCreate(yaml.SequenceNode, "items"))
-	if err != nil {
-		return nil, nil, errors.Wrap(err)
-	}
-	// add the input as items, give each resource an id
-	for i := range input {
-		item := input[i]
-
-		// create an id for tracking the resource through the program
-		err := item.PipeE(yaml.SetAnnotation("config.k8s.io/id", fmt.Sprintf("%d", id)))
-		if err != nil {
-			return nil, nil, errors.Wrap(err)
-		}
-		ids[id] = &tuple{in: item}
-		id++
-
-		items.YNode().Content = append(items.YNode().Content, item.YNode())
-	}
-
-	// convert the ResourceList into a starlark dictionary by
-	// first converting it into a map[string]interface{}
-	value, err := nodeToValue(resourceList)
-	return value, ids, err
-}
-
-// resourceListToOutput converts the output of the starlark program to the filter output
-func (sf *Filter) resourceListToOutput(
-	value starlark.Value, ids map[int]*tuple) ([]*yaml.RNode, error) {
-	// convert the modified resourceList back into a slice of RNodes
-	// by first converting to a map[string]interface{}
-	out, err := util.Unmarshal(value)
-	if err != nil {
-		return nil, errors.Wrap(err)
-	}
-	o := out.(map[string]interface{})
-
-	// parse the function config
-	if _, found := o["functionConfig"]; found {
-		fc := (o["functionConfig"].(map[string]interface{}))
-		b, err := yaml.Marshal(fc)
-		if err != nil {
-			return nil, errors.Wrap(err)
-		}
-		sf.FunctionConfig, err = yaml.Parse(string(b))
-		if err != nil {
-			return nil, errors.Wrap(err)
-		}
-	}
-
-	// parse the items
-	// copy the items out of the ResourceList, and into the Filter output
-	var results []*yaml.RNode
-	it := (o["items"].([]interface{}))
-	for i := range it {
-		// convert the resource back to the native yaml form
-		b, err := yaml.Marshal(it[i])
-		if err != nil {
-			return nil, errors.Wrap(err)
-		}
-		node, err := yaml.Parse(string(b))
-		if err != nil {
-			return nil, errors.Wrap(err)
-		}
-
-		// match it to an input
-		idS, err := node.Pipe(yaml.GetAnnotation("config.k8s.io/id"))
-		if err != nil {
-			return nil, errors.Wrap(err)
-		}
-		if idS == nil {
-			// no matching input -- new resource
-			results = append(results, node)
-			continue
-		}
-
-		id, err := strconv.Atoi(idS.YNode().Value)
-		if err != nil {
-			return nil, errors.Wrap(err)
-		}
-		if match, found := ids[id]; found {
-			// matching resources
-			match.out = node
-		} else {
-			// no matching input with the same id -- new resource
-			// this may be an error case, the outputs probably shouldn't have ids
-			// assigned by the starlark program
-			results = append(results, node)
-		}
-	}
-
-	// retain the comments instead of dropping them by copying them from the original inputs
-	for i := 0; i < len(ids); i++ {
-		v := ids[i]
-		if v.out == nil {
-			continue
-		}
-		if err := comments.CopyComments(v.in, v.out); err != nil {
-			return nil, errors.Wrap(err)
-		}
-		results = append(results, v.out)
-	}
-
-	// delete the ids from resources, these were only to track through the starlark program
-	// and that is finished.
-	for i := range results {
-		err := results[i].PipeE(yaml.ClearAnnotation("config.k8s.io/id"))
-		if err != nil {
-			return nil, err
-		}
-	}
-	return results, nil
-}