kubernetes-sigs/kustomize
1
0
Fork 0
mirror of https://github.com/kubernetes-sigs/kustomize.git synced 2026-06-12 01:14:22 +00:00
Code Issues Packages Projects Releases 1 Wiki Activity

Refactor starlark runtime ontop of runtimeutil

Browse Source
This commit is contained in:
Phillip Wittrock
2020-05-04 16:07:50 -07:00
parent 594c48d19a
commit 174b2ed62e
7 changed files with 256 additions and 278 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
kyaml/starlark/context.go → kyaml/fn/runtime/starlark/context.go
View File

@@ -60,22 +60,6 @@ func env() (starlark.Value, error) {
return value, nil
}
func nodeToValue(node *yaml.RNode) (starlark.Value, error) {
s, err := node.String()
if err != nil {
return nil, errors.Wrap(err)
}
var in map[string]interface{}
if err := yaml.Unmarshal([]byte(s), &in); err != nil {
return nil, errors.Wrap(err)
}
value, err := util.Marshal(in)
if err != nil {
return nil, errors.Wrap(err)
}
return value, nil
}
func interfaceToValue(i interface{}) (starlark.Value, error) {
b, err := json.Marshal(i)
if err != nil {

0
kyaml/starlark/doc.go → kyaml/fn/runtime/starlark/doc.go
View File

9
kyaml/starlark/example_test.go → kyaml/fn/runtime/starlark/example_test.go
View File

@@ -11,8 +11,9 @@ import (
"os"
"path/filepath"
"sigs.k8s.io/kustomize/kyaml/fn/runtime/runtimeutil"
"sigs.k8s.io/kustomize/kyaml/fn/runtime/starlark"
"sigs.k8s.io/kustomize/kyaml/kio"
"sigs.k8s.io/kustomize/kyaml/starlark"
"sigs.k8s.io/kustomize/kyaml/yaml"
)
@@ -75,6 +76,7 @@ run(ctx.resource_list["items"])
// name: deployment-1
// annotations:
// foo: bar
// config.kubernetes.io/path: 'deployment_deployment-1.yaml'
// spec:
// template:
// spec:
@@ -88,6 +90,7 @@ run(ctx.resource_list["items"])
// name: deployment-2
// annotations:
// foo: bar
// config.kubernetes.io/path: 'deployment_deployment-2.yaml'
// spec:
// template:
// spec:
@@ -141,7 +144,7 @@ def run(items, value):
run(ctx.resource_list["items"], ctx.resource_list["functionConfig"]["spec"]["value"])
`,
FunctionConfig: fc,
FunctionFilter: runtimeutil.FunctionFilter{FunctionConfig: fc},
}
// output contains the transformed resources
@@ -165,6 +168,7 @@ run(ctx.resource_list["items"], ctx.resource_list["functionConfig"]["spec"]["val
// name: deployment-1
// annotations:
// foo: hello world
// config.kubernetes.io/path: 'deployment_deployment-1.yaml'
// spec:
// template:
// spec:
@@ -178,6 +182,7 @@ run(ctx.resource_list["items"], ctx.resource_list["functionConfig"]["spec"]["val
// name: deployment-2
// annotations:
// foo: hello world
// config.kubernetes.io/path: 'deployment_deployment-2.yaml'
// spec:
// template:
// spec:

224
kyaml/fn/runtime/starlark/starlark.go Normal file
View File

@@ -0,0 +1,224 @@
// Copyright 2019 The Kubernetes Authors.
// SPDX-License-Identifier: Apache-2.0
package starlark
import (
"bytes"
"fmt"
"io"
"io/ioutil"
"net/http"
"github.com/qri-io/starlib/util"
"go.starlark.net/starlark"
"sigs.k8s.io/kustomize/kyaml/comments"
"sigs.k8s.io/kustomize/kyaml/errors"
"sigs.k8s.io/kustomize/kyaml/fn/runtime/runtimeutil"
"sigs.k8s.io/kustomize/kyaml/kio/filters"
"sigs.k8s.io/kustomize/kyaml/yaml"
)
// Filter transforms a set of resources through the provided program
type Filter struct {
Name string
// Program is a starlark script which will be run against the resources
Program string
// URL is the url of a starlark program to fetch and run
URL string
// Path is the path to a starlark program to read and run
Path string
runtimeutil.FunctionFilter
ids map[string]*yaml.RNode
}
func (sf *Filter) String() string {
return fmt.Sprintf(
"name: %v path: %v url: %v program: %v", sf.Name, sf.Path, sf.URL, sf.Program)
}
func (sf *Filter) Filter(nodes []*yaml.RNode) ([]*yaml.RNode, error) {
err := sf.setup()
if err != nil {
return nil, err
}
sf.FunctionFilter.Run = sf.Run
return sf.FunctionFilter.Filter(nodes)
}
func (sf *Filter) setup() error {
if sf.URL != "" && sf.Path != "" ||
sf.URL != "" && sf.Program != "" ||
sf.Path != "" && sf.Program != "" {
return errors.Errorf("Filter Path, Program and URL are mutually exclusive")
}
// read the program from a file
if sf.Path != "" {
b, err := ioutil.ReadFile(sf.Path)
if err != nil {
return err
}
sf.Program = string(b)
}
// read the program from a URL
if sf.URL != "" {
err := func() error {
resp, err := http.Get(sf.URL)
if err != nil {
return err
}
defer resp.Body.Close()
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
return err
}
sf.Program = string(b)
return nil
}()
if err != nil {
return err
}
}
return nil
}
func (sf *Filter) Run(reader io.Reader, writer io.Writer) error {
// retain map of inputs to outputs by id so if the name is changed by the
// starlark program, we are able to match the same resources
value, err := sf.readResourceList(reader)
if err != nil {
return errors.Wrap(err)
}
// run the starlark as program as transformation function
thread := &starlark.Thread{Name: sf.Name}
ctx := &Context{resourceList: value}
pd, err := ctx.predeclared()
if err != nil {
return errors.Wrap(err)
}
_, err = starlark.ExecFile(thread, sf.Name, sf.Program, pd)
if err != nil {
return errors.Wrap(err)
}
return sf.writeResourceList(value, writer)
}
// inputToResourceList transforms input into a starlark.Value
func (sf *Filter) readResourceList(reader io.Reader) (starlark.Value, error) {
// read and parse the inputs
rl := bytes.Buffer{}
_, err := rl.ReadFrom(reader)
if err != nil {
return nil, errors.Wrap(err)
}
rn, err := yaml.Parse(rl.String())
if err != nil {
return nil, errors.Wrap(err)
}
// set the id on each node to map inputs to outputs
var id int
sf.ids = map[string]*yaml.RNode{}
items, err := rn.Pipe(yaml.Lookup("items"))
if err != nil {
return nil, errors.Wrap(err)
}
err = items.VisitElements(func(node *yaml.RNode) error {
id++
idStr := fmt.Sprintf("%v", id)
sf.ids[idStr] = node
return node.PipeE(yaml.SetAnnotation("config.k8s.io/id", idStr))
})
if err != nil {
return nil, errors.Wrap(err)
}
// convert to a starlark value
b, err := yaml.Marshal(rn.Document()) // convert to bytes
if err != nil {
return nil, errors.Wrap(err)
}
var in map[string]interface{}
err = yaml.Unmarshal(b, &in) // convert to map[string]interface{}
if err != nil {
return nil, errors.Wrap(err)
}
return util.Marshal(in) // convert to starlark value
}
// resourceListToOutput converts the output of the starlark program to the filter output
func (sf *Filter) writeResourceList(value starlark.Value, writer io.Writer) error {
// convert the modified resourceList back into a slice of RNodes
// by first converting to a map[string]interface{}
out, err := util.Unmarshal(value)
if err != nil {
return errors.Wrap(err)
}
b, err := yaml.Marshal(out)
if err != nil {
return errors.Wrap(err)
}
rl, err := yaml.Parse(string(b))
if err != nil {
return errors.Wrap(err)
}
// preserve the comments from the input
items, err := rl.Pipe(yaml.Lookup("items"))
if err != nil {
return errors.Wrap(err)
}
err = items.VisitElements(func(node *yaml.RNode) error {
anID, err := node.Pipe(yaml.GetAnnotation("config.k8s.io/id"))
if err != nil {
return errors.Wrap(err)
}
if anID == nil {
return nil
}
var in *yaml.RNode
var found bool
if in, found = sf.ids[anID.YNode().Value]; !found {
return nil
}
if err := node.PipeE(yaml.ClearAnnotation("config.k8s.io/id")); err != nil {
return errors.Wrap(err)
}
if err := comments.CopyComments(in, node); err != nil {
return errors.Wrap(err)
}
// starlark will serialize the resources sorting the fields alphabetically,
// format them to have a better ordering
fmtFltr := filters.FormatFilter{}
if _, err := fmtFltr.Filter([]*yaml.RNode{node}); err != nil {
return errors.Wrap(err)
}
return nil
})
if err != nil {
return errors.Wrap(err)
}
s, err := rl.String()
if err != nil {
return errors.Wrap(err)
}
_, err = writer.Write([]byte(s))
return err
}

30
kyaml/starlark/starlark_test.go → kyaml/fn/runtime/starlark/starlark_test.go
View File

@@ -56,6 +56,7 @@ metadata:
name: nginx-deployment
annotations:
foo: bar
config.kubernetes.io/path: 'deployment_nginx-deployment.yaml'
spec:
template:
spec:
@@ -95,6 +96,7 @@ metadata:
name: nginx-deployment
annotations:
foo: annotation-value
config.kubernetes.io/path: 'deployment_nginx-deployment.yaml'
spec:
template:
spec:
@@ -133,6 +135,7 @@ metadata:
name: nginx-deployment
annotations:
foo: Deployment enables declarative updates for Pods and ReplicaSets.
config.kubernetes.io/path: 'deployment_nginx-deployment.yaml'
spec:
template:
spec:
@@ -174,6 +177,7 @@ metadata:
name: nginx-deployment
annotations:
foo: bar
config.kubernetes.io/path: 'deployment_nginx-deployment.yaml'
spec:
template:
spec:
@@ -213,6 +217,8 @@ apiVersion: apps/v1
kind: Deployment
metadata:
name: nginx-deployment
annotations:
config.kubernetes.io/path: 'deployment_nginx-deployment.yaml'
spec:
template:
spec:
@@ -266,6 +272,7 @@ metadata:
name: nginx-deployment-1
annotations:
foo: bar
config.kubernetes.io/path: 'deployment_nginx-deployment-1.yaml'
spec:
template:
spec:
@@ -280,6 +287,7 @@ metadata:
name: nginx-deployment-2
annotations:
foo: bar
config.kubernetes.io/path: 'deployment_nginx-deployment-2.yaml'
spec:
template:
spec:
@@ -318,13 +326,10 @@ run(ctx.resource_list["items"])
expected: `
apiVersion: apps/v1
kind: Deployment
metadata:
name: nginx-deployment-2
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: nginx-deployment-1
annotations:
config.kubernetes.io/path: 'deployment_nginx-deployment-1.yaml'
spec:
template:
spec:
@@ -332,6 +337,13 @@ spec:
- name: nginx
# head comment
image: nginx:1.8.1 # {"$ref": "#/definitions/io.k8s.cli.substitutions.image"}
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: nginx-deployment-2
annotations:
config.kubernetes.io/path: 'deployment_nginx-deployment-2.yaml'
`,
},
{
@@ -357,6 +369,8 @@ apiVersion: apps/v1
kind: Deployment
metadata:
name: nginx-deployment-1
annotations:
config.kubernetes.io/path: 'deployment_nginx-deployment-1.yaml'
`,
},
{
@@ -395,6 +409,7 @@ metadata:
name: nginx-deployment
annotations:
foo: hello world
config.kubernetes.io/path: 'deployment_nginx-deployment.yaml'
spec:
template:
spec:
@@ -406,7 +421,7 @@ spec:
expectedFunctionConfig: `
kind: Script
spec:
value: hello world
value: "hello world"
`,
},
@@ -447,6 +462,7 @@ metadata:
name: nginx-deployment
annotations:
foo: hello world
config.kubernetes.io/path: 'deployment_nginx-deployment.yaml'
spec:
template:
spec:
@@ -458,7 +474,7 @@ spec:
expectedFunctionConfig: `
kind: Script
spec:
value: updated
value: "hello world"
`,
},
}

4
kyaml/runfn/runfn.go
View File

@@ -16,9 +16,9 @@ import (
"sigs.k8s.io/kustomize/kyaml/errors"
"sigs.k8s.io/kustomize/kyaml/fn/runtime/container"
"sigs.k8s.io/kustomize/kyaml/fn/runtime/runtimeutil"
"sigs.k8s.io/kustomize/kyaml/fn/runtime/starlark"
"sigs.k8s.io/kustomize/kyaml/kio"
"sigs.k8s.io/kustomize/kyaml/kio/kioutil"
"sigs.k8s.io/kustomize/kyaml/starlark"
"sigs.k8s.io/kustomize/kyaml/yaml"
)
@@ -375,7 +375,7 @@ func (r *RunFns) ffp(spec runtimeutil.FunctionSpec, api *yaml.RNode) (kio.Filter
return &starlark.Filter{
Name: spec.Starlark.Name,
Path: p,
FunctionConfig: api,
FunctionFilter: runtimeutil.FunctionFilter{FunctionConfig: api},
}, nil
}
return nil, nil

251
kyaml/starlark/starlark.go
View File

@@ -1,251 +0,0 @@
// Copyright 2019 The Kubernetes Authors.
// SPDX-License-Identifier: Apache-2.0
package starlark
import (
"fmt"
"io/ioutil"
"net/http"
"strconv"
"github.com/qri-io/starlib/util"
"go.starlark.net/starlark"
"sigs.k8s.io/kustomize/kyaml/comments"
"sigs.k8s.io/kustomize/kyaml/errors"
"sigs.k8s.io/kustomize/kyaml/kio/filters"
"sigs.k8s.io/kustomize/kyaml/yaml"
)
// Filter transforms a set of resources through the provided program
type Filter struct {
Name string
// Program is a starlark script which will be run against the resources
Program string
// URL is the url of a starlark program to fetch and run
URL string
// Path is the path to a starlark program to read and run
Path string
// FunctionConfig is the value to be provided for resourceList.functionConfig as specified by
// https://github.com/kubernetes-sigs/kustomize/blob/master/cmd/config/docs/api-conventions/functions-spec.md.
FunctionConfig *yaml.RNode
}
func (sf *Filter) String() string {
return fmt.Sprintf("name: %v path: %v url: %v program: %v", sf.Name, sf.Path, sf.URL, sf.Program)
}
func (sf *Filter) Filter(input []*yaml.RNode) ([]*yaml.RNode, error) {
if sf.URL != "" && sf.Path != "" ||
sf.URL != "" && sf.Program != "" ||
sf.Path != "" && sf.Program != "" {
return nil, errors.Errorf("Filter Path, Program and URL are mutually exclusive")
}
// read the program from a file
if sf.Path != "" {
b, err := ioutil.ReadFile(sf.Path)
if err != nil {
return nil, err
}
sf.Program = string(b)
}
// read the program from a URL
if sf.URL != "" {
err := func() error {
resp, err := http.Get(sf.URL)
if err != nil {
return err
}
defer resp.Body.Close()
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
return err
}
sf.Program = string(b)
return nil
}()
if err != nil {
return nil, err
}
}
// retain map of inputs to outputs by id so if the name is changed by the
// starlark program, we are able to match the same resources
value, ids, err := sf.inputToResourceList(input)
if err != nil {
return nil, errors.Wrap(err)
}
// run the starlark as program as transformation function
thread := &starlark.Thread{Name: sf.Name}
ctx := &Context{
resourceList: value,
}
pd, err := ctx.predeclared()
if err != nil {
return nil, errors.Wrap(err)
}
_, err = starlark.ExecFile(thread, sf.Name, sf.Program, pd)
if err != nil {
return nil, errors.Wrap(err)
}
results, err := sf.resourceListToOutput(value, ids)
if err != nil {
return nil, errors.Wrap(err)
}
// starlark will serialize the resources sorting the fields alphabetically,
// format them to have a better ordering
return filters.FormatFilter{}.Filter(results)
}
// tuple maps an input resource to the output resource
type tuple struct {
// in is the RNode provided to the starlark program
in *yaml.RNode
// out is the RNode emitted by the starlark program with the id matching in
out *yaml.RNode
}
// inputToResourceList transforms input into a starlark.Value
func (sf *Filter) inputToResourceList(
input []*yaml.RNode) (starlark.Value, map[int]*tuple, error) {
var id int
ids := map[int]*tuple{}
// convert into a ResourceList which will be converted to a starlark dictionary
// create the ResourceList
resourceList, err := yaml.Parse(`kind: ResourceList`)
if err != nil {
return nil, nil, errors.Wrap(err)
}
// set the functionConfig
if sf.FunctionConfig != nil {
if err := resourceList.PipeE(
yaml.FieldSetter{Name: "functionConfig", Value: sf.FunctionConfig}); err != nil {
return nil, nil, err
}
}
// the inputs should be provided as the list "items"
items, err := resourceList.Pipe(yaml.LookupCreate(yaml.SequenceNode, "items"))
if err != nil {
return nil, nil, errors.Wrap(err)
}
// add the input as items, give each resource an id
for i := range input {
item := input[i]
// create an id for tracking the resource through the program
err := item.PipeE(yaml.SetAnnotation("config.k8s.io/id", fmt.Sprintf("%d", id)))
if err != nil {
return nil, nil, errors.Wrap(err)
}
ids[id] = &tuple{in: item}
id++
items.YNode().Content = append(items.YNode().Content, item.YNode())
}
// convert the ResourceList into a starlark dictionary by
// first converting it into a map[string]interface{}
value, err := nodeToValue(resourceList)
return value, ids, err
}
// resourceListToOutput converts the output of the starlark program to the filter output
func (sf *Filter) resourceListToOutput(
value starlark.Value, ids map[int]*tuple) ([]*yaml.RNode, error) {
// convert the modified resourceList back into a slice of RNodes
// by first converting to a map[string]interface{}
out, err := util.Unmarshal(value)
if err != nil {
return nil, errors.Wrap(err)
}
o := out.(map[string]interface{})
// parse the function config
if _, found := o["functionConfig"]; found {
fc := (o["functionConfig"].(map[string]interface{}))
b, err := yaml.Marshal(fc)
if err != nil {
return nil, errors.Wrap(err)
}
sf.FunctionConfig, err = yaml.Parse(string(b))
if err != nil {
return nil, errors.Wrap(err)
}
}
// parse the items
// copy the items out of the ResourceList, and into the Filter output
var results []*yaml.RNode
it := (o["items"].([]interface{}))
for i := range it {
// convert the resource back to the native yaml form
b, err := yaml.Marshal(it[i])
if err != nil {
return nil, errors.Wrap(err)
}
node, err := yaml.Parse(string(b))
if err != nil {
return nil, errors.Wrap(err)
}
// match it to an input
idS, err := node.Pipe(yaml.GetAnnotation("config.k8s.io/id"))
if err != nil {
return nil, errors.Wrap(err)
}
if idS == nil {
// no matching input -- new resource
results = append(results, node)
continue
}
id, err := strconv.Atoi(idS.YNode().Value)
if err != nil {
return nil, errors.Wrap(err)
}
if match, found := ids[id]; found {
// matching resources
match.out = node
} else {
// no matching input with the same id -- new resource
// this may be an error case, the outputs probably shouldn't have ids
// assigned by the starlark program
results = append(results, node)
}
}
// retain the comments instead of dropping them by copying them from the original inputs
for i := 0; i < len(ids); i++ {
v := ids[i]
if v.out == nil {
continue
}
if err := comments.CopyComments(v.in, v.out); err != nil {
return nil, errors.Wrap(err)
}
results = append(results, v.out)
}
// delete the ids from resources, these were only to track through the starlark program
// and that is finished.
for i := range results {
err := results[i].PipeE(yaml.ClearAnnotation("config.k8s.io/id"))
if err != nil {
return nil, err
}
}
return results, nil
}