diff --git a/kyaml/starlark/context.go b/kyaml/fn/runtime/starlark/context.go similarity index 80% rename from kyaml/starlark/context.go rename to kyaml/fn/runtime/starlark/context.go index c13eae8f5..d72f5940c 100644 --- a/kyaml/starlark/context.go +++ b/kyaml/fn/runtime/starlark/context.go @@ -60,22 +60,6 @@ func env() (starlark.Value, error) { return value, nil } -func nodeToValue(node *yaml.RNode) (starlark.Value, error) { - s, err := node.String() - if err != nil { - return nil, errors.Wrap(err) - } - var in map[string]interface{} - if err := yaml.Unmarshal([]byte(s), &in); err != nil { - return nil, errors.Wrap(err) - } - value, err := util.Marshal(in) - if err != nil { - return nil, errors.Wrap(err) - } - return value, nil -} - func interfaceToValue(i interface{}) (starlark.Value, error) { b, err := json.Marshal(i) if err != nil { diff --git a/kyaml/starlark/doc.go b/kyaml/fn/runtime/starlark/doc.go similarity index 100% rename from kyaml/starlark/doc.go rename to kyaml/fn/runtime/starlark/doc.go diff --git a/kyaml/starlark/example_test.go b/kyaml/fn/runtime/starlark/example_test.go similarity index 93% rename from kyaml/starlark/example_test.go rename to kyaml/fn/runtime/starlark/example_test.go index 3c67bd6b3..5c4b3c1f7 100644 --- a/kyaml/starlark/example_test.go +++ b/kyaml/fn/runtime/starlark/example_test.go @@ -11,8 +11,9 @@ import ( "os" "path/filepath" + "sigs.k8s.io/kustomize/kyaml/fn/runtime/runtimeutil" + "sigs.k8s.io/kustomize/kyaml/fn/runtime/starlark" "sigs.k8s.io/kustomize/kyaml/kio" - "sigs.k8s.io/kustomize/kyaml/starlark" "sigs.k8s.io/kustomize/kyaml/yaml" ) @@ -75,6 +76,7 @@ run(ctx.resource_list["items"]) // name: deployment-1 // annotations: // foo: bar + // config.kubernetes.io/path: 'deployment_deployment-1.yaml' // spec: // template: // spec: @@ -88,6 +90,7 @@ run(ctx.resource_list["items"]) // name: deployment-2 // annotations: // foo: bar + // config.kubernetes.io/path: 'deployment_deployment-2.yaml' // spec: // template: // spec: @@ -141,7 +144,7 @@ def run(items, value): run(ctx.resource_list["items"], ctx.resource_list["functionConfig"]["spec"]["value"]) `, - FunctionConfig: fc, + FunctionFilter: runtimeutil.FunctionFilter{FunctionConfig: fc}, } // output contains the transformed resources @@ -165,6 +168,7 @@ run(ctx.resource_list["items"], ctx.resource_list["functionConfig"]["spec"]["val // name: deployment-1 // annotations: // foo: hello world + // config.kubernetes.io/path: 'deployment_deployment-1.yaml' // spec: // template: // spec: @@ -178,6 +182,7 @@ run(ctx.resource_list["items"], ctx.resource_list["functionConfig"]["spec"]["val // name: deployment-2 // annotations: // foo: hello world + // config.kubernetes.io/path: 'deployment_deployment-2.yaml' // spec: // template: // spec: diff --git a/kyaml/fn/runtime/starlark/starlark.go b/kyaml/fn/runtime/starlark/starlark.go new file mode 100644 index 000000000..1d2bd37e2 --- /dev/null +++ b/kyaml/fn/runtime/starlark/starlark.go @@ -0,0 +1,224 @@ +// Copyright 2019 The Kubernetes Authors. +// SPDX-License-Identifier: Apache-2.0 + +package starlark + +import ( + "bytes" + "fmt" + "io" + "io/ioutil" + "net/http" + + "github.com/qri-io/starlib/util" + "go.starlark.net/starlark" + "sigs.k8s.io/kustomize/kyaml/comments" + "sigs.k8s.io/kustomize/kyaml/errors" + "sigs.k8s.io/kustomize/kyaml/fn/runtime/runtimeutil" + "sigs.k8s.io/kustomize/kyaml/kio/filters" + "sigs.k8s.io/kustomize/kyaml/yaml" +) + +// Filter transforms a set of resources through the provided program +type Filter struct { + Name string + + // Program is a starlark script which will be run against the resources + Program string + + // URL is the url of a starlark program to fetch and run + URL string + + // Path is the path to a starlark program to read and run + Path string + + runtimeutil.FunctionFilter + + ids map[string]*yaml.RNode +} + +func (sf *Filter) String() string { + return fmt.Sprintf( + "name: %v path: %v url: %v program: %v", sf.Name, sf.Path, sf.URL, sf.Program) +} + +func (sf *Filter) Filter(nodes []*yaml.RNode) ([]*yaml.RNode, error) { + err := sf.setup() + if err != nil { + return nil, err + } + sf.FunctionFilter.Run = sf.Run + + return sf.FunctionFilter.Filter(nodes) +} + +func (sf *Filter) setup() error { + if sf.URL != "" && sf.Path != "" || + sf.URL != "" && sf.Program != "" || + sf.Path != "" && sf.Program != "" { + return errors.Errorf("Filter Path, Program and URL are mutually exclusive") + } + + // read the program from a file + if sf.Path != "" { + b, err := ioutil.ReadFile(sf.Path) + if err != nil { + return err + } + sf.Program = string(b) + } + + // read the program from a URL + if sf.URL != "" { + err := func() error { + resp, err := http.Get(sf.URL) + if err != nil { + return err + } + defer resp.Body.Close() + b, err := ioutil.ReadAll(resp.Body) + if err != nil { + return err + } + sf.Program = string(b) + return nil + }() + if err != nil { + return err + } + } + + return nil +} + +func (sf *Filter) Run(reader io.Reader, writer io.Writer) error { + // retain map of inputs to outputs by id so if the name is changed by the + // starlark program, we are able to match the same resources + value, err := sf.readResourceList(reader) + if err != nil { + return errors.Wrap(err) + } + + // run the starlark as program as transformation function + thread := &starlark.Thread{Name: sf.Name} + + ctx := &Context{resourceList: value} + pd, err := ctx.predeclared() + if err != nil { + return errors.Wrap(err) + } + _, err = starlark.ExecFile(thread, sf.Name, sf.Program, pd) + if err != nil { + return errors.Wrap(err) + } + + return sf.writeResourceList(value, writer) +} + +// inputToResourceList transforms input into a starlark.Value +func (sf *Filter) readResourceList(reader io.Reader) (starlark.Value, error) { + // read and parse the inputs + rl := bytes.Buffer{} + _, err := rl.ReadFrom(reader) + if err != nil { + return nil, errors.Wrap(err) + } + rn, err := yaml.Parse(rl.String()) + if err != nil { + return nil, errors.Wrap(err) + } + + // set the id on each node to map inputs to outputs + var id int + sf.ids = map[string]*yaml.RNode{} + items, err := rn.Pipe(yaml.Lookup("items")) + if err != nil { + return nil, errors.Wrap(err) + } + err = items.VisitElements(func(node *yaml.RNode) error { + id++ + idStr := fmt.Sprintf("%v", id) + sf.ids[idStr] = node + return node.PipeE(yaml.SetAnnotation("config.k8s.io/id", idStr)) + }) + if err != nil { + return nil, errors.Wrap(err) + } + + // convert to a starlark value + b, err := yaml.Marshal(rn.Document()) // convert to bytes + if err != nil { + return nil, errors.Wrap(err) + } + var in map[string]interface{} + err = yaml.Unmarshal(b, &in) // convert to map[string]interface{} + if err != nil { + return nil, errors.Wrap(err) + } + return util.Marshal(in) // convert to starlark value +} + +// resourceListToOutput converts the output of the starlark program to the filter output +func (sf *Filter) writeResourceList(value starlark.Value, writer io.Writer) error { + // convert the modified resourceList back into a slice of RNodes + // by first converting to a map[string]interface{} + out, err := util.Unmarshal(value) + if err != nil { + return errors.Wrap(err) + } + b, err := yaml.Marshal(out) + if err != nil { + return errors.Wrap(err) + } + + rl, err := yaml.Parse(string(b)) + if err != nil { + return errors.Wrap(err) + } + + // preserve the comments from the input + items, err := rl.Pipe(yaml.Lookup("items")) + if err != nil { + return errors.Wrap(err) + } + err = items.VisitElements(func(node *yaml.RNode) error { + anID, err := node.Pipe(yaml.GetAnnotation("config.k8s.io/id")) + if err != nil { + return errors.Wrap(err) + } + if anID == nil { + return nil + } + + var in *yaml.RNode + var found bool + if in, found = sf.ids[anID.YNode().Value]; !found { + return nil + } + if err := node.PipeE(yaml.ClearAnnotation("config.k8s.io/id")); err != nil { + return errors.Wrap(err) + } + if err := comments.CopyComments(in, node); err != nil { + return errors.Wrap(err) + } + + // starlark will serialize the resources sorting the fields alphabetically, + // format them to have a better ordering + fmtFltr := filters.FormatFilter{} + if _, err := fmtFltr.Filter([]*yaml.RNode{node}); err != nil { + return errors.Wrap(err) + } + return nil + }) + if err != nil { + return errors.Wrap(err) + } + + s, err := rl.String() + if err != nil { + return errors.Wrap(err) + } + + _, err = writer.Write([]byte(s)) + return err +} diff --git a/kyaml/starlark/starlark_test.go b/kyaml/fn/runtime/starlark/starlark_test.go similarity index 91% rename from kyaml/starlark/starlark_test.go rename to kyaml/fn/runtime/starlark/starlark_test.go index 3b0f59805..8e2ec4eae 100644 --- a/kyaml/starlark/starlark_test.go +++ b/kyaml/fn/runtime/starlark/starlark_test.go @@ -56,6 +56,7 @@ metadata: name: nginx-deployment annotations: foo: bar + config.kubernetes.io/path: 'deployment_nginx-deployment.yaml' spec: template: spec: @@ -95,6 +96,7 @@ metadata: name: nginx-deployment annotations: foo: annotation-value + config.kubernetes.io/path: 'deployment_nginx-deployment.yaml' spec: template: spec: @@ -133,6 +135,7 @@ metadata: name: nginx-deployment annotations: foo: Deployment enables declarative updates for Pods and ReplicaSets. + config.kubernetes.io/path: 'deployment_nginx-deployment.yaml' spec: template: spec: @@ -174,6 +177,7 @@ metadata: name: nginx-deployment annotations: foo: bar + config.kubernetes.io/path: 'deployment_nginx-deployment.yaml' spec: template: spec: @@ -213,6 +217,8 @@ apiVersion: apps/v1 kind: Deployment metadata: name: nginx-deployment + annotations: + config.kubernetes.io/path: 'deployment_nginx-deployment.yaml' spec: template: spec: @@ -266,6 +272,7 @@ metadata: name: nginx-deployment-1 annotations: foo: bar + config.kubernetes.io/path: 'deployment_nginx-deployment-1.yaml' spec: template: spec: @@ -280,6 +287,7 @@ metadata: name: nginx-deployment-2 annotations: foo: bar + config.kubernetes.io/path: 'deployment_nginx-deployment-2.yaml' spec: template: spec: @@ -318,13 +326,10 @@ run(ctx.resource_list["items"]) expected: ` apiVersion: apps/v1 kind: Deployment -metadata: - name: nginx-deployment-2 ---- -apiVersion: apps/v1 -kind: Deployment metadata: name: nginx-deployment-1 + annotations: + config.kubernetes.io/path: 'deployment_nginx-deployment-1.yaml' spec: template: spec: @@ -332,6 +337,13 @@ spec: - name: nginx # head comment image: nginx:1.8.1 # {"$ref": "#/definitions/io.k8s.cli.substitutions.image"} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nginx-deployment-2 + annotations: + config.kubernetes.io/path: 'deployment_nginx-deployment-2.yaml' `, }, { @@ -357,6 +369,8 @@ apiVersion: apps/v1 kind: Deployment metadata: name: nginx-deployment-1 + annotations: + config.kubernetes.io/path: 'deployment_nginx-deployment-1.yaml' `, }, { @@ -395,6 +409,7 @@ metadata: name: nginx-deployment annotations: foo: hello world + config.kubernetes.io/path: 'deployment_nginx-deployment.yaml' spec: template: spec: @@ -406,7 +421,7 @@ spec: expectedFunctionConfig: ` kind: Script spec: - value: hello world + value: "hello world" `, }, @@ -447,6 +462,7 @@ metadata: name: nginx-deployment annotations: foo: hello world + config.kubernetes.io/path: 'deployment_nginx-deployment.yaml' spec: template: spec: @@ -458,7 +474,7 @@ spec: expectedFunctionConfig: ` kind: Script spec: - value: updated + value: "hello world" `, }, } diff --git a/kyaml/runfn/runfn.go b/kyaml/runfn/runfn.go index 436ced6c7..4f5ed7f91 100644 --- a/kyaml/runfn/runfn.go +++ b/kyaml/runfn/runfn.go @@ -16,9 +16,9 @@ import ( "sigs.k8s.io/kustomize/kyaml/errors" "sigs.k8s.io/kustomize/kyaml/fn/runtime/container" "sigs.k8s.io/kustomize/kyaml/fn/runtime/runtimeutil" + "sigs.k8s.io/kustomize/kyaml/fn/runtime/starlark" "sigs.k8s.io/kustomize/kyaml/kio" "sigs.k8s.io/kustomize/kyaml/kio/kioutil" - "sigs.k8s.io/kustomize/kyaml/starlark" "sigs.k8s.io/kustomize/kyaml/yaml" ) @@ -375,7 +375,7 @@ func (r *RunFns) ffp(spec runtimeutil.FunctionSpec, api *yaml.RNode) (kio.Filter return &starlark.Filter{ Name: spec.Starlark.Name, Path: p, - FunctionConfig: api, + FunctionFilter: runtimeutil.FunctionFilter{FunctionConfig: api}, }, nil } return nil, nil diff --git a/kyaml/starlark/starlark.go b/kyaml/starlark/starlark.go deleted file mode 100644 index e673a754f..000000000 --- a/kyaml/starlark/starlark.go +++ /dev/null @@ -1,251 +0,0 @@ -// Copyright 2019 The Kubernetes Authors. -// SPDX-License-Identifier: Apache-2.0 - -package starlark - -import ( - "fmt" - "io/ioutil" - "net/http" - "strconv" - - "github.com/qri-io/starlib/util" - "go.starlark.net/starlark" - "sigs.k8s.io/kustomize/kyaml/comments" - "sigs.k8s.io/kustomize/kyaml/errors" - "sigs.k8s.io/kustomize/kyaml/kio/filters" - "sigs.k8s.io/kustomize/kyaml/yaml" -) - -// Filter transforms a set of resources through the provided program -type Filter struct { - Name string - - // Program is a starlark script which will be run against the resources - Program string - - // URL is the url of a starlark program to fetch and run - URL string - - // Path is the path to a starlark program to read and run - Path string - - // FunctionConfig is the value to be provided for resourceList.functionConfig as specified by - // https://github.com/kubernetes-sigs/kustomize/blob/master/cmd/config/docs/api-conventions/functions-spec.md. - FunctionConfig *yaml.RNode -} - -func (sf *Filter) String() string { - return fmt.Sprintf("name: %v path: %v url: %v program: %v", sf.Name, sf.Path, sf.URL, sf.Program) -} - -func (sf *Filter) Filter(input []*yaml.RNode) ([]*yaml.RNode, error) { - if sf.URL != "" && sf.Path != "" || - sf.URL != "" && sf.Program != "" || - sf.Path != "" && sf.Program != "" { - return nil, errors.Errorf("Filter Path, Program and URL are mutually exclusive") - } - - // read the program from a file - if sf.Path != "" { - b, err := ioutil.ReadFile(sf.Path) - if err != nil { - return nil, err - } - sf.Program = string(b) - } - - // read the program from a URL - if sf.URL != "" { - err := func() error { - resp, err := http.Get(sf.URL) - if err != nil { - return err - } - defer resp.Body.Close() - b, err := ioutil.ReadAll(resp.Body) - if err != nil { - return err - } - sf.Program = string(b) - return nil - }() - if err != nil { - return nil, err - } - } - - // retain map of inputs to outputs by id so if the name is changed by the - // starlark program, we are able to match the same resources - value, ids, err := sf.inputToResourceList(input) - if err != nil { - return nil, errors.Wrap(err) - } - - // run the starlark as program as transformation function - thread := &starlark.Thread{Name: sf.Name} - - ctx := &Context{ - resourceList: value, - } - pd, err := ctx.predeclared() - if err != nil { - return nil, errors.Wrap(err) - } - _, err = starlark.ExecFile(thread, sf.Name, sf.Program, pd) - if err != nil { - return nil, errors.Wrap(err) - } - - results, err := sf.resourceListToOutput(value, ids) - if err != nil { - return nil, errors.Wrap(err) - } - - // starlark will serialize the resources sorting the fields alphabetically, - // format them to have a better ordering - return filters.FormatFilter{}.Filter(results) -} - -// tuple maps an input resource to the output resource -type tuple struct { - // in is the RNode provided to the starlark program - in *yaml.RNode - // out is the RNode emitted by the starlark program with the id matching in - out *yaml.RNode -} - -// inputToResourceList transforms input into a starlark.Value -func (sf *Filter) inputToResourceList( - input []*yaml.RNode) (starlark.Value, map[int]*tuple, error) { - var id int - ids := map[int]*tuple{} - - // convert into a ResourceList which will be converted to a starlark dictionary - // create the ResourceList - resourceList, err := yaml.Parse(`kind: ResourceList`) - if err != nil { - return nil, nil, errors.Wrap(err) - } - - // set the functionConfig - if sf.FunctionConfig != nil { - if err := resourceList.PipeE( - yaml.FieldSetter{Name: "functionConfig", Value: sf.FunctionConfig}); err != nil { - return nil, nil, err - } - } - - // the inputs should be provided as the list "items" - items, err := resourceList.Pipe(yaml.LookupCreate(yaml.SequenceNode, "items")) - if err != nil { - return nil, nil, errors.Wrap(err) - } - // add the input as items, give each resource an id - for i := range input { - item := input[i] - - // create an id for tracking the resource through the program - err := item.PipeE(yaml.SetAnnotation("config.k8s.io/id", fmt.Sprintf("%d", id))) - if err != nil { - return nil, nil, errors.Wrap(err) - } - ids[id] = &tuple{in: item} - id++ - - items.YNode().Content = append(items.YNode().Content, item.YNode()) - } - - // convert the ResourceList into a starlark dictionary by - // first converting it into a map[string]interface{} - value, err := nodeToValue(resourceList) - return value, ids, err -} - -// resourceListToOutput converts the output of the starlark program to the filter output -func (sf *Filter) resourceListToOutput( - value starlark.Value, ids map[int]*tuple) ([]*yaml.RNode, error) { - // convert the modified resourceList back into a slice of RNodes - // by first converting to a map[string]interface{} - out, err := util.Unmarshal(value) - if err != nil { - return nil, errors.Wrap(err) - } - o := out.(map[string]interface{}) - - // parse the function config - if _, found := o["functionConfig"]; found { - fc := (o["functionConfig"].(map[string]interface{})) - b, err := yaml.Marshal(fc) - if err != nil { - return nil, errors.Wrap(err) - } - sf.FunctionConfig, err = yaml.Parse(string(b)) - if err != nil { - return nil, errors.Wrap(err) - } - } - - // parse the items - // copy the items out of the ResourceList, and into the Filter output - var results []*yaml.RNode - it := (o["items"].([]interface{})) - for i := range it { - // convert the resource back to the native yaml form - b, err := yaml.Marshal(it[i]) - if err != nil { - return nil, errors.Wrap(err) - } - node, err := yaml.Parse(string(b)) - if err != nil { - return nil, errors.Wrap(err) - } - - // match it to an input - idS, err := node.Pipe(yaml.GetAnnotation("config.k8s.io/id")) - if err != nil { - return nil, errors.Wrap(err) - } - if idS == nil { - // no matching input -- new resource - results = append(results, node) - continue - } - - id, err := strconv.Atoi(idS.YNode().Value) - if err != nil { - return nil, errors.Wrap(err) - } - if match, found := ids[id]; found { - // matching resources - match.out = node - } else { - // no matching input with the same id -- new resource - // this may be an error case, the outputs probably shouldn't have ids - // assigned by the starlark program - results = append(results, node) - } - } - - // retain the comments instead of dropping them by copying them from the original inputs - for i := 0; i < len(ids); i++ { - v := ids[i] - if v.out == nil { - continue - } - if err := comments.CopyComments(v.in, v.out); err != nil { - return nil, errors.Wrap(err) - } - results = append(results, v.out) - } - - // delete the ids from resources, these were only to track through the starlark program - // and that is finished. - for i := range results { - err := results[i].PipeE(yaml.ClearAnnotation("config.k8s.io/id")) - if err != nil { - return nil, err - } - } - return results, nil -}