From 94a55210e127d99b13578b6a9c6795d3cd544044 Mon Sep 17 00:00:00 2001 From: Donny Xia Date: Thu, 25 Jun 2020 15:26:21 -0700 Subject: [PATCH 1/6] skeleton for kustomize function --- api/internal/plugins/fnplugin/fnplugin.go | 91 +++++++++++++++++++++++ api/internal/plugins/loader/loader.go | 1 + 2 files changed, 92 insertions(+) create mode 100644 api/internal/plugins/fnplugin/fnplugin.go diff --git a/api/internal/plugins/fnplugin/fnplugin.go b/api/internal/plugins/fnplugin/fnplugin.go new file mode 100644 index 000000000..32cd8c3f4 --- /dev/null +++ b/api/internal/plugins/fnplugin/fnplugin.go @@ -0,0 +1,91 @@ +package fnplugin + +import ( + "sigs.k8s.io/kustomize/api/resmap" + "sigs.k8s.io/kustomize/api/resource" + "sigs.k8s.io/kustomize/kyaml/fn/runtime/runtimeutil" + "sigs.k8s.io/kustomize/kyaml/runfn" + "sigs.k8s.io/kustomize/kyaml/yaml" +) + +// FnPlugin holds the information about thie function +type FnPlugin struct { + // Plugin configuration data. + cfg []byte + + // PluginHelpers + h *resmap.PluginHelpers + + // Function runner + RunFns runfn.RunFns +} + +func bytesToRNode(yml []byte) (*yaml.RNode, error) { + rnode, err := yaml.Parse(string(yml)) + if err != nil { + return nil, err + } + return rnode, nil +} + +func resourceToRNode(res *resource.Resource) (*yaml.RNode, error) { + yml, err := res.AsYAML() + if err != nil { + return nil, err + } + + return bytesToRNode(yml) +} + +func getFunctionSpec(res *resource.Resource) *runtimeutil.FunctionSpec { + rnode, err := resourceToRNode(res) + if err != nil { + return nil + } + + return runtimeutil.GetFunctionSpec(rnode) +} + +// IsFnPlugin returns a resource is a function plugin spec or not +func IsFnPlugin(res *resource.Resource) bool { + return getFunctionSpec(res) != nil +} + +// NewFnPlugin returns a FnPlugin struct +func NewFnPlugin(res *resource.Resource) *FnPlugin { + return &FnPlugin{} +} + +// Config accepts the plugin helper and plugin config +func (f *FnPlugin) Config(h *resmap.PluginHelpers, config []byte) error { + f.h = h + f.cfg = config + // config is the content of the config file for the functions. + // If there are multiple functions in on config file, they will + // be passed in one by one. + fn, err := bytesToRNode(config) + if err != nil { + return err + } + + f.RunFns.Functions = append(f.RunFns.Functions, fn) + + return nil +} + +// Transform does the transformation when the plugin is a transformer +func (f *FnPlugin) Transform(rm resmap.ResMap) error { + // convert input to ResourceList + // add functionConfig + // invoke function + // convert back to ResMap + return nil +} + +// invoke call the actual function and send the input to it. It captures +// and returns the output +func (f *FnPlugin) invoke(input []byte) ([]byte, error) { + // setup input and output + var output []byte + return output, nil +} diff --git a/api/internal/plugins/loader/loader.go b/api/internal/plugins/loader/loader.go index c61f264c4..43f4b2cbb 100644 --- a/api/internal/plugins/loader/loader.go +++ b/api/internal/plugins/loader/loader.go @@ -16,6 +16,7 @@ import ( "sigs.k8s.io/kustomize/api/ifc" "sigs.k8s.io/kustomize/api/internal/plugins/builtinhelpers" "sigs.k8s.io/kustomize/api/internal/plugins/execplugin" + "sigs.k8s.io/kustomize/api/internal/plugins/fnplugin" "sigs.k8s.io/kustomize/api/internal/plugins/utils" "sigs.k8s.io/kustomize/api/konfig" "sigs.k8s.io/kustomize/api/resid" From 8d543d8483cbe481d2f07dd0c03e29d07dc1132f Mon Sep 17 00:00:00 2001 From: Donny Xia Date: Mon, 29 Jun 2020 11:12:20 -0700 Subject: [PATCH 2/6] Refactor --- api/internal/plugins/execplugin/execplugin.go | 7 +- api/internal/plugins/execplugin/fnplugin.go | 219 ------------------ api/internal/plugins/execplugin/utils.go | 108 --------- api/internal/plugins/execplugin/utils_test.go | 111 --------- api/internal/plugins/fnplugin/fnplugin.go | 190 ++++++++++++--- api/internal/plugins/loader/loader.go | 6 +- api/internal/plugins/utils/utils.go | 100 ++++++++ api/internal/plugins/utils/utils_test.go | 101 ++++++++ 8 files changed, 364 insertions(+), 478 deletions(-) delete mode 100644 api/internal/plugins/execplugin/fnplugin.go delete mode 100644 api/internal/plugins/execplugin/utils.go delete mode 100644 api/internal/plugins/execplugin/utils_test.go diff --git a/api/internal/plugins/execplugin/execplugin.go b/api/internal/plugins/execplugin/execplugin.go index d4c7d0069..22b379eb5 100644 --- a/api/internal/plugins/execplugin/execplugin.go +++ b/api/internal/plugins/execplugin/execplugin.go @@ -14,6 +14,7 @@ import ( "github.com/google/shlex" "github.com/pkg/errors" + "sigs.k8s.io/kustomize/api/internal/plugins/utils" "sigs.k8s.io/kustomize/api/resmap" "sigs.k8s.io/yaml" ) @@ -108,12 +109,12 @@ func (p *ExecPlugin) Generate() (resmap.ResMap, error) { if err != nil { return nil, err } - return UpdateResourceOptions(rm) + return utils.UpdateResourceOptions(rm) } func (p *ExecPlugin) Transform(rm resmap.ResMap) error { // add ResIds as annotations to all objects so that we can add them back - inputRM, err := getResMapWithIdAnnotation(rm) + inputRM, err := utils.GetResMapWithIDAnnotation(rm) if err != nil { return err } @@ -131,7 +132,7 @@ func (p *ExecPlugin) Transform(rm resmap.ResMap) error { } // update the original ResMap based on the output - return updateResMapValues(p.path, p.h, output, rm) + return utils.UpdateResMapValues(p.path, p.h, output, rm) } // invokePlugin writes plugin config to a temp file, then diff --git a/api/internal/plugins/execplugin/fnplugin.go b/api/internal/plugins/execplugin/fnplugin.go deleted file mode 100644 index fa56db953..000000000 --- a/api/internal/plugins/execplugin/fnplugin.go +++ /dev/null @@ -1,219 +0,0 @@ -// Copyright 2019 The Kubernetes Authors. -// SPDX-License-Identifier: Apache-2.0 - -package execplugin - -import ( - "bytes" - "fmt" - - "github.com/pkg/errors" - - "sigs.k8s.io/kustomize/api/resmap" - "sigs.k8s.io/kustomize/api/resource" - "sigs.k8s.io/kustomize/api/types" - - "sigs.k8s.io/kustomize/kyaml/kio" - "sigs.k8s.io/kustomize/kyaml/yaml" - - "sigs.k8s.io/kustomize/kyaml/fn/runtime/runtimeutil" - "sigs.k8s.io/kustomize/kyaml/runfn" -) - -type FnPlugin struct { - // Function runner - runFns runfn.RunFns - - // Plugin configuration data. - cfg []byte - - // Plugin name cache for error output - pluginName string - - // PluginHelpers - h *resmap.PluginHelpers -} - -func bytesToRNode(yml []byte) (*yaml.RNode, error) { - rnode, err := yaml.Parse(string(yml)) - if err != nil { - return nil, err - } - return rnode, nil -} - -func resourceToRNode(res *resource.Resource) (*yaml.RNode, error) { - yml, err := res.AsYAML() - if err != nil { - return nil, err - } - - return bytesToRNode(yml) -} - -func GetFunctionSpec(res *resource.Resource) (*runtimeutil.FunctionSpec, error) { - rnode, err := resourceToRNode(res) - if err != nil { - return nil, err - } - - fSpec := runtimeutil.GetFunctionSpec(rnode) - if fSpec == nil { - return nil, fmt.Errorf("resource %v doesn't contain function spec", res.GetGvk()) - } - - return fSpec, nil -} - -func toStorageMounts(mounts []string) []runtimeutil.StorageMount { - var sms []runtimeutil.StorageMount - for _, mount := range mounts { - sms = append(sms, runtimeutil.StringToStorageMount(mount)) - } - return sms -} - -func NewFnPlugin(o *types.FnPluginLoadingOptions) *FnPlugin { - //log.Printf("options: %v\n", o) - return &FnPlugin{ - runFns: runfn.RunFns{ - Functions: []*yaml.RNode{}, - Network: o.Network, - NetworkName: o.NetworkName, - EnableStarlark: o.EnableStar, - EnableExec: o.EnableExec, - StorageMounts: toStorageMounts(o.Mounts), - }, - } -} - -func (p *FnPlugin) Cfg() []byte { - return p.cfg -} - -func (p *FnPlugin) Config(h *resmap.PluginHelpers, config []byte) error { - p.h = h - p.cfg = config - - rnode, err := bytesToRNode(p.cfg) - if err != nil { - return err - } - - meta, err := rnode.GetMeta() - if err != nil { - return err - } - - p.pluginName = fmt.Sprintf("api: %s, kind: %s, name: %s", - meta.APIVersion, meta.Kind, meta.Name) - //log.Printf("config based pluginName: %s", p.pluginName) - - return nil -} - -func (p *FnPlugin) Generate() (resmap.ResMap, error) { - output, err := p.invokePlugin(nil) - if err != nil { - return nil, err - } - rm, err := p.h.ResmapFactory().NewResMapFromBytes(output) - if err != nil { - return nil, err - } - return UpdateResourceOptions(rm) -} - -func (p *FnPlugin) Transform(rm resmap.ResMap) error { - // add ResIds as annotations to all objects so that we can add them back - inputRM, err := getResMapWithIdAnnotation(rm) - if err != nil { - return err - } - - // encode the ResMap so it can be fed to the plugin - resources, err := inputRM.AsYaml() - if err != nil { - return err - } - - // invoke the plugin with resources as the input - output, err := p.invokePlugin(resources) - if err != nil { - return fmt.Errorf("%v %s", err, string(output)) - } - - // update the original ResMap based on the output - return updateResMapValues(p.pluginName, p.h, output, rm) -} - -// invokePlugin uses Function runner to run function as plugin -func (p *FnPlugin) invokePlugin(input []byte) ([]byte, error) { - // get config rnode - rnode, err := bytesToRNode(p.cfg) - if err != nil { - return nil, err - } - err = rnode.PipeE(yaml.SetAnnotation("config.kubernetes.io/local-config", "true")) - if err != nil { - return nil, err - } - - // we need to add config as input for generators. Some of them don't work with FunctionConfig - // and in addition kio.Pipeline won't create anything if there are no objects - // see https://github.com/kubernetes-sigs/kustomize/blob/master/kyaml/kio/kio.go#L93 - if input == nil { - yaml, err := rnode.String() - if err != nil { - return nil, err - } - input = []byte(yaml) - } - - // Transform to ResourceList - var inOut bytes.Buffer - inIn := bytes.NewReader(input) - err = kio.Pipeline{ - Inputs: []kio.Reader{&kio.ByteReader{Reader: inIn}}, - Outputs: []kio.Writer{kio.ByteWriter{ - Writer: &inOut, - WrappingKind: kio.ResourceListKind, - WrappingAPIVersion: kio.ResourceListAPIVersion}}, - }.Execute() - if err != nil { - return nil, errors.Wrap( - err, "couldn't transform to ResourceList") - } - //log.Printf("converted to:\n%s\n", inOut.String()) - - // Configure and Execute Fn - var runFnsOut bytes.Buffer - p.runFns.Input = bytes.NewReader(inOut.Bytes()) - p.runFns.Functions = append(p.runFns.Functions, rnode) - p.runFns.Output = &runFnsOut - - err = p.runFns.Execute() - if err != nil { - return nil, errors.Wrap( - err, "couldn't execute function") - } - - //log.Printf("fn returned:\n%s\n", runFnsOut.String()) - - // Convert back to a single multi-yaml doc - var outOut bytes.Buffer - outIn := bytes.NewReader(runFnsOut.Bytes()) - - err = kio.Pipeline{ - Inputs: []kio.Reader{&kio.ByteReader{Reader: outIn}}, - Outputs: []kio.Writer{kio.ByteWriter{Writer: &outOut}}, - }.Execute() - if err != nil { - return nil, errors.Wrap( - err, "couldn't transform from ResourceList") - } - - //log.Printf("converted back to:\n%s\n", outOut.String()) - - return outOut.Bytes(), nil -} diff --git a/api/internal/plugins/execplugin/utils.go b/api/internal/plugins/execplugin/utils.go deleted file mode 100644 index 4c7ac8e66..000000000 --- a/api/internal/plugins/execplugin/utils.go +++ /dev/null @@ -1,108 +0,0 @@ -// Copyright 2019 The Kubernetes Authors. -// SPDX-License-Identifier: Apache-2.0 - -package execplugin - -import ( - "fmt" - "strconv" - - "sigs.k8s.io/kustomize/api/resid" - "sigs.k8s.io/kustomize/api/resmap" - "sigs.k8s.io/kustomize/api/types" - "sigs.k8s.io/yaml" -) - -const ( - idAnnotation = "kustomize.config.k8s.io/id" - HashAnnotation = "kustomize.config.k8s.io/needs-hash" - BehaviorAnnotation = "kustomize.config.k8s.io/behavior" -) - -// Returns a new copy of the given ResMap with the ResIds annotated in each Resource -func getResMapWithIdAnnotation(rm resmap.ResMap) (resmap.ResMap, error) { - inputRM := rm.DeepCopy() - for _, r := range inputRM.Resources() { - idString, err := yaml.Marshal(r.CurId()) - if err != nil { - return nil, err - } - annotations := r.GetAnnotations() - if annotations == nil { - annotations = make(map[string]string) - } - annotations[idAnnotation] = string(idString) - r.SetAnnotations(annotations) - } - return inputRM, nil -} - -// updateResMapValues updates the Resource value in the given ResMap -// with the emitted Resource values in output. -func updateResMapValues(pluginName string, h *resmap.PluginHelpers, output []byte, rm resmap.ResMap) error { - outputRM, err := h.ResmapFactory().NewResMapFromBytes(output) - if err != nil { - return err - } - for _, r := range outputRM.Resources() { - // for each emitted Resource, find the matching Resource in the original ResMap - // using its id - annotations := r.GetAnnotations() - idString, ok := annotations[idAnnotation] - if !ok { - return fmt.Errorf("the transformer %s should not remove annotation %s", - pluginName, idAnnotation) - } - id := resid.ResId{} - err := yaml.Unmarshal([]byte(idString), &id) - if err != nil { - return err - } - res, err := rm.GetByCurrentId(id) - if err != nil { - return fmt.Errorf("unable to find unique match to %s", id.String()) - } - // remove the annotation set by Kustomize to track the resource - delete(annotations, idAnnotation) - if len(annotations) == 0 { - annotations = nil - } - r.SetAnnotations(annotations) - - // update the resource value with the transformed object - res.ResetPrimaryData(r) - } - return nil -} - -// updateResourceOptions updates the generator options for each resource in the -// given ResMap based on plugin provided annotations. -func UpdateResourceOptions(rm resmap.ResMap) (resmap.ResMap, error) { - for _, r := range rm.Resources() { - // Disable name hashing by default and require plugin to explicitly - // request it for each resource. - annotations := r.GetAnnotations() - behavior := annotations[BehaviorAnnotation] - var needsHash bool - if val, ok := annotations[HashAnnotation]; ok { - b, err := strconv.ParseBool(val) - if err != nil { - return nil, fmt.Errorf( - "the annotation %q contains an invalid value (%q)", - HashAnnotation, val) - } - needsHash = b - } - delete(annotations, HashAnnotation) - delete(annotations, BehaviorAnnotation) - if len(annotations) == 0 { - annotations = nil - } - r.SetAnnotations(annotations) - r.SetOptions(types.NewGenArgs( - &types.GeneratorArgs{ - Behavior: behavior, - Options: &types.GeneratorOptions{DisableNameSuffixHash: !needsHash}})) - } - return rm, nil -} diff --git a/api/internal/plugins/execplugin/utils_test.go b/api/internal/plugins/execplugin/utils_test.go deleted file mode 100644 index 93f9cb754..000000000 --- a/api/internal/plugins/execplugin/utils_test.go +++ /dev/null @@ -1,111 +0,0 @@ -// Copyright 2019 The Kubernetes Authors. -// SPDX-License-Identifier: Apache-2.0 - -package execplugin_test - -import ( - "fmt" - "testing" - - . "sigs.k8s.io/kustomize/api/internal/plugins/execplugin" - "sigs.k8s.io/kustomize/api/k8sdeps/kunstruct" - "sigs.k8s.io/kustomize/api/resmap" - "sigs.k8s.io/kustomize/api/resource" - "sigs.k8s.io/kustomize/api/types" -) - -func makeConfigMap(rf *resource.Factory, name, behavior string, hashValue *string) *resource.Resource { - r := rf.FromMap(map[string]interface{}{ - "apiVersion": "v1", - "kind": "ConfigMap", - "metadata": map[string]interface{}{"name": name}, - }) - annotations := map[string]string{} - if behavior != "" { - annotations[BehaviorAnnotation] = behavior - } - if hashValue != nil { - annotations[HashAnnotation] = *hashValue - } - if len(annotations) > 0 { - r.SetAnnotations(annotations) - } - return r -} - -func makeConfigMapOptions(rf *resource.Factory, name, behavior string, disableHash bool) *resource.Resource { - return rf.FromMapAndOption(map[string]interface{}{ - "apiVersion": "v1", - "kind": "ConfigMap", - "metadata": map[string]interface{}{"name": name}, - }, &types.GeneratorArgs{ - Behavior: behavior, - Options: &types.GeneratorOptions{DisableNameSuffixHash: disableHash}}) -} - -func strptr(s string) *string { - return &s -} - -func TestUpdateResourceOptions(t *testing.T) { - rf := resource.NewFactory(kunstruct.NewKunstructuredFactoryImpl()) - in := resmap.New() - expected := resmap.New() - cases := []struct { - behavior string - needsHash bool - hashValue *string - }{ - {hashValue: strptr("false")}, - {hashValue: strptr("true"), needsHash: true}, - {behavior: "replace"}, - {behavior: "merge"}, - {behavior: "create"}, - {behavior: "nonsense"}, - {behavior: "merge", hashValue: strptr("false")}, - {behavior: "merge", hashValue: strptr("true"), needsHash: true}, - } - for i, c := range cases { - name := fmt.Sprintf("test%d", i) - in.Append(makeConfigMap(rf, name, c.behavior, c.hashValue)) - expected.Append(makeConfigMapOptions(rf, name, c.behavior, !c.needsHash)) - } - actual, err := UpdateResourceOptions(in) - if err != nil { - t.Fatalf("unexpected error: %v", err.Error()) - } - for i, a := range expected.Resources() { - b := actual.GetByIndex(i) - if b == nil { - t.Fatalf("resource %d missing from processed map", i) - } - if !a.Equals(b) { - t.Errorf("expected %v got %v", a, b) - } - if a.NeedHashSuffix() != b.NeedHashSuffix() { - t.Errorf("") - } - if a.Behavior() != b.Behavior() { - t.Errorf("expected %v got %v", a.Behavior(), b.Behavior()) - } - } -} - -func TestUpdateResourceOptionsWithInvalidHashAnnotationValues(t *testing.T) { - rf := resource.NewFactory(kunstruct.NewKunstructuredFactoryImpl()) - cases := []string{ - "", - "FaLsE", - "TrUe", - "potato", - } - for i, c := range cases { - name := fmt.Sprintf("test%d", i) - in := resmap.New() - in.Append(makeConfigMap(rf, name, "", &c)) - _, err := UpdateResourceOptions(in) - if err == nil { - t.Errorf("expected error from value %q", c) - } - } -} diff --git a/api/internal/plugins/fnplugin/fnplugin.go b/api/internal/plugins/fnplugin/fnplugin.go index 32cd8c3f4..97f03d9a8 100644 --- a/api/internal/plugins/fnplugin/fnplugin.go +++ b/api/internal/plugins/fnplugin/fnplugin.go @@ -1,23 +1,38 @@ +// Copyright 2019 The Kubernetes Authors. +// SPDX-License-Identifier: Apache-2.0 + package fnplugin import ( + "bytes" + "fmt" + + "github.com/pkg/errors" + "sigs.k8s.io/kustomize/api/resmap" "sigs.k8s.io/kustomize/api/resource" + "sigs.k8s.io/kustomize/api/types" + + "sigs.k8s.io/kustomize/kyaml/kio" + "sigs.k8s.io/kustomize/kyaml/yaml" + + "sigs.k8s.io/kustomize/api/internal/plugins/utils" "sigs.k8s.io/kustomize/kyaml/fn/runtime/runtimeutil" "sigs.k8s.io/kustomize/kyaml/runfn" - "sigs.k8s.io/kustomize/kyaml/yaml" ) -// FnPlugin holds the information about thie function type FnPlugin struct { + // Function runner + runFns runfn.RunFns + // Plugin configuration data. cfg []byte + // Plugin name cache for error output + pluginName string + // PluginHelpers h *resmap.PluginHelpers - - // Function runner - RunFns runfn.RunFns } func bytesToRNode(yml []byte) (*yaml.RNode, error) { @@ -37,7 +52,7 @@ func resourceToRNode(res *resource.Resource) (*yaml.RNode, error) { return bytesToRNode(yml) } -func getFunctionSpec(res *resource.Resource) *runtimeutil.FunctionSpec { +func GetFunctionSpec(res *resource.Resource) *runtimeutil.FunctionSpec { rnode, err := resourceToRNode(res) if err != nil { return nil @@ -46,46 +61,153 @@ func getFunctionSpec(res *resource.Resource) *runtimeutil.FunctionSpec { return runtimeutil.GetFunctionSpec(rnode) } -// IsFnPlugin returns a resource is a function plugin spec or not -func IsFnPlugin(res *resource.Resource) bool { - return getFunctionSpec(res) != nil +func toStorageMounts(mounts []string) []runtimeutil.StorageMount { + var sms []runtimeutil.StorageMount + for _, mount := range mounts { + sms = append(sms, runtimeutil.StringToStorageMount(mount)) + } + return sms } -// NewFnPlugin returns a FnPlugin struct -func NewFnPlugin(res *resource.Resource) *FnPlugin { - return &FnPlugin{} +func NewFnPlugin(o *types.FnPluginLoadingOptions) *FnPlugin { + return &FnPlugin{ + runFns: runfn.RunFns{ + Functions: []*yaml.RNode{}, + Network: o.Network, + NetworkName: o.NetworkName, + EnableStarlark: o.EnableStar, + EnableExec: o.EnableExec, + StorageMounts: toStorageMounts(o.Mounts), + }, + } } -// Config accepts the plugin helper and plugin config -func (f *FnPlugin) Config(h *resmap.PluginHelpers, config []byte) error { - f.h = h - f.cfg = config - // config is the content of the config file for the functions. - // If there are multiple functions in on config file, they will - // be passed in one by one. - fn, err := bytesToRNode(config) +func (p *FnPlugin) Cfg() []byte { + return p.cfg +} + +func (p *FnPlugin) Config(h *resmap.PluginHelpers, config []byte) error { + p.h = h + p.cfg = config + + fn, err := bytesToRNode(p.cfg) if err != nil { return err } - f.RunFns.Functions = append(f.RunFns.Functions, fn) + meta, err := fn.GetMeta() + if err != nil { + return err + } + + p.pluginName = fmt.Sprintf("api: %s, kind: %s, name: %s", + meta.APIVersion, meta.Kind, meta.Name) return nil } -// Transform does the transformation when the plugin is a transformer -func (f *FnPlugin) Transform(rm resmap.ResMap) error { - // convert input to ResourceList - // add functionConfig - // invoke function - // convert back to ResMap - return nil +func (p *FnPlugin) Generate() (resmap.ResMap, error) { + output, err := p.invokePlugin(nil) + if err != nil { + return nil, err + } + rm, err := p.h.ResmapFactory().NewResMapFromBytes(output) + if err != nil { + return nil, err + } + return utils.UpdateResourceOptions(rm) } -// invoke call the actual function and send the input to it. It captures -// and returns the output -func (f *FnPlugin) invoke(input []byte) ([]byte, error) { - // setup input and output - var output []byte - return output, nil +func (p *FnPlugin) Transform(rm resmap.ResMap) error { + // add ResIds as annotations to all objects so that we can add them back + inputRM, err := utils.GetResMapWithIDAnnotation(rm) + if err != nil { + return err + } + + // encode the ResMap so it can be fed to the plugin + resources, err := inputRM.AsYaml() + if err != nil { + return err + } + + // invoke the plugin with resources as the input + output, err := p.invokePlugin(resources) + if err != nil { + return fmt.Errorf("%v %s", err, string(output)) + } + + // update the original ResMap based on the output + return utils.UpdateResMapValues(p.pluginName, p.h, output, rm) +} + +// invokePlugin uses Function runner to run function as plugin +func (p *FnPlugin) invokePlugin(input []byte) ([]byte, error) { + // get config rnode + rnode, err := bytesToRNode(p.cfg) + if err != nil { + return nil, err + } + err = rnode.PipeE(yaml.SetAnnotation("config.kubernetes.io/local-config", "true")) + if err != nil { + return nil, err + } + + // we need to add config as input for generators. Some of them don't work with FunctionConfig + // and in addition kio.Pipeline won't create anything if there are no objects + // see https://github.com/kubernetes-sigs/kustomize/blob/master/kyaml/kio/kio.go#L93 + if input == nil { + yaml, err := rnode.String() + if err != nil { + return nil, err + } + input = []byte(yaml) + } + + // Transform to ResourceList + var inOut bytes.Buffer + inIn := bytes.NewReader(input) + err = kio.Pipeline{ + Inputs: []kio.Reader{&kio.ByteReader{Reader: inIn}}, + Outputs: []kio.Writer{kio.ByteWriter{ + Writer: &inOut, + WrappingKind: kio.ResourceListKind, + WrappingAPIVersion: kio.ResourceListAPIVersion}}, + }.Execute() + if err != nil { + return nil, errors.Wrap( + err, "couldn't transform to ResourceList") + } + //log.Printf("converted to:\n%s\n", inOut.String()) + + // Configure and Execute Fn + var runFnsOut bytes.Buffer + p.runFns.Input = bytes.NewReader(inOut.Bytes()) + p.runFns.Functions = append(p.runFns.Functions, rnode) + p.runFns.Output = &runFnsOut + + err = p.runFns.Execute() + if err != nil { + return nil, errors.Wrap( + err, "couldn't execute function") + } + + //log.Printf("fn returned:\n%s\n", runFnsOut.String()) + + // Convert back to a single multi-yaml doc + var outOut bytes.Buffer + outIn := bytes.NewReader(runFnsOut.Bytes()) + + err = kio.Pipeline{ + Inputs: []kio.Reader{&kio.ByteReader{Reader: outIn}}, + Outputs: []kio.Writer{kio.ByteWriter{Writer: &outOut}}, + }.Execute() + if err != nil { + return nil, errors.Wrap( + err, "couldn't transform from ResourceList") + } + + //log.Printf("converted back to:\n%s\n", outOut.String()) + + return outOut.Bytes(), nil } diff --git a/api/internal/plugins/loader/loader.go b/api/internal/plugins/loader/loader.go index 43f4b2cbb..aaa7c3600 100644 --- a/api/internal/plugins/loader/loader.go +++ b/api/internal/plugins/loader/loader.go @@ -168,9 +168,9 @@ func (l *Loader) makeBuiltinPlugin(r resid.Gvk) (resmap.Configurable, error) { } func (l *Loader) loadPlugin(res *resource.Resource) (resmap.Configurable, error) { - _, err := execplugin.GetFunctionSpec(res) - if err == nil { - return execplugin.NewFnPlugin(&l.pc.FnpLoadingOptions), nil + spec := fnplugin.GetFunctionSpec(res) + if spec != nil { + return fnplugin.NewFnPlugin(&l.pc.FnpLoadingOptions), nil } return l.loadExecOrGoPlugin(res.OrgId()) } diff --git a/api/internal/plugins/utils/utils.go b/api/internal/plugins/utils/utils.go index af4976cea..46706627b 100644 --- a/api/internal/plugins/utils/utils.go +++ b/api/internal/plugins/utils/utils.go @@ -4,13 +4,25 @@ package utils import ( + "fmt" "os" "path/filepath" "runtime" + "strconv" "time" "sigs.k8s.io/kustomize/api/filesys" "sigs.k8s.io/kustomize/api/konfig" + "sigs.k8s.io/kustomize/api/resid" + "sigs.k8s.io/kustomize/api/resmap" + "sigs.k8s.io/kustomize/api/types" + "sigs.k8s.io/yaml" +) + +const ( + idAnnotation = "kustomize.config.k8s.io/id" + HashAnnotation = "kustomize.config.k8s.io/needs-hash" + BehaviorAnnotation = "kustomize.config.k8s.io/behavior" ) func GoBin() string { @@ -113,3 +125,91 @@ func FileExists(path string) bool { } return true } + +// GetResMapWithIDAnnotation returns a new copy of the given ResMap with the ResIds annotated in each Resource +func GetResMapWithIDAnnotation(rm resmap.ResMap) (resmap.ResMap, error) { + inputRM := rm.DeepCopy() + for _, r := range inputRM.Resources() { + idString, err := yaml.Marshal(r.CurId()) + if err != nil { + return nil, err + } + annotations := r.GetAnnotations() + if annotations == nil { + annotations = make(map[string]string) + } + annotations[idAnnotation] = string(idString) + r.SetAnnotations(annotations) + } + return inputRM, nil +} + +// UpdateResMapValues updates the Resource value in the given ResMap +// with the emitted Resource values in output. +func UpdateResMapValues(pluginName string, h *resmap.PluginHelpers, output []byte, rm resmap.ResMap) error { + outputRM, err := h.ResmapFactory().NewResMapFromBytes(output) + if err != nil { + return err + } + for _, r := range outputRM.Resources() { + // for each emitted Resource, find the matching Resource in the original ResMap + // using its id + annotations := r.GetAnnotations() + idString, ok := annotations[idAnnotation] + if !ok { + return fmt.Errorf("the transformer %s should not remove annotation %s", + pluginName, idAnnotation) + } + id := resid.ResId{} + err := yaml.Unmarshal([]byte(idString), &id) + if err != nil { + return err + } + res, err := rm.GetByCurrentId(id) + if err != nil { + return fmt.Errorf("unable to find unique match to %s", id.String()) + } + // remove the annotation set by Kustomize to track the resource + delete(annotations, idAnnotation) + if len(annotations) == 0 { + annotations = nil + } + r.SetAnnotations(annotations) + + // update the resource value with the transformed object + res.ResetPrimaryData(r) + } + return nil +} + +// UpdateResourceOptions updates the generator options for each resource in the +// given ResMap based on plugin provided annotations. +func UpdateResourceOptions(rm resmap.ResMap) (resmap.ResMap, error) { + for _, r := range rm.Resources() { + // Disable name hashing by default and require plugin to explicitly + // request it for each resource. + annotations := r.GetAnnotations() + behavior := annotations[BehaviorAnnotation] + var needsHash bool + if val, ok := annotations[HashAnnotation]; ok { + b, err := strconv.ParseBool(val) + if err != nil { + return nil, fmt.Errorf( + "the annotation %q contains an invalid value (%q)", + HashAnnotation, val) + } + needsHash = b + } + delete(annotations, HashAnnotation) + delete(annotations, BehaviorAnnotation) + if len(annotations) == 0 { + annotations = nil + } + r.SetAnnotations(annotations) + r.SetOptions(types.NewGenArgs( + &types.GeneratorArgs{ + Behavior: behavior, + Options: &types.GeneratorOptions{DisableNameSuffixHash: !needsHash}})) + } + return rm, nil +} diff --git a/api/internal/plugins/utils/utils_test.go b/api/internal/plugins/utils/utils_test.go index e4341287b..356e7680a 100644 --- a/api/internal/plugins/utils/utils_test.go +++ b/api/internal/plugins/utils/utils_test.go @@ -4,12 +4,17 @@ package utils import ( + "fmt" "path/filepath" "strings" "testing" "sigs.k8s.io/kustomize/api/filesys" + "sigs.k8s.io/kustomize/api/k8sdeps/kunstruct" "sigs.k8s.io/kustomize/api/konfig" + "sigs.k8s.io/kustomize/api/resmap" + "sigs.k8s.io/kustomize/api/resource" + "sigs.k8s.io/kustomize/api/types" ) func TestDeterminePluginSrcRoot(t *testing.T) { @@ -24,3 +29,99 @@ func TestDeterminePluginSrcRoot(t *testing.T) { t.Errorf("expected suffix '%s' in '%s'", konfig.RelPluginHome, actual) } } + +func makeConfigMap(rf *resource.Factory, name, behavior string, hashValue *string) *resource.Resource { + r := rf.FromMap(map[string]interface{}{ + "apiVersion": "v1", + "kind": "ConfigMap", + "metadata": map[string]interface{}{"name": name}, + }) + annotations := map[string]string{} + if behavior != "" { + annotations[BehaviorAnnotation] = behavior + } + if hashValue != nil { + annotations[HashAnnotation] = *hashValue + } + if len(annotations) > 0 { + r.SetAnnotations(annotations) + } + return r +} + +func makeConfigMapOptions(rf *resource.Factory, name, behavior string, disableHash bool) *resource.Resource { + return rf.FromMapAndOption(map[string]interface{}{ + "apiVersion": "v1", + "kind": "ConfigMap", + "metadata": map[string]interface{}{"name": name}, + }, &types.GeneratorArgs{ + Behavior: behavior, + Options: &types.GeneratorOptions{DisableNameSuffixHash: disableHash}}) +} + +func strptr(s string) *string { + return &s +} + +func TestUpdateResourceOptions(t *testing.T) { + rf := resource.NewFactory(kunstruct.NewKunstructuredFactoryImpl()) + in := resmap.New() + expected := resmap.New() + cases := []struct { + behavior string + needsHash bool + hashValue *string + }{ + {hashValue: strptr("false")}, + {hashValue: strptr("true"), needsHash: true}, + {behavior: "replace"}, + {behavior: "merge"}, + {behavior: "create"}, + {behavior: "nonsense"}, + {behavior: "merge", hashValue: strptr("false")}, + {behavior: "merge", hashValue: strptr("true"), needsHash: true}, + } + for i, c := range cases { + name := fmt.Sprintf("test%d", i) + in.Append(makeConfigMap(rf, name, c.behavior, c.hashValue)) + expected.Append(makeConfigMapOptions(rf, name, c.behavior, !c.needsHash)) + } + actual, err := UpdateResourceOptions(in) + if err != nil { + t.Fatalf("unexpected error: %v", err.Error()) + } + for i, a := range expected.Resources() { + b := actual.GetByIndex(i) + if b == nil { + t.Fatalf("resource %d missing from processed map", i) + } + if !a.Equals(b) { + t.Errorf("expected %v got %v", a, b) + } + if a.NeedHashSuffix() != b.NeedHashSuffix() { + t.Errorf("") + } + if a.Behavior() != b.Behavior() { + t.Errorf("expected %v got %v", a.Behavior(), b.Behavior()) + } + } +} + +func TestUpdateResourceOptionsWithInvalidHashAnnotationValues(t *testing.T) { + rf := resource.NewFactory(kunstruct.NewKunstructuredFactoryImpl()) + cases := []string{ + "", + "FaLsE", + "TrUe", + "potato", + } + for i, c := range cases { + name := fmt.Sprintf("test%d", i) + in := resmap.New() + in.Append(makeConfigMap(rf, name, "", &c)) + _, err := UpdateResourceOptions(in) + if err == nil { + t.Errorf("expected error from value %q", c) + } + } +} From 3019230283115c970c52a400b16311852bff4b05 Mon Sep 17 00:00:00 2001 From: Donny Xia Date: Mon, 29 Jun 2020 13:35:14 -0700 Subject: [PATCH 3/6] Move functionConfig from items to upper level --- api/internal/plugins/fnplugin/fnplugin.go | 116 ++++++++++++++-------- 1 file changed, 76 insertions(+), 40 deletions(-) diff --git a/api/internal/plugins/fnplugin/fnplugin.go b/api/internal/plugins/fnplugin/fnplugin.go index 97f03d9a8..896d48b8c 100644 --- a/api/internal/plugins/fnplugin/fnplugin.go +++ b/api/internal/plugins/fnplugin/fnplugin.go @@ -9,18 +9,17 @@ import ( "github.com/pkg/errors" + "sigs.k8s.io/kustomize/api/internal/plugins/utils" "sigs.k8s.io/kustomize/api/resmap" "sigs.k8s.io/kustomize/api/resource" "sigs.k8s.io/kustomize/api/types" - - "sigs.k8s.io/kustomize/kyaml/kio" - "sigs.k8s.io/kustomize/kyaml/yaml" - - "sigs.k8s.io/kustomize/api/internal/plugins/utils" "sigs.k8s.io/kustomize/kyaml/fn/runtime/runtimeutil" + "sigs.k8s.io/kustomize/kyaml/kio" "sigs.k8s.io/kustomize/kyaml/runfn" + "sigs.k8s.io/kustomize/kyaml/yaml" ) +// FnPlugin is the struct to hold function information type FnPlugin struct { // Function runner runFns runfn.RunFns @@ -52,6 +51,7 @@ func resourceToRNode(res *resource.Resource) (*yaml.RNode, error) { return bytesToRNode(yml) } +// GetFunctionSpec return function spec is there is. Otherwise return nil func GetFunctionSpec(res *resource.Resource) *runtimeutil.FunctionSpec { rnode, err := resourceToRNode(res) if err != nil { @@ -69,6 +69,7 @@ func toStorageMounts(mounts []string) []runtimeutil.StorageMount { return sms } +// NewFnPlugin creates a FnPlugin struct func NewFnPlugin(o *types.FnPluginLoadingOptions) *FnPlugin { return &FnPlugin{ runFns: runfn.RunFns{ @@ -82,10 +83,12 @@ func NewFnPlugin(o *types.FnPluginLoadingOptions) *FnPlugin { } } +// Cfg returns function config func (p *FnPlugin) Cfg() []byte { return p.cfg } +// Config is called by kustomize to pass-in config information func (p *FnPlugin) Config(h *resmap.PluginHelpers, config []byte) error { p.h = h p.cfg = config @@ -106,6 +109,7 @@ func (p *FnPlugin) Config(h *resmap.PluginHelpers, config []byte) error { return nil } +// Generate is called when run as generator func (p *FnPlugin) Generate() (resmap.ResMap, error) { output, err := p.invokePlugin(nil) if err != nil { @@ -118,6 +122,7 @@ func (p *FnPlugin) Generate() (resmap.ResMap, error) { return utils.UpdateResourceOptions(rm) } +// Transform is called when run as transformer func (p *FnPlugin) Transform(rm resmap.ResMap) error { // add ResIds as annotations to all objects so that we can add them back inputRM, err := utils.GetResMapWithIDAnnotation(rm) @@ -141,50 +146,85 @@ func (p *FnPlugin) Transform(rm resmap.ResMap) error { return utils.UpdateResMapValues(p.pluginName, p.h, output, rm) } +func toResourceList(input []byte) (bytes.Buffer, error) { + var out bytes.Buffer + if input == nil { + out.WriteString(fmt.Sprintf("apiVersion: %s\nkind: %s", kio.ResourceListAPIVersion, kio.ResourceListKind)) + } else { + in := bytes.NewReader(input) + err := kio.Pipeline{ + Inputs: []kio.Reader{&kio.ByteReader{Reader: in}}, + Outputs: []kio.Writer{kio.ByteWriter{ + Writer: &out, + WrappingKind: kio.ResourceListKind, + WrappingAPIVersion: kio.ResourceListAPIVersion}}, + }.Execute() + if err != nil { + return out, errors.Wrap( + err, "couldn't transform to ResourceList") + } + } + return out, nil +} + +func injectAnnotation(input *yaml.RNode, k, v string) error { + err := input.PipeE(yaml.SetAnnotation(k, v)) + if err != nil { + return err + } + return nil +} + +// injectFunctionConfig injects the `functionConfig` field into the resource list. +// The value is the function configuaration. +func injectFunctionConfig(input *bytes.Buffer, functionConfig *yaml.RNode) error { + nodes, err := bytesToRNode(input.Bytes()) + if err != nil { + return err + } + err = nodes.PipeE( + yaml.LookupCreate(yaml.ScalarNode, "functionConfig"), + yaml.Set(functionConfig), + ) + if err != nil { + return err + } + input.Reset() + s, err := nodes.String() + if err != nil { + return err + } + input.WriteString(s) + return nil +} + // invokePlugin uses Function runner to run function as plugin func (p *FnPlugin) invokePlugin(input []byte) ([]byte, error) { - // get config rnode - rnode, err := bytesToRNode(p.cfg) + // get function config rnode + functionConfig, err := bytesToRNode(p.cfg) if err != nil { return nil, err } - err = rnode.PipeE(yaml.SetAnnotation("config.kubernetes.io/local-config", "true")) + err = injectAnnotation(functionConfig, "config.kubernetes.io/local-config", "true") if err != nil { return nil, err } - // we need to add config as input for generators. Some of them don't work with FunctionConfig - // and in addition kio.Pipeline won't create anything if there are no objects - // see https://github.com/kubernetes-sigs/kustomize/blob/master/kyaml/kio/kio.go#L93 - if input == nil { - yaml, err := rnode.String() - if err != nil { - return nil, err - } - input = []byte(yaml) - } - // Transform to ResourceList - var inOut bytes.Buffer - inIn := bytes.NewReader(input) - err = kio.Pipeline{ - Inputs: []kio.Reader{&kio.ByteReader{Reader: inIn}}, - Outputs: []kio.Writer{kio.ByteWriter{ - Writer: &inOut, - WrappingKind: kio.ResourceListKind, - WrappingAPIVersion: kio.ResourceListAPIVersion}}, - }.Execute() + inputBuffer, err := toResourceList(input) if err != nil { - return nil, errors.Wrap( - err, "couldn't transform to ResourceList") + return nil, err + } + err = injectFunctionConfig(&inputBuffer, functionConfig) + if err != nil { + return nil, err } - //log.Printf("converted to:\n%s\n", inOut.String()) // Configure and Execute Fn - var runFnsOut bytes.Buffer - p.runFns.Input = bytes.NewReader(inOut.Bytes()) - p.runFns.Functions = append(p.runFns.Functions, rnode) - p.runFns.Output = &runFnsOut + var ouputBuffer bytes.Buffer + p.runFns.Input = bytes.NewReader(inputBuffer.Bytes()) + p.runFns.Functions = append(p.runFns.Functions, functionConfig) + p.runFns.Output = &ouputBuffer err = p.runFns.Execute() if err != nil { @@ -192,11 +232,9 @@ func (p *FnPlugin) invokePlugin(input []byte) ([]byte, error) { err, "couldn't execute function") } - //log.Printf("fn returned:\n%s\n", runFnsOut.String()) - // Convert back to a single multi-yaml doc var outOut bytes.Buffer - outIn := bytes.NewReader(runFnsOut.Bytes()) + outIn := bytes.NewReader(ouputBuffer.Bytes()) err = kio.Pipeline{ Inputs: []kio.Reader{&kio.ByteReader{Reader: outIn}}, @@ -207,7 +245,5 @@ func (p *FnPlugin) invokePlugin(input []byte) ([]byte, error) { err, "couldn't transform from ResourceList") } - //log.Printf("converted back to:\n%s\n", outOut.String()) - return outOut.Bytes(), nil } From e3ec184e928ab145c39060391de27a3b25dd1b1e Mon Sep 17 00:00:00 2001 From: Donny Xia Date: Mon, 29 Jun 2020 14:07:48 -0700 Subject: [PATCH 4/6] Add unit test for fnplugin --- .../plugins/fnplugin/fnplugin_test.go | 122 ++++++++++++++++++ 1 file changed, 122 insertions(+) create mode 100644 api/internal/plugins/fnplugin/fnplugin_test.go diff --git a/api/internal/plugins/fnplugin/fnplugin_test.go b/api/internal/plugins/fnplugin/fnplugin_test.go new file mode 100644 index 000000000..0f880d03b --- /dev/null +++ b/api/internal/plugins/fnplugin/fnplugin_test.go @@ -0,0 +1,122 @@ +package fnplugin + +import ( + "bytes" + "fmt" + "sigs.k8s.io/kustomize/kyaml/kio" + "testing" +) + +func TestToResourceList(t *testing.T) { + in := []byte(`apiVersion: v1 +data: + key1: oldValue +kind: ConfigMap +metadata: + annotations: + kustomize.config.k8s.io/id: | + kind: ConfigMap + name: config1 + version: v1 + name: config1 +--- +apiVersion: v1 +data: + key1: oldValue +kind: ConfigMap +metadata: + annotations: + kustomize.config.k8s.io/id: | + kind: ConfigMap + name: config2 + version: v1 + name: config2`) + outBuffer, err := toResourceList(in) + if err != nil { + t.Fatal(err) + } + + expected := `apiVersion: config.kubernetes.io/v1alpha1 +kind: ResourceList +items: +- apiVersion: v1 + data: + key1: oldValue + kind: ConfigMap + metadata: + kustomize.config.k8s.io/id: | + kind: ConfigMap + name: config1 + version: v1 + name: config1 +- apiVersion: v1 + data: + key1: oldValue + kind: ConfigMap + metadata: + kustomize.config.k8s.io/id: | + kind: ConfigMap + name: config2 + version: v1 + name: config2 +` + + if outBuffer.String() != expected { + t.Fatalf("output \n%s\n doesn't match expected \n%s\n", outBuffer.String(), expected) + } +} + +func TestToResourceListWithEmptyInput(t *testing.T) { + expected := fmt.Sprintf("apiVersion: %s\nkind: %s", kio.ResourceListAPIVersion, kio.ResourceListKind) + outBuffer, err := toResourceList(nil) + if err != nil { + t.Fatal(err) + } + if outBuffer.String() != expected { + t.Fatalf("output \n%s\n doesn't match expected \n%s\n", outBuffer.String(), expected) + } +} + +func TestInjectFunctionConfig(t *testing.T) { + input := []byte(`apiVersion: config.kubernetes.io/v1alpha1 +kind: ResourceList`) + functionConfig, err := bytesToRNode([]byte(`apiVersion: foo-corp.com/v1 +kind: FulfillmentCenter +metadata: + name: staging + metadata: + annotations: + config.k8s.io/function: | + container: + image: gcr.io/example/foo:v1.0.0 +spec: + address: "100 Main St."`)) + if err != nil { + t.Fatal(err) + } + inputBuffer := bytes.Buffer{} + inputBuffer.Write(input) + err = injectFunctionConfig(&inputBuffer, functionConfig) + if err != nil { + t.Fatal(err) + } + expected := `apiVersion: config.kubernetes.io/v1alpha1 +kind: ResourceList +functionConfig: + apiVersion: foo-corp.com/v1 + kind: FulfillmentCenter + metadata: + name: staging + metadata: + annotations: + config.k8s.io/function: | + container: + image: gcr.io/example/foo:v1.0.0 + spec: + address: "100 Main St." +` + + if inputBuffer.String() != expected { + t.Fatalf("output \n%s\n doesn't match expected \n%s\n", inputBuffer.String(), expected) + } +} From 8bb612889cd5266825d50aba580e437622bf587e Mon Sep 17 00:00:00 2001 From: Donny Xia Date: Mon, 29 Jun 2020 16:51:56 -0700 Subject: [PATCH 5/6] Improve function invocation --- api/internal/plugins/fnplugin/fnplugin.go | 88 +++---------- .../plugins/fnplugin/fnplugin_test.go | 122 ------------------ api/krusty/fnplugin_test.go | 60 +++++++++ 3 files changed, 78 insertions(+), 192 deletions(-) delete mode 100644 api/internal/plugins/fnplugin/fnplugin_test.go diff --git a/api/internal/plugins/fnplugin/fnplugin.go b/api/internal/plugins/fnplugin/fnplugin.go index 896d48b8c..72dbf4271 100644 --- a/api/internal/plugins/fnplugin/fnplugin.go +++ b/api/internal/plugins/fnplugin/fnplugin.go @@ -14,7 +14,6 @@ import ( "sigs.k8s.io/kustomize/api/resource" "sigs.k8s.io/kustomize/api/types" "sigs.k8s.io/kustomize/kyaml/fn/runtime/runtimeutil" - "sigs.k8s.io/kustomize/kyaml/kio" "sigs.k8s.io/kustomize/kyaml/runfn" "sigs.k8s.io/kustomize/kyaml/yaml" ) @@ -146,27 +145,6 @@ func (p *FnPlugin) Transform(rm resmap.ResMap) error { return utils.UpdateResMapValues(p.pluginName, p.h, output, rm) } -func toResourceList(input []byte) (bytes.Buffer, error) { - var out bytes.Buffer - if input == nil { - out.WriteString(fmt.Sprintf("apiVersion: %s\nkind: %s", kio.ResourceListAPIVersion, kio.ResourceListKind)) - } else { - in := bytes.NewReader(input) - err := kio.Pipeline{ - Inputs: []kio.Reader{&kio.ByteReader{Reader: in}}, - Outputs: []kio.Writer{kio.ByteWriter{ - Writer: &out, - WrappingKind: kio.ResourceListKind, - WrappingAPIVersion: kio.ResourceListAPIVersion}}, - }.Execute() - if err != nil { - return out, errors.Wrap( - err, "couldn't transform to ResourceList") - } - } - return out, nil -} - func injectAnnotation(input *yaml.RNode, k, v string) error { err := input.PipeE(yaml.SetAnnotation(k, v)) if err != nil { @@ -175,29 +153,6 @@ func injectAnnotation(input *yaml.RNode, k, v string) error { return nil } -// injectFunctionConfig injects the `functionConfig` field into the resource list. -// The value is the function configuaration. -func injectFunctionConfig(input *bytes.Buffer, functionConfig *yaml.RNode) error { - nodes, err := bytesToRNode(input.Bytes()) - if err != nil { - return err - } - err = nodes.PipeE( - yaml.LookupCreate(yaml.ScalarNode, "functionConfig"), - yaml.Set(functionConfig), - ) - if err != nil { - return err - } - input.Reset() - s, err := nodes.String() - if err != nil { - return err - } - input.WriteString(s) - return nil -} - // invokePlugin uses Function runner to run function as plugin func (p *FnPlugin) invokePlugin(input []byte) ([]byte, error) { // get function config rnode @@ -205,24 +160,30 @@ func (p *FnPlugin) invokePlugin(input []byte) ([]byte, error) { if err != nil { return nil, err } + + // This annotation will let kustomize ingnore this item in output err = injectAnnotation(functionConfig, "config.kubernetes.io/local-config", "true") if err != nil { return nil, err } - - // Transform to ResourceList - inputBuffer, err := toResourceList(input) - if err != nil { - return nil, err - } - err = injectFunctionConfig(&inputBuffer, functionConfig) - if err != nil { - return nil, err + // we need to add config as input for generators. Some of them don't work with FunctionConfig + // and in addition kio.Pipeline won't create anything if there are no objects + // see https://github.com/kubernetes-sigs/kustomize/blob/master/kyaml/kio/kio.go#L93 + // Since we added `local-config` annotation so it will be ignored in generator output + // TODO(donnyxia): This is actually not used by generator and only used to bypass a kio limitation. + // Need better solution. + if input == nil { + yaml, err := functionConfig.String() + if err != nil { + return nil, err + } + input = []byte(yaml) } - // Configure and Execute Fn + // Configure and Execute Fn. We don't need to convert resources to ResourceList here + // because function runtime will do that. See kyaml/fn/runtime/runtimeutil/runtimeutil.go var ouputBuffer bytes.Buffer - p.runFns.Input = bytes.NewReader(inputBuffer.Bytes()) + p.runFns.Input = bytes.NewReader(input) p.runFns.Functions = append(p.runFns.Functions, functionConfig) p.runFns.Output = &ouputBuffer @@ -232,18 +193,5 @@ func (p *FnPlugin) invokePlugin(input []byte) ([]byte, error) { err, "couldn't execute function") } - // Convert back to a single multi-yaml doc - var outOut bytes.Buffer - outIn := bytes.NewReader(ouputBuffer.Bytes()) - - err = kio.Pipeline{ - Inputs: []kio.Reader{&kio.ByteReader{Reader: outIn}}, - Outputs: []kio.Writer{kio.ByteWriter{Writer: &outOut}}, - }.Execute() - if err != nil { - return nil, errors.Wrap( - err, "couldn't transform from ResourceList") - } - - return outOut.Bytes(), nil + return ouputBuffer.Bytes(), nil } diff --git a/api/internal/plugins/fnplugin/fnplugin_test.go b/api/internal/plugins/fnplugin/fnplugin_test.go deleted file mode 100644 index 0f880d03b..000000000 --- a/api/internal/plugins/fnplugin/fnplugin_test.go +++ /dev/null @@ -1,122 +0,0 @@ -package fnplugin - -import ( - "bytes" - "fmt" - "sigs.k8s.io/kustomize/kyaml/kio" - "testing" -) - -func TestToResourceList(t *testing.T) { - in := []byte(`apiVersion: v1 -data: - key1: oldValue -kind: ConfigMap -metadata: - annotations: - kustomize.config.k8s.io/id: | - kind: ConfigMap - name: config1 - version: v1 - name: config1 ---- -apiVersion: v1 -data: - key1: oldValue -kind: ConfigMap -metadata: - annotations: - kustomize.config.k8s.io/id: | - kind: ConfigMap - name: config2 - version: v1 - name: config2`) - outBuffer, err := toResourceList(in) - if err != nil { - t.Fatal(err) - } - - expected := `apiVersion: config.kubernetes.io/v1alpha1 -kind: ResourceList -items: -- apiVersion: v1 - data: - key1: oldValue - kind: ConfigMap - metadata: - kustomize.config.k8s.io/id: | - kind: ConfigMap - name: config1 - version: v1 - name: config1 -- apiVersion: v1 - data: - key1: oldValue - kind: ConfigMap - metadata: - kustomize.config.k8s.io/id: | - kind: ConfigMap - name: config2 - version: v1 - name: config2 -` - - if outBuffer.String() != expected { - t.Fatalf("output \n%s\n doesn't match expected \n%s\n", outBuffer.String(), expected) - } -} - -func TestToResourceListWithEmptyInput(t *testing.T) { - expected := fmt.Sprintf("apiVersion: %s\nkind: %s", kio.ResourceListAPIVersion, kio.ResourceListKind) - outBuffer, err := toResourceList(nil) - if err != nil { - t.Fatal(err) - } - if outBuffer.String() != expected { - t.Fatalf("output \n%s\n doesn't match expected \n%s\n", outBuffer.String(), expected) - } -} - -func TestInjectFunctionConfig(t *testing.T) { - input := []byte(`apiVersion: config.kubernetes.io/v1alpha1 -kind: ResourceList`) - functionConfig, err := bytesToRNode([]byte(`apiVersion: foo-corp.com/v1 -kind: FulfillmentCenter -metadata: - name: staging - metadata: - annotations: - config.k8s.io/function: | - container: - image: gcr.io/example/foo:v1.0.0 -spec: - address: "100 Main St."`)) - if err != nil { - t.Fatal(err) - } - inputBuffer := bytes.Buffer{} - inputBuffer.Write(input) - err = injectFunctionConfig(&inputBuffer, functionConfig) - if err != nil { - t.Fatal(err) - } - expected := `apiVersion: config.kubernetes.io/v1alpha1 -kind: ResourceList -functionConfig: - apiVersion: foo-corp.com/v1 - kind: FulfillmentCenter - metadata: - name: staging - metadata: - annotations: - config.k8s.io/function: | - container: - image: gcr.io/example/foo:v1.0.0 - spec: - address: "100 Main St." -` - - if inputBuffer.String() != expected { - t.Fatalf("output \n%s\n doesn't match expected \n%s\n", inputBuffer.String(), expected) - } -} diff --git a/api/krusty/fnplugin_test.go b/api/krusty/fnplugin_test.go index 1aaa2ac90..5a2081622 100644 --- a/api/krusty/fnplugin_test.go +++ b/api/krusty/fnplugin_test.go @@ -404,3 +404,63 @@ spec: memory: 50M `) } + +func TestFnContainerTransformerWithConfig(t *testing.T) { + skipIfNoDocker(t) + + th := kusttest_test.MakeEnhancedHarness(t) + defer th.Reset() + + th.WriteK("/app", ` +resources: +- data1.yaml +- data2.yaml +transformers: +- label_namespace.yaml +`) + + th.WriteF("/app/data1.yaml", `apiVersion: v1 +kind: Namespace +metadata: + name: my-namespace +`) + th.WriteF("/app/data2.yaml", `apiVersion: v1 +kind: Namespace +metadata: + name: another-namespace +`) + + th.WriteF("/app/label_namespace.yaml", `apiVersion: v1 +kind: ConfigMap +metadata: + name: label_namespace + annotations: + config.kubernetes.io/function: |- + container: + image: gcr.io/kpt-functions/label-namespace@sha256:4f030738d6d25a207641ca517916431517578bd0eb8d98a8bde04e3bb9315dcd +data: + label_name: my-ns-name + label_value: function-test +`) + + m := th.Run("/app", th.MakeOptionsPluginsEnabled()) + th.AssertActualEqualsExpected(m, ` +apiVersion: v1 +kind: Namespace +metadata: + annotations: + config.kubernetes.io/path: namespace_my-namespace.yaml + labels: + my-ns-name: function-test + name: my-namespace +--- +apiVersion: v1 +kind: Namespace +metadata: + annotations: + config.kubernetes.io/path: namespace_another-namespace.yaml + labels: + my-ns-name: function-test + name: another-namespace +`) +} From 98a92a644333b8ecb05bc5e980d735654067fd5d Mon Sep 17 00:00:00 2001 From: Donny Xia Date: Mon, 29 Jun 2020 16:57:49 -0700 Subject: [PATCH 6/6] Remove comment from fnexectest.sh --- api/krusty/fnplugin_test/fnexectest.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/api/krusty/fnplugin_test/fnexectest.sh b/api/krusty/fnplugin_test/fnexectest.sh index ed98a837c..d8e7c9eab 100755 --- a/api/krusty/fnplugin_test/fnexectest.sh +++ b/api/krusty/fnplugin_test/fnexectest.sh @@ -1,7 +1,5 @@ #!/bin/sh -# not sure if we want to generate bash scripts, since we always want to run -# only trusted executables cat <