From 8bb612889cd5266825d50aba580e437622bf587e Mon Sep 17 00:00:00 2001 From: Donny Xia Date: Mon, 29 Jun 2020 16:51:56 -0700 Subject: [PATCH] Improve function invocation --- api/internal/plugins/fnplugin/fnplugin.go | 88 +++---------- .../plugins/fnplugin/fnplugin_test.go | 122 ------------------ api/krusty/fnplugin_test.go | 60 +++++++++ 3 files changed, 78 insertions(+), 192 deletions(-) delete mode 100644 api/internal/plugins/fnplugin/fnplugin_test.go diff --git a/api/internal/plugins/fnplugin/fnplugin.go b/api/internal/plugins/fnplugin/fnplugin.go index 896d48b8c..72dbf4271 100644 --- a/api/internal/plugins/fnplugin/fnplugin.go +++ b/api/internal/plugins/fnplugin/fnplugin.go @@ -14,7 +14,6 @@ import ( "sigs.k8s.io/kustomize/api/resource" "sigs.k8s.io/kustomize/api/types" "sigs.k8s.io/kustomize/kyaml/fn/runtime/runtimeutil" - "sigs.k8s.io/kustomize/kyaml/kio" "sigs.k8s.io/kustomize/kyaml/runfn" "sigs.k8s.io/kustomize/kyaml/yaml" ) @@ -146,27 +145,6 @@ func (p *FnPlugin) Transform(rm resmap.ResMap) error { return utils.UpdateResMapValues(p.pluginName, p.h, output, rm) } -func toResourceList(input []byte) (bytes.Buffer, error) { - var out bytes.Buffer - if input == nil { - out.WriteString(fmt.Sprintf("apiVersion: %s\nkind: %s", kio.ResourceListAPIVersion, kio.ResourceListKind)) - } else { - in := bytes.NewReader(input) - err := kio.Pipeline{ - Inputs: []kio.Reader{&kio.ByteReader{Reader: in}}, - Outputs: []kio.Writer{kio.ByteWriter{ - Writer: &out, - WrappingKind: kio.ResourceListKind, - WrappingAPIVersion: kio.ResourceListAPIVersion}}, - }.Execute() - if err != nil { - return out, errors.Wrap( - err, "couldn't transform to ResourceList") - } - } - return out, nil -} - func injectAnnotation(input *yaml.RNode, k, v string) error { err := input.PipeE(yaml.SetAnnotation(k, v)) if err != nil { @@ -175,29 +153,6 @@ func injectAnnotation(input *yaml.RNode, k, v string) error { return nil } -// injectFunctionConfig injects the `functionConfig` field into the resource list. -// The value is the function configuaration. -func injectFunctionConfig(input *bytes.Buffer, functionConfig *yaml.RNode) error { - nodes, err := bytesToRNode(input.Bytes()) - if err != nil { - return err - } - err = nodes.PipeE( - yaml.LookupCreate(yaml.ScalarNode, "functionConfig"), - yaml.Set(functionConfig), - ) - if err != nil { - return err - } - input.Reset() - s, err := nodes.String() - if err != nil { - return err - } - input.WriteString(s) - return nil -} - // invokePlugin uses Function runner to run function as plugin func (p *FnPlugin) invokePlugin(input []byte) ([]byte, error) { // get function config rnode @@ -205,24 +160,30 @@ func (p *FnPlugin) invokePlugin(input []byte) ([]byte, error) { if err != nil { return nil, err } + + // This annotation will let kustomize ingnore this item in output err = injectAnnotation(functionConfig, "config.kubernetes.io/local-config", "true") if err != nil { return nil, err } - - // Transform to ResourceList - inputBuffer, err := toResourceList(input) - if err != nil { - return nil, err - } - err = injectFunctionConfig(&inputBuffer, functionConfig) - if err != nil { - return nil, err + // we need to add config as input for generators. Some of them don't work with FunctionConfig + // and in addition kio.Pipeline won't create anything if there are no objects + // see https://github.com/kubernetes-sigs/kustomize/blob/master/kyaml/kio/kio.go#L93 + // Since we added `local-config` annotation so it will be ignored in generator output + // TODO(donnyxia): This is actually not used by generator and only used to bypass a kio limitation. + // Need better solution. + if input == nil { + yaml, err := functionConfig.String() + if err != nil { + return nil, err + } + input = []byte(yaml) } - // Configure and Execute Fn + // Configure and Execute Fn. We don't need to convert resources to ResourceList here + // because function runtime will do that. See kyaml/fn/runtime/runtimeutil/runtimeutil.go var ouputBuffer bytes.Buffer - p.runFns.Input = bytes.NewReader(inputBuffer.Bytes()) + p.runFns.Input = bytes.NewReader(input) p.runFns.Functions = append(p.runFns.Functions, functionConfig) p.runFns.Output = &ouputBuffer @@ -232,18 +193,5 @@ func (p *FnPlugin) invokePlugin(input []byte) ([]byte, error) { err, "couldn't execute function") } - // Convert back to a single multi-yaml doc - var outOut bytes.Buffer - outIn := bytes.NewReader(ouputBuffer.Bytes()) - - err = kio.Pipeline{ - Inputs: []kio.Reader{&kio.ByteReader{Reader: outIn}}, - Outputs: []kio.Writer{kio.ByteWriter{Writer: &outOut}}, - }.Execute() - if err != nil { - return nil, errors.Wrap( - err, "couldn't transform from ResourceList") - } - - return outOut.Bytes(), nil + return ouputBuffer.Bytes(), nil } diff --git a/api/internal/plugins/fnplugin/fnplugin_test.go b/api/internal/plugins/fnplugin/fnplugin_test.go deleted file mode 100644 index 0f880d03b..000000000 --- a/api/internal/plugins/fnplugin/fnplugin_test.go +++ /dev/null @@ -1,122 +0,0 @@ -package fnplugin - -import ( - "bytes" - "fmt" - "sigs.k8s.io/kustomize/kyaml/kio" - "testing" -) - -func TestToResourceList(t *testing.T) { - in := []byte(`apiVersion: v1 -data: - key1: oldValue -kind: ConfigMap -metadata: - annotations: - kustomize.config.k8s.io/id: | - kind: ConfigMap - name: config1 - version: v1 - name: config1 ---- -apiVersion: v1 -data: - key1: oldValue -kind: ConfigMap -metadata: - annotations: - kustomize.config.k8s.io/id: | - kind: ConfigMap - name: config2 - version: v1 - name: config2`) - outBuffer, err := toResourceList(in) - if err != nil { - t.Fatal(err) - } - - expected := `apiVersion: config.kubernetes.io/v1alpha1 -kind: ResourceList -items: -- apiVersion: v1 - data: - key1: oldValue - kind: ConfigMap - metadata: - kustomize.config.k8s.io/id: | - kind: ConfigMap - name: config1 - version: v1 - name: config1 -- apiVersion: v1 - data: - key1: oldValue - kind: ConfigMap - metadata: - kustomize.config.k8s.io/id: | - kind: ConfigMap - name: config2 - version: v1 - name: config2 -` - - if outBuffer.String() != expected { - t.Fatalf("output \n%s\n doesn't match expected \n%s\n", outBuffer.String(), expected) - } -} - -func TestToResourceListWithEmptyInput(t *testing.T) { - expected := fmt.Sprintf("apiVersion: %s\nkind: %s", kio.ResourceListAPIVersion, kio.ResourceListKind) - outBuffer, err := toResourceList(nil) - if err != nil { - t.Fatal(err) - } - if outBuffer.String() != expected { - t.Fatalf("output \n%s\n doesn't match expected \n%s\n", outBuffer.String(), expected) - } -} - -func TestInjectFunctionConfig(t *testing.T) { - input := []byte(`apiVersion: config.kubernetes.io/v1alpha1 -kind: ResourceList`) - functionConfig, err := bytesToRNode([]byte(`apiVersion: foo-corp.com/v1 -kind: FulfillmentCenter -metadata: - name: staging - metadata: - annotations: - config.k8s.io/function: | - container: - image: gcr.io/example/foo:v1.0.0 -spec: - address: "100 Main St."`)) - if err != nil { - t.Fatal(err) - } - inputBuffer := bytes.Buffer{} - inputBuffer.Write(input) - err = injectFunctionConfig(&inputBuffer, functionConfig) - if err != nil { - t.Fatal(err) - } - expected := `apiVersion: config.kubernetes.io/v1alpha1 -kind: ResourceList -functionConfig: - apiVersion: foo-corp.com/v1 - kind: FulfillmentCenter - metadata: - name: staging - metadata: - annotations: - config.k8s.io/function: | - container: - image: gcr.io/example/foo:v1.0.0 - spec: - address: "100 Main St." -` - - if inputBuffer.String() != expected { - t.Fatalf("output \n%s\n doesn't match expected \n%s\n", inputBuffer.String(), expected) - } -} diff --git a/api/krusty/fnplugin_test.go b/api/krusty/fnplugin_test.go index 1aaa2ac90..5a2081622 100644 --- a/api/krusty/fnplugin_test.go +++ b/api/krusty/fnplugin_test.go @@ -404,3 +404,63 @@ spec: memory: 50M `) } + +func TestFnContainerTransformerWithConfig(t *testing.T) { + skipIfNoDocker(t) + + th := kusttest_test.MakeEnhancedHarness(t) + defer th.Reset() + + th.WriteK("/app", ` +resources: +- data1.yaml +- data2.yaml +transformers: +- label_namespace.yaml +`) + + th.WriteF("/app/data1.yaml", `apiVersion: v1 +kind: Namespace +metadata: + name: my-namespace +`) + th.WriteF("/app/data2.yaml", `apiVersion: v1 +kind: Namespace +metadata: + name: another-namespace +`) + + th.WriteF("/app/label_namespace.yaml", `apiVersion: v1 +kind: ConfigMap +metadata: + name: label_namespace + annotations: + config.kubernetes.io/function: |- + container: + image: gcr.io/kpt-functions/label-namespace@sha256:4f030738d6d25a207641ca517916431517578bd0eb8d98a8bde04e3bb9315dcd +data: + label_name: my-ns-name + label_value: function-test +`) + + m := th.Run("/app", th.MakeOptionsPluginsEnabled()) + th.AssertActualEqualsExpected(m, ` +apiVersion: v1 +kind: Namespace +metadata: + annotations: + config.kubernetes.io/path: namespace_my-namespace.yaml + labels: + my-ns-name: function-test + name: my-namespace +--- +apiVersion: v1 +kind: Namespace +metadata: + annotations: + config.kubernetes.io/path: namespace_another-namespace.yaml + labels: + my-ns-name: function-test + name: another-namespace +`) +}