From 3def13d47992b19d72137e791f4325acdf822790 Mon Sep 17 00:00:00 2001 From: Phillip Wittrock Date: Mon, 23 Mar 2020 11:19:14 -0700 Subject: [PATCH] Support for FieldSpec based operations on kyaml objects --- api/filters/fsslice/doc.go | 6 + api/filters/fsslice/example_test.go | 62 +++++ api/filters/fsslice/fieldspec_filter.go | 145 ++++++++++ api/filters/fsslice/fsslice.go | 78 ++++++ api/filters/fsslice/fsslice_test.go | 354 ++++++++++++++++++++++++ 5 files changed, 645 insertions(+) create mode 100644 api/filters/fsslice/doc.go create mode 100644 api/filters/fsslice/example_test.go create mode 100644 api/filters/fsslice/fieldspec_filter.go create mode 100644 api/filters/fsslice/fsslice.go create mode 100644 api/filters/fsslice/fsslice_test.go diff --git a/api/filters/fsslice/doc.go b/api/filters/fsslice/doc.go new file mode 100644 index 000000000..407f11c47 --- /dev/null +++ b/api/filters/fsslice/doc.go @@ -0,0 +1,6 @@ +// Copyright 2020 The Kubernetes Authors. +// SPDX-License-Identifier: Apache-2.0 + +// Package fsslice contains a yaml.Filter to modify a resource using an +// FsSlice to identify fields to be updated within the resource. +package fsslice diff --git a/api/filters/fsslice/example_test.go b/api/filters/fsslice/example_test.go new file mode 100644 index 000000000..ce14d4a58 --- /dev/null +++ b/api/filters/fsslice/example_test.go @@ -0,0 +1,62 @@ +// Copyright 2020 The Kubernetes Authors. +// SPDX-License-Identifier: Apache-2.0 + +package fsslice_test + +import ( + "bytes" + "log" + "os" + + "sigs.k8s.io/kustomize/api/filters/fsslice" + "sigs.k8s.io/kustomize/api/types" + "sigs.k8s.io/kustomize/kyaml/kio" + "sigs.k8s.io/kustomize/kyaml/yaml" +) + +func ExampleFilter() { + in := &kio.ByteReader{ + Reader: bytes.NewBufferString(` +apiVersion: example.com/v1 +kind: Foo +metadata: + name: instance +--- +apiVersion: example.com/v1 +kind: Bar +metadata: + name: instance +`), + } + fltr := fsslice.Filter{ + CreateKind: yaml.ScalarNode, + SetValue: fsslice.SetScalar("green"), + FsSlice: []types.FieldSpec{ + {Path: "a/b", CreateIfNotPresent: true}, + }, + } + + err := kio.Pipeline{ + Inputs: []kio.Reader{in}, + Filters: []kio.Filter{kio.FilterAll(fltr)}, + Outputs: []kio.Writer{kio.ByteWriter{Writer: os.Stdout}}, + }.Execute() + if err != nil { + log.Fatal(err) + } + + // Output: + // apiVersion: example.com/v1 + // kind: Foo + // metadata: + // name: instance + // a: + // b: green + // --- + // apiVersion: example.com/v1 + // kind: Bar + // metadata: + // name: instance + // a: + // b: green +} diff --git a/api/filters/fsslice/fieldspec_filter.go b/api/filters/fsslice/fieldspec_filter.go new file mode 100644 index 000000000..7ff2387f8 --- /dev/null +++ b/api/filters/fsslice/fieldspec_filter.go @@ -0,0 +1,145 @@ +// Copyright 2019 The Kubernetes Authors. +// SPDX-License-Identifier: Apache-2.0 + +package fsslice + +import ( + "strings" + + "sigs.k8s.io/kustomize/api/types" + "sigs.k8s.io/kustomize/kyaml/errors" + "sigs.k8s.io/kustomize/kyaml/yaml" +) + +// fieldSpecFilter applies a single fieldSpec to a single object +// fieldSpecFilter stores internal state and should not be reused +type fieldSpecFilter struct { + // FieldSpec contains the path to the value to set. + FieldSpec types.FieldSpec `yaml:"fieldSpec"` + + // Set the field using this function + SetValue SetFn + + // CreateKind defines the type of node to create if the field is not found + CreateKind yaml.Kind + + // path keeps internal state about the current path + path []string +} + +func (fltr fieldSpecFilter) Filter(obj *yaml.RNode) (*yaml.RNode, error) { + // check if the FieldSpec applies to the object + if match, err := isMatchGVK(fltr.FieldSpec, obj); !match || err != nil { + return obj, errors.Wrap(err) + } + fltr.path = strings.Split(fltr.FieldSpec.Path, "/") + if err := fltr.filter(obj); err != nil { + s, _ := obj.String() + return nil, errors.WrapPrefixf(err, + "obj %v at path %v", s, fltr.FieldSpec.Path) + } + return obj, nil +} + +func (fltr fieldSpecFilter) filter(obj *yaml.RNode) error { + if len(fltr.path) == 0 { + // found the field -- set its value + return fltr.SetValue(obj) + } + switch obj.YNode().Kind { + case yaml.SequenceNode: + return fltr.seq(obj) + case yaml.MappingNode: + return fltr.field(obj) + } + // not found -- this might be an error since the type doesn't match + + return errors.Errorf("unsupported yaml node") +} + +// field calls filter on the field matching the next path element +func (fltr fieldSpecFilter) field(obj *yaml.RNode) error { + fieldName, isSeq := isSequenceField(fltr.path[0]) + + // lookup the field matching the next path element + var lookupField yaml.Filter + switch { + case !fltr.FieldSpec.CreateIfNotPresent || fltr.CreateKind == 0 || isSeq: + // dont' create the field if we don't find it + lookupField = yaml.Lookup(fieldName) + case len(fltr.path) <= 1: + // create the field if it is missing: use the provided node kind + lookupField = yaml.LookupCreate(fltr.CreateKind, fieldName) + default: + // create the field if it is missing: must be a mapping node + lookupField = yaml.LookupCreate(yaml.MappingNode, fieldName) + } + + // locate (or maybe create) the field + field, err := obj.Pipe(lookupField) + if err != nil || field == nil { + return errors.WrapPrefixf(err, "fieldName: %s", fieldName) + } + + // copy the current fltr and change the path on the copy + var next = fltr + // call filter for the next path element on the matching field + next.path = fltr.path[1:] + return next.filter(field) +} + +// seq calls filter on all sequence elements +func (fltr fieldSpecFilter) seq(obj *yaml.RNode) error { + if err := obj.VisitElements(func(node *yaml.RNode) error { + // recurse on each element -- re-allocating a fieldSpecFilter is + // not strictly required, but is more consistent with field + // and less likely to have side effects + // keep the entire path -- it does not contain parts for sequences + return fltr.filter(node) + }); err != nil { + return errors.WrapPrefixf(err, + "visit traversal on path: %v", fltr.path) + } + + return nil +} + +// isSequenceField returns true if the path element is for a sequence field. +// isSequence also returns the path element with the '[]' suffix trimmed +func isSequenceField(name string) (string, bool) { + isSeq := strings.HasSuffix(name, "[]") + name = strings.TrimSuffix(name, "[]") + return name, isSeq +} + +// isMatchGVK returns true if the fs.GVK matches the obj GVK. +func isMatchGVK(fs types.FieldSpec, obj *yaml.RNode) (bool, error) { + meta, err := obj.GetMeta() + if err != nil { + return false, err + } + if fs.Kind != "" && fs.Kind != meta.Kind { + // kind doesn't match + return false, err + } + + // parse the group and version from the apiVersion field + var group, version string + parts := strings.SplitN(meta.APIVersion, "/", 2) + group = parts[0] + if len(parts) > 1 { + version = parts[1] + } + + if fs.Group != "" && fs.Group != group { + // group doesn't match + return false, nil + } + + if fs.Version != "" && fs.Version != version { + // version doesn't match + return false, nil + } + + return true, nil +} diff --git a/api/filters/fsslice/fsslice.go b/api/filters/fsslice/fsslice.go new file mode 100644 index 000000000..e5c896d19 --- /dev/null +++ b/api/filters/fsslice/fsslice.go @@ -0,0 +1,78 @@ +// Copyright 2019 The Kubernetes Authors. +// SPDX-License-Identifier: Apache-2.0 + +package fsslice + +import ( + "strings" + + "sigs.k8s.io/kustomize/api/resid" + "sigs.k8s.io/kustomize/api/types" + "sigs.k8s.io/kustomize/kyaml/yaml" +) + +// SetFn sets a value +type SetFn func(*yaml.RNode) error + +// SetScalar returns a SetFn to set a scalar value +func SetScalar(value string) SetFn { + return func(node *yaml.RNode) error { + return node.PipeE(yaml.FieldSetter{StringValue: value}) + } +} + +// SetEntry returns a SetFn to set an entry in a map +func SetEntry(key, value string) SetFn { + return func(node *yaml.RNode) error { + return node.PipeE(yaml.FieldSetter{ + Name: key, StringValue: value}) + } +} + +var _ yaml.Filter = Filter{} + +// Filter uses an FsSlice to modify fields on a single object +type Filter struct { + // FieldSpecList list of FieldSpecs to set + FsSlice types.FsSlice `yaml:"fsSlice"` + + // SetValue is called on each field that matches one of the FieldSpecs + SetValue SetFn + + // CreateKind is used to create fields that do not exist + CreateKind yaml.Kind +} + +func (fltr Filter) Filter(obj *yaml.RNode) (*yaml.RNode, error) { + for i := range fltr.FsSlice { + // apply this FieldSpec + // create a new filter for each iteration because they + // store internal state about the field paths + _, err := (&fieldSpecFilter{ + FieldSpec: fltr.FsSlice[i], + SetValue: fltr.SetValue, + CreateKind: fltr.CreateKind, + }).Filter(obj) + if err != nil { + return nil, err + } + } + return obj, nil +} + +// GetGVK parses the metadata into a GVK +func GetGVK(meta yaml.ResourceMeta) resid.Gvk { + // parse the group and version from the apiVersion field + var group, version string + parts := strings.SplitN(meta.APIVersion, "/", 2) + group = parts[0] + if len(parts) > 1 { + version = parts[1] + } + + return resid.Gvk{ + Group: group, + Version: version, + Kind: meta.Kind, + } +} diff --git a/api/filters/fsslice/fsslice_test.go b/api/filters/fsslice/fsslice_test.go new file mode 100644 index 000000000..a000fe14e --- /dev/null +++ b/api/filters/fsslice/fsslice_test.go @@ -0,0 +1,354 @@ +// Copyright 2019 The Kubernetes Authors. +// SPDX-License-Identifier: Apache-2.0 + +package fsslice_test + +import ( + "bytes" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "sigs.k8s.io/kustomize/api/filters/fsslice" + "sigs.k8s.io/kustomize/api/resid" + "sigs.k8s.io/kustomize/kyaml/kio" + "sigs.k8s.io/kustomize/kyaml/yaml" +) + +type TestCase struct { + name string + input string + expected string + filter fsslice.Filter + fsSlice string + error string +} + +var tests = []TestCase{ + { + name: "update", + fsSlice: ` +- path: a/b + group: foo + kind: Bar +`, + input: ` +apiVersion: foo/v1beta1 +kind: Bar +a: + b: c +`, + expected: ` +apiVersion: foo/v1beta1 +kind: Bar +a: + b: e +`, + filter: fsslice.Filter{ + SetValue: fsslice.SetScalar("e"), + }, + }, + + { + name: "update-kind-not-match", + fsSlice: ` +- path: a/b + group: foo + kind: Bar1 +`, + input: ` +apiVersion: foo/v1beta1 +kind: Bar2 +a: + b: c +`, + expected: ` +apiVersion: foo/v1beta1 +kind: Bar2 +a: + b: c +`, + filter: fsslice.Filter{ + SetValue: fsslice.SetScalar("e"), + }, + }, + + { + name: "update-group-not-match", + fsSlice: ` +- path: a/b + group: foo1 + kind: Bar +`, + input: ` +apiVersion: foo2/v1beta1 +kind: Bar +a: + b: c +`, + expected: ` +apiVersion: foo2/v1beta1 +kind: Bar +a: + b: c +`, + filter: fsslice.Filter{ + SetValue: fsslice.SetScalar("e"), + }, + }, + + { + name: "update-version-not-match", + fsSlice: ` +- path: a/b + group: foo + version: v1beta1 + kind: Bar +`, + input: ` +apiVersion: foo/v1beta2 +kind: Bar +a: + b: c +`, + expected: ` +apiVersion: foo/v1beta2 +kind: Bar +a: + b: c +`, + filter: fsslice.Filter{ + SetValue: fsslice.SetScalar("e"), + }, + }, + + { + name: "bad-version", + fsSlice: ` +- path: a/b + group: foo + version: v1beta1 + kind: Bar +`, + input: ` +apiVersion: foo/v1beta2/something +kind: Bar +a: + b: c +`, + expected: ` +apiVersion: foo/v1beta2/something +kind: Bar +a: + b: c +`, + filter: fsslice.Filter{ + SetValue: fsslice.SetScalar("e"), + }, + }, + + { + name: "bad-meta", + fsSlice: ` +- path: a/b + group: foo + version: v1beta1 + kind: Bar +`, + input: ` +a: + b: c +`, + filter: fsslice.Filter{ + SetValue: fsslice.SetScalar("e"), + }, + error: "missing Resource metadata", + }, + + { + name: "miss-match-type", + fsSlice: ` +- path: a/b/c + kind: Bar +`, + input: ` +kind: Bar +a: + b: a +`, + error: "obj kind: Bar\na:\n b: a\n at path a/b/c: unsupported yaml node", + filter: fsslice.Filter{ + SetValue: fsslice.SetScalar("e"), + }, + }, + + { + name: "add", + fsSlice: ` +- path: a/b/c/d + group: foo + create: true + kind: Bar +`, + input: ` +apiVersion: foo/v1beta1 +kind: Bar +a: {} +`, + expected: ` +apiVersion: foo/v1beta1 +kind: Bar +a: {b: {c: {d: e}}} +`, + filter: fsslice.Filter{ + SetValue: fsslice.SetScalar("e"), + CreateKind: yaml.ScalarNode, + }, + }, + + { + name: "update-in-sequence", + fsSlice: ` +- path: a/b[]/c/d + group: foo + kind: Bar +`, + input: ` +apiVersion: foo/v1beta1 +kind: Bar +a: + b: + - c: + d: a +`, + expected: ` +apiVersion: foo/v1beta1 +kind: Bar +a: + b: + - c: + d: e +`, + filter: fsslice.Filter{ + SetValue: fsslice.SetScalar("e"), + }, + }, + + // Don't create a sequence + { + name: "empty-sequence-no-create", + fsSlice: ` +- path: a/b[]/c/d + group: foo + create: true + kind: Bar +`, + input: ` +apiVersion: foo/v1beta1 +kind: Bar +a: {} +`, + expected: ` +apiVersion: foo/v1beta1 +kind: Bar +a: {} +`, + filter: fsslice.Filter{ + SetValue: fsslice.SetScalar("e"), + CreateKind: yaml.ScalarNode, + }, + }, + + // Create a new field for an element in a sequence + { + name: "empty-sequence-create", + fsSlice: ` +- path: a/b[]/c/d + group: foo + create: true + kind: Bar +`, + input: ` +apiVersion: foo/v1beta1 +kind: Bar +a: + b: + - c: {} +`, + expected: ` +apiVersion: foo/v1beta1 +kind: Bar +a: + b: + - c: {d: e} +`, + filter: fsslice.Filter{ + SetValue: fsslice.SetScalar("e"), + CreateKind: yaml.ScalarNode, + }, + }, +} + +func TestFilter_Filter(t *testing.T) { + for i := range tests { + test := tests[i] + t.Run(test.name, func(t *testing.T) { + err := yaml.Unmarshal([]byte(test.fsSlice), &test.filter.FsSlice) + if !assert.NoError(t, err) { + t.FailNow() + } + + out := &bytes.Buffer{} + rw := &kio.ByteReadWriter{ + Reader: bytes.NewBufferString(test.input), + Writer: out, + OmitReaderAnnotations: true, + } + + // run the filter + err = kio.Pipeline{ + Inputs: []kio.Reader{rw}, + Filters: []kio.Filter{kio.FilterAll(test.filter)}, + Outputs: []kio.Writer{rw}, + }.Execute() + if test.error != "" { + if !assert.EqualError(t, err, test.error) { + t.FailNow() + } + // stop rest of test + return + } + + if !assert.NoError(t, err) { + t.FailNow() + } + + // check results + if !assert.Equal(t, + strings.TrimSpace(test.expected), + strings.TrimSpace(out.String())) { + t.FailNow() + } + }) + } +} + +func TestGetGVK(t *testing.T) { + obj, err := yaml.Parse(` +apiVersion: apps/v1 +kind: Deployment +`) + if !assert.NoError(t, err) { + t.FailNow() + } + meta, err := obj.GetMeta() + if !assert.NoError(t, err) { + t.FailNow() + } + + gvk := fsslice.GetGVK(meta) + expected := resid.Gvk{Group: "apps", Version: "v1", Kind: "Deployment"} + if !assert.Equal(t, expected, gvk) { + t.FailNow() + } +}