diff --git a/.travis.yml b/.travis.yml index 550045319..57db382e9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -35,6 +35,7 @@ install: true script: - ./travis/pre-commit.sh + - ./travis/kyaml-pre-commit.sh # TBD. Suppressing for now. notifications: diff --git a/kyaml/Makefile b/kyaml/Makefile new file mode 100644 index 000000000..9bdd630b2 --- /dev/null +++ b/kyaml/Makefile @@ -0,0 +1,29 @@ +# Copyright 2019 The Kubernetes Authors. +# SPDX-License-Identifier: Apache-2.0 + +.PHONY: generate fix vet fmt test build tidy + +all: generate fix vet fmt test build tidy + +fix: + go fix ./... + +fmt: + go fmt ./... + +generate: + go generate ./... + +tidy: + go mod tidy + +lint: + (which $(go env GOPATH)/bin/golangci-lint || go get github.com/golangci/golangci-lint/cmd/golangci-lint@v1.19.1) + $(go env GOPATH)/bin/golangci-lint run ./... + +test: + go test -cover ./... + +vet: + go vet ./... + diff --git a/kyaml/copyutil/copyutil.go b/kyaml/copyutil/copyutil.go new file mode 100644 index 000000000..56554e850 --- /dev/null +++ b/kyaml/copyutil/copyutil.go @@ -0,0 +1,128 @@ +// Copyright 2019 The Kubernetes Authors. +// SPDX-License-Identifier: Apache-2.0 + +// The copyutil package contains libraries for copying directories of configuration. +package copyutil + +import ( + "bytes" + "io/ioutil" + "os" + "path/filepath" + "strings" + + "sigs.k8s.io/kustomize/kyaml/sets" +) + +// CopyDir copies a src directory to a dst directory. CopyDir skips copying the .git directory from the src. +func CopyDir(src string, dst string) error { + return filepath.Walk(src, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + // don't copy the .git dir + if path != src { + rel := strings.TrimPrefix(path, src) + if strings.HasPrefix(rel, string(filepath.Separator)+".git") { + return nil + } + } + + // path is an absolute path, rather than a path relative to src. + // e.g. if src is /path/to/package, then path might be /path/to/package/and/sub/dir + // we need the path relative to src `and/sub/dir` when we are copying the files to dest. + copyTo := strings.TrimPrefix(path, src) + + // make directories that don't exist + if info.IsDir() { + return os.MkdirAll(filepath.Join(dst, copyTo), info.Mode()) + } + + // copy file by reading and writing it + b, err := ioutil.ReadFile(filepath.Join(src, copyTo)) + if err != nil { + return err + } + err = ioutil.WriteFile(filepath.Join(dst, copyTo), b, info.Mode()) + if err != nil { + return err + } + + return nil + }) +} + +// Diff returns a list of files that differ between the source and destination. +// +// Diff is guaranteed to return a non-empty set if any files differ, but +// this set is not guaranteed to contain all differing files. +func Diff(sourceDir, destDir string) (sets.String, error) { + // get set of filenames in the package source + upstreamFiles := sets.String{} + err := filepath.Walk(sourceDir, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + + // skip git repo if it exists + if strings.Contains(path, ".git") { + return nil + } + + upstreamFiles.Insert(strings.TrimPrefix(strings.TrimPrefix(path, sourceDir), string(filepath.Separator))) + return nil + }) + if err != nil { + return sets.String{}, err + } + + // get set of filenames in the cloned package + localFiles := sets.String{} + err = filepath.Walk(destDir, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + + // skip git repo if it exists + if strings.Contains(path, ".git") { + return nil + } + + localFiles.Insert(strings.TrimPrefix(strings.TrimPrefix(path, destDir), string(filepath.Separator))) + return nil + }) + if err != nil { + return sets.String{}, err + } + + // verify the source and cloned packages have the same set of filenames + diff := upstreamFiles.SymmetricDifference(localFiles) + + // verify file contents match + for _, f := range upstreamFiles.Intersection(localFiles).List() { + fi, err := os.Stat(filepath.Join(destDir, f)) + if err != nil { + return diff, err + } + if fi.Mode().IsDir() { + // already checked that this directory exists in the local files + continue + } + + // compare upstreamFiles + b1, err := ioutil.ReadFile(filepath.Join(destDir, f)) + if err != nil { + return diff, err + } + b2, err := ioutil.ReadFile(filepath.Join(sourceDir, f)) + if err != nil { + return diff, err + } + if !bytes.Equal(b1, b2) { + diff.Insert(f) + } + } + + // return the differing files + return diff, nil +} diff --git a/kyaml/copyutil/copyutil_test.go b/kyaml/copyutil/copyutil_test.go new file mode 100644 index 000000000..b2025018c --- /dev/null +++ b/kyaml/copyutil/copyutil_test.go @@ -0,0 +1,199 @@ +// Copyright 2019 The Kubernetes Authors. +// SPDX-License-Identifier: Apache-2.0 + +package copyutil_test + +import ( + "io/ioutil" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + . "sigs.k8s.io/kustomize/kyaml/copyutil" +) + +// TestDiff_identical verifies identical directories return an empty set +func TestDiff_identical(t *testing.T) { + s, err := ioutil.TempDir("", "copyutilsrc") + assert.NoError(t, err) + d, err := ioutil.TempDir("", "copyutildest") + assert.NoError(t, err) + + err = os.Mkdir(filepath.Join(s, "a1"), 0700) + assert.NoError(t, err) + err = ioutil.WriteFile( + filepath.Join(s, "a1", "f.yaml"), []byte(`a`), 0600) + assert.NoError(t, err) + + err = os.Mkdir(filepath.Join(d, "a1"), 0700) + assert.NoError(t, err) + err = ioutil.WriteFile( + filepath.Join(d, "a1", "f.yaml"), []byte(`a`), 0600) + assert.NoError(t, err) + + diff, err := Diff(s, d) + assert.NoError(t, err) + assert.Empty(t, diff.List()) +} + +// TestDiff_additionalSourceFiles verifies if there are additional files +// in the source, the diff will contain them +func TestDiff_additionalSourceFiles(t *testing.T) { + s, err := ioutil.TempDir("", "copyutilsrc") + assert.NoError(t, err) + d, err := ioutil.TempDir("", "copyutildest") + assert.NoError(t, err) + + err = os.Mkdir(filepath.Join(s, "a1"), 0700) + assert.NoError(t, err) + + err = os.Mkdir(filepath.Join(s, "a2"), 0700) + assert.NoError(t, err) + + err = os.Mkdir(filepath.Join(d, "a1"), 0700) + assert.NoError(t, err) + + diff, err := Diff(s, d) + assert.NoError(t, err) + assert.ElementsMatch(t, diff.List(), []string{"a2"}) +} + +// TestDiff_additionalDestFiles verifies if there are additional files +// in the dest, the diff will contain them +func TestDiff_additionalDestFiles(t *testing.T) { + s, err := ioutil.TempDir("", "copyutilsrc") + assert.NoError(t, err) + d, err := ioutil.TempDir("", "copyutildest") + assert.NoError(t, err) + + err = os.Mkdir(filepath.Join(s, "a1"), 0700) + assert.NoError(t, err) + + err = os.Mkdir(filepath.Join(d, "a1"), 0700) + assert.NoError(t, err) + + err = os.Mkdir(filepath.Join(d, "a2"), 0700) + assert.NoError(t, err) + + diff, err := Diff(s, d) + assert.NoError(t, err) + assert.ElementsMatch(t, diff.List(), []string{"a2"}) + +} + +// TestDiff_srcDestContentsDiffer verifies if the file contents +// differ between the source and destination, the diff +// contains the differing files +func TestDiff_srcDestContentsDiffer(t *testing.T) { + s, err := ioutil.TempDir("", "copyutilsrc") + assert.NoError(t, err) + d, err := ioutil.TempDir("", "copyutildest") + assert.NoError(t, err) + + err = os.Mkdir(filepath.Join(s, "a1"), 0700) + assert.NoError(t, err) + err = ioutil.WriteFile( + filepath.Join(s, "a1", "f.yaml"), []byte(`a`), 0600) + assert.NoError(t, err) + + err = os.Mkdir(filepath.Join(d, "a1"), 0700) + assert.NoError(t, err) + err = ioutil.WriteFile( + filepath.Join(d, "a1", "f.yaml"), []byte(`b`), 0600) + assert.NoError(t, err) + + diff, err := Diff(s, d) + assert.NoError(t, err) + assert.ElementsMatch(t, diff.List(), []string{"a1/f.yaml"}) +} + +// TestDiff_srcDestContentsDifferInDirs verifies if identical files +// exist in different directories, they are included in the diff +func TestDiff_srcDestContentsDifferInDirs(t *testing.T) { + s, err := ioutil.TempDir("", "copyutilsrc") + assert.NoError(t, err) + d, err := ioutil.TempDir("", "copyutildest") + assert.NoError(t, err) + + err = os.Mkdir(filepath.Join(s, "a1"), 0700) + assert.NoError(t, err) + err = ioutil.WriteFile( + filepath.Join(s, "a1", "f.yaml"), []byte(`a`), 0600) + assert.NoError(t, err) + + err = os.Mkdir(filepath.Join(d, "b1"), 0700) + assert.NoError(t, err) + err = ioutil.WriteFile( + filepath.Join(d, "b1", "f.yaml"), []byte(`a`), 0600) + assert.NoError(t, err) + + diff, err := Diff(s, d) + assert.NoError(t, err) + assert.ElementsMatch(t, diff.List(), []string{ + "a1", "a1/f.yaml", "b1/f.yaml", "b1"}) +} + +// TestDiff_skipGitSrc verifies that .git directories in the source +// are not looked at +func TestDiff_skipGitSrc(t *testing.T) { + s, err := ioutil.TempDir("", "copyutilsrc") + assert.NoError(t, err) + d, err := ioutil.TempDir("", "copyutildest") + assert.NoError(t, err) + + err = os.Mkdir(filepath.Join(s, "a1"), 0700) + assert.NoError(t, err) + err = ioutil.WriteFile( + filepath.Join(s, "a1", "f.yaml"), []byte(`a`), 0600) + assert.NoError(t, err) + + // git should be ignored + err = os.Mkdir(filepath.Join(s, ".git"), 0700) + assert.NoError(t, err) + err = ioutil.WriteFile( + filepath.Join(s, ".git", "f.yaml"), []byte(`a`), 0600) + assert.NoError(t, err) + + err = os.Mkdir(filepath.Join(d, "a1"), 0700) + assert.NoError(t, err) + err = ioutil.WriteFile( + filepath.Join(d, "a1", "f.yaml"), []byte(`a`), 0600) + assert.NoError(t, err) + + diff, err := Diff(s, d) + assert.NoError(t, err) + assert.Empty(t, diff.List()) +} + +// TestDiff_skipGitDest verifies that .git directories in the destination +// are not looked at +func TestDiff_skipGitDest(t *testing.T) { + s, err := ioutil.TempDir("", "copyutilsrc") + assert.NoError(t, err) + d, err := ioutil.TempDir("", "copyutildest") + assert.NoError(t, err) + + err = os.Mkdir(filepath.Join(s, "a1"), 0700) + assert.NoError(t, err) + err = ioutil.WriteFile( + filepath.Join(s, "a1", "f.yaml"), []byte(`a`), 0600) + assert.NoError(t, err) + + err = os.Mkdir(filepath.Join(d, "a1"), 0700) + assert.NoError(t, err) + err = ioutil.WriteFile( + filepath.Join(d, "a1", "f.yaml"), []byte(`a`), 0600) + assert.NoError(t, err) + + // git should be ignored + err = os.Mkdir(filepath.Join(d, ".git"), 0700) + assert.NoError(t, err) + err = ioutil.WriteFile( + filepath.Join(d, ".git", "f.yaml"), []byte(`a`), 0600) + assert.NoError(t, err) + + diff, err := Diff(s, d) + assert.NoError(t, err) + assert.Empty(t, diff.List()) +} diff --git a/kyaml/go.mod b/kyaml/go.mod new file mode 100644 index 000000000..16723e205 --- /dev/null +++ b/kyaml/go.mod @@ -0,0 +1,11 @@ +module sigs.k8s.io/kustomize/kyaml + +go 1.12 + +require ( + github.com/davecgh/go-spew v1.1.1 + github.com/go-errors/errors v1.0.1 + github.com/stretchr/testify v1.4.0 + github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca + gopkg.in/yaml.v3 v3.0.0-20191026110619-0b21df46bc1d +) diff --git a/kyaml/go.sum b/kyaml/go.sum new file mode 100644 index 000000000..72708136d --- /dev/null +++ b/kyaml/go.sum @@ -0,0 +1,18 @@ +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-errors/errors v1.0.1 h1:LUHzmkK3GUKUrL/1gfBUxAHzcev3apQlezX/+O7ma6w= +github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca h1:1CFlNzQhALwjS9mBAUkycX616GzgsuYUOCHA5+HSlXI= +github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20191026110619-0b21df46bc1d h1:LCPbGQ34PMrwad11aMZ+dbz5SAsq/0ySjRwQ8I9Qwd8= +gopkg.in/yaml.v3 v3.0.0-20191026110619-0b21df46bc1d/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/kyaml/kio/byteio_reader.go b/kyaml/kio/byteio_reader.go new file mode 100644 index 000000000..1f6503a2e --- /dev/null +++ b/kyaml/kio/byteio_reader.go @@ -0,0 +1,198 @@ +// Copyright 2019 The Kubernetes Authors. +// SPDX-License-Identifier: Apache-2.0 + +package kio + +import ( + "bytes" + "fmt" + "io" + "sort" + "strings" + + "sigs.k8s.io/kustomize/kyaml/kio/kioutil" + "sigs.k8s.io/kustomize/kyaml/yaml" +) + +const ( + ResourceListKind = "ResourceList" + ResourceListApiVersion = "kyaml.kustomize.dev/v1alpha1" +) + +// ByteReadWriter reads from an input and writes to an output +type ByteReadWriter struct { + // Reader is where ResourceNodes are decoded from. + Reader io.Reader + + // Writer is where ResourceNodes are encoded. + Writer io.Writer + + // OmitReaderAnnotations will configures Read to skip setting the kyaml.kustomize.dev/kio/index + // annotation on Resources as they are Read. + OmitReaderAnnotations bool + + // KeepReaderAnnotations if set will keep the Reader specific annotations when writing + // the Resources, otherwise they will be cleared. + KeepReaderAnnotations bool + + // Style is a style that is set on the Resource Node Document. + Style yaml.Style + + FunctionConfig *yaml.RNode + + WrappingApiVersion string + WrappingKind string +} + +func (rw *ByteReadWriter) Read() ([]*yaml.RNode, error) { + b := &ByteReader{ + Reader: rw.Reader, + OmitReaderAnnotations: rw.OmitReaderAnnotations, + } + val, err := b.Read() + rw.FunctionConfig = b.FunctionConfig + rw.WrappingApiVersion = b.WrappingApiVersion + rw.WrappingKind = b.WrappingKind + return val, err +} + +func (rw *ByteReadWriter) Write(nodes []*yaml.RNode) error { + return ByteWriter{ + Writer: rw.Writer, + KeepReaderAnnotations: rw.KeepReaderAnnotations, + Style: rw.Style, + FunctionConfig: rw.FunctionConfig, + WrappingApiVersion: rw.WrappingApiVersion, + WrappingKind: rw.WrappingKind, + }.Write(nodes) +} + +// ByteReader decodes ResourceNodes from bytes. +// By default, Read will set the kyaml.kustomize.dev/kio/index annotation on each RNode as it +// is read so they can be written back in the same order. +type ByteReader struct { + // Reader is where ResourceNodes are decoded from. + Reader io.Reader + + // OmitReaderAnnotations will configures Read to skip setting the kyaml.kustomize.dev/kio/index + // annotation on Resources as they are Read. + OmitReaderAnnotations bool + + // SetAnnotations is a map of caller specified annotations to set on resources as they are read + // These are independent of the annotations controlled by OmitReaderAnnotations + SetAnnotations map[string]string + + FunctionConfig *yaml.RNode + + WrappingApiVersion string + WrappingKind string +} + +var _ Reader = &ByteReader{} + +func (r *ByteReader) Read() ([]*yaml.RNode, error) { + output := ResourceNodeSlice{} + + // by manually splitting resources -- otherwise the decoder will get the Resource + // boundaries wrong for header comments. + input := &bytes.Buffer{} + _, err := io.Copy(input, r.Reader) + if err != nil { + return nil, err + } + values := strings.Split(input.String(), "\n---\n") + + index := 0 + for i := range values { + decoder := yaml.NewDecoder(bytes.NewBufferString(values[i])) + node, err := r.decode(index, decoder) + if err == io.EOF { + continue + } + if err != nil { + return nil, err + } + if yaml.IsMissingOrNull(node) { + // empty value + continue + } + + // ok if no metadata -- assume not an InputList + meta, _ := node.GetMeta() + + // the elements are wrapped in an InputList, unwrap them + // don't check apiVersion, we haven't standardized on the domain + if (meta.Kind == ResourceListKind || meta.Kind == "List") && + node.Field("items") != nil { + r.WrappingKind = meta.Kind + r.WrappingApiVersion = meta.ApiVersion + + // unwrap the list + fc := node.Field("functionConfig") + if fc != nil { + r.FunctionConfig = fc.Value + } + + items := node.Field("items") + if items != nil { + for i := range items.Value.Content() { + // add items + output = append(output, yaml.NewRNode(items.Value.Content()[i])) + } + + } + continue + } + + // add the node to the list + output = append(output, node) + + // increment the index annotation value + index++ + } + return output, nil +} + +func isEmptyDocument(node *yaml.Node) bool { + // node is a Document with no content -- e.g. "---\n---" + return node.Kind == yaml.DocumentNode && + node.Content[0].Tag == yaml.NullNodeTag +} + +func (r *ByteReader) decode(index int, decoder *yaml.Decoder) (*yaml.RNode, error) { + node := &yaml.Node{} + err := decoder.Decode(node) + if err == io.EOF { + return nil, io.EOF + } + if err != nil { + return nil, err + } + + if isEmptyDocument(node) { + return nil, nil + } + + // set annotations on the read Resources + // sort the annotations by key so the output Resources is consistent (otherwise the + // annotations will be in a random order) + n := yaml.NewRNode(node) + if r.SetAnnotations == nil { + r.SetAnnotations = map[string]string{} + } + if !r.OmitReaderAnnotations { + r.SetAnnotations[kioutil.IndexAnnotation] = fmt.Sprintf("%d", index) + } + var keys []string + for k := range r.SetAnnotations { + keys = append(keys, k) + } + sort.Strings(keys) + for _, k := range keys { + _, err = n.Pipe(yaml.SetAnnotation(k, r.SetAnnotations[k])) + if err != nil { + return nil, err + } + } + return yaml.NewRNode(node), nil +} diff --git a/kyaml/kio/byteio_reader_test.go b/kyaml/kio/byteio_reader_test.go new file mode 100644 index 000000000..61edfb8e9 --- /dev/null +++ b/kyaml/kio/byteio_reader_test.go @@ -0,0 +1,310 @@ +// Copyright 2019 The Kubernetes Authors. +// SPDX-License-Identifier: Apache-2.0 + +package kio_test + +import ( + "bytes" + "testing" + + "github.com/stretchr/testify/assert" + . "sigs.k8s.io/kustomize/kyaml/kio" +) + +// getByteReaderTestInput returns test input +func getByteReaderTestInput(t *testing.T) *bytes.Buffer { + b := &bytes.Buffer{} + _, err := b.WriteString(` +--- +a: b # first resource +c: d +--- +# second resource +e: f +g: +- h +--- +--- +i: j +`) + if !assert.NoError(t, err) { + assert.FailNow(t, "") + } + return b +} + +func TestByteReader_Read_wrappedResourceßßList(t *testing.T) { + r := &ByteReader{Reader: bytes.NewBufferString(`apiVersion: kyaml.kustomize.dev/v1alpha1 +kind: ResourceList +functionConfig: + foo: bar + elems: + - a + - b + - c +items: +- kind: Deployment + spec: + replicas: 1 +- kind: Service + spec: + selectors: + foo: bar +`)} + nodes, err := r.Read() + if !assert.NoError(t, err) { + return + } + + // verify the contents + if !assert.Len(t, nodes, 2) { + return + } + expected := []string{ + `kind: Deployment +spec: + replicas: 1 +`, + `kind: Service +spec: + selectors: + foo: bar +`, + } + for i := range nodes { + if !assert.Equal(t, expected[i], nodes[i].MustString()) { + return + } + } + + // verify the function config + assert.Equal(t, `foo: bar +elems: +- a +- b +- c +`, r.FunctionConfig.MustString()) + + assert.Equal(t, ResourceListKind, r.WrappingKind) + assert.Equal(t, ResourceListApiVersion, r.WrappingApiVersion) + +} + +func TestByteReader_Read_wrappedList(t *testing.T) { + r := &ByteReader{Reader: bytes.NewBufferString(`apiVersion: v1 +kind: List +items: +- kind: Deployment + spec: + replicas: 1 +- kind: Service + spec: + selectors: + foo: bar +`)} + nodes, err := r.Read() + if !assert.NoError(t, err) { + return + } + + // verify the contents + if !assert.Len(t, nodes, 2) { + return + } + expected := []string{ + `kind: Deployment +spec: + replicas: 1 +`, + `kind: Service +spec: + selectors: + foo: bar +`, + } + for i := range nodes { + if !assert.Equal(t, expected[i], nodes[i].MustString()) { + return + } + } + + // verify the function config + assert.Nil(t, r.FunctionConfig) + assert.Equal(t, "List", r.WrappingKind) + assert.Equal(t, "v1", r.WrappingApiVersion) +} + +// TestByteReader_Read tests the default Read behavior +// - Resources are read into a slice +// - ReaderAnnotations are set on the ResourceNodes +func TestByteReader_Read(t *testing.T) { + nodes, err := (&ByteReader{Reader: getByteReaderTestInput(t)}).Read() + if !assert.NoError(t, err) { + return + } + + if !assert.Len(t, nodes, 3) { + return + } + expected := []string{ + `a: b # first resource +c: d +metadata: + annotations: + kyaml.kustomize.dev/kio/index: 0 +`, + `# second resource +e: f +g: +- h +metadata: + annotations: + kyaml.kustomize.dev/kio/index: 1 +`, + `i: j +metadata: + annotations: + kyaml.kustomize.dev/kio/index: 2 +`, + } + for i := range nodes { + val, err := nodes[i].String() + if !assert.NoError(t, err) { + return + } + if !assert.Equal(t, expected[i], val) { + return + } + } +} + +// TestByteReader_Read_omitReaderAnnotations tests +// - Resources are read into a slice +// - ReaderAnnotations are not set on the ResourceNodes +func TestByteReader_Read_omitReaderAnnotations(t *testing.T) { + nodes, err := (&ByteReader{ + Reader: getByteReaderTestInput(t), + OmitReaderAnnotations: true}).Read() + if !assert.NoError(t, err) { + return + } + + // should have parsed 3 resources + if !assert.Len(t, nodes, 3) { + return + } + expected := []string{ + "a: b # first resource\nc: d\n", + "# second resource\ne: f\ng:\n- h\n", + "i: j\n", + } + for i := range nodes { + val, err := nodes[i].String() + if !assert.NoError(t, err) { + return + } + if !assert.Equal(t, expected[i], val) { + return + } + } +} + +// TestByteReader_Read_omitReaderAnnotations tests +// - Resources are read into a slice +// - ReaderAnnotations are NOT set on the ResourceNodes +// - Additional annotations ARE set on the ResourceNodes +func TestByteReader_Read_setAnnotationsOmitReaderAnnotations(t *testing.T) { + nodes, err := (&ByteReader{ + Reader: getByteReaderTestInput(t), + SetAnnotations: map[string]string{"foo": "bar"}, + OmitReaderAnnotations: true, + }).Read() + if !assert.NoError(t, err) { + return + } + + if !assert.Len(t, nodes, 3) { + return + } + expected := []string{ + `a: b # first resource +c: d +metadata: + annotations: + foo: bar +`, + `# second resource +e: f +g: +- h +metadata: + annotations: + foo: bar +`, + `i: j +metadata: + annotations: + foo: bar +`, + } + for i := range nodes { + val, err := nodes[i].String() + if !assert.NoError(t, err) { + return + } + if !assert.Equal(t, expected[i], val) { + return + } + } +} + +// TestByteReader_Read_omitReaderAnnotations tests +// - Resources are read into a slice +// - ReaderAnnotations ARE set on the ResourceNodes +// - Additional annotations ARE set on the ResourceNodes +func TestByteReader_Read_setAnnotations(t *testing.T) { + nodes, err := (&ByteReader{ + Reader: getByteReaderTestInput(t), + SetAnnotations: map[string]string{"foo": "bar"}, + }).Read() + if !assert.NoError(t, err) { + return + } + + if !assert.Len(t, nodes, 3) { + return + } + expected := []string{ + `a: b # first resource +c: d +metadata: + annotations: + foo: bar + kyaml.kustomize.dev/kio/index: 0 +`, + `# second resource +e: f +g: +- h +metadata: + annotations: + foo: bar + kyaml.kustomize.dev/kio/index: 1 +`, + `i: j +metadata: + annotations: + foo: bar + kyaml.kustomize.dev/kio/index: 2 +`, + } + for i := range nodes { + val, err := nodes[i].String() + if !assert.NoError(t, err) { + return + } + if !assert.Equal(t, expected[i], val) { + return + } + } +} diff --git a/kyaml/kio/byteio_writer.go b/kyaml/kio/byteio_writer.go new file mode 100644 index 000000000..2d05957bf --- /dev/null +++ b/kyaml/kio/byteio_writer.go @@ -0,0 +1,122 @@ +// Copyright 2019 The Kubernetes Authors. +// SPDX-License-Identifier: Apache-2.0 + +package kio + +import ( + "io" + + "sigs.k8s.io/kustomize/kyaml/kio/kioutil" + "sigs.k8s.io/kustomize/kyaml/yaml" +) + +// Writer writes ResourceNodes to bytes. +type ByteWriter struct { + // Writer is where ResourceNodes are encoded. + Writer io.Writer + + // KeepReaderAnnotations if set will keep the Reader specific annotations when writing + // the Resources, otherwise they will be cleared. + KeepReaderAnnotations bool + + // ClearAnnotations is a list of annotations to clear when writing the Resources. + ClearAnnotations []string + + // Style is a style that is set on the Resource Node Document. + Style yaml.Style + + // FunctionConfig is the function config for an ResourceList. If non-nil + // wrap the results in an ResourceList. + FunctionConfig *yaml.RNode + + // WrappingKind if set will cause ByteWriter to wrap the Resources in + // an 'items' field in this kind. e.g. if WrappingKind is 'List', + // ByteWriter will wrap the Resources in a List .items field. + WrappingKind string + + // WrappingApiVersion is the apiVersion for WrappingKind + WrappingApiVersion string + + // Sort if set, will cause ByteWriter to sort the the nodes before writing them. + Sort bool +} + +var _ Writer = ByteWriter{} + +func (w ByteWriter) Write(nodes []*yaml.RNode) error { + if w.Sort { + if err := kioutil.SortNodes(nodes); err != nil { + return err + } + } + + encoder := yaml.NewEncoder(w.Writer) + defer encoder.Close() + for i := range nodes { + + // clean resources by removing annotations set by the Reader + if !w.KeepReaderAnnotations { + _, err := nodes[i].Pipe(yaml.ClearAnnotation(kioutil.IndexAnnotation)) + if err != nil { + return err + } + } + for _, a := range w.ClearAnnotations { + _, err := nodes[i].Pipe(yaml.ClearAnnotation(a)) + if err != nil { + return err + } + } + + // TODO(pwittrock): factor this into a a common module for pruning empty values + _, err := nodes[i].Pipe(yaml.Lookup("metadata"), yaml.FieldClearer{ + Name: "annotations", IfEmpty: true}) + if err != nil { + return err + } + _, err = nodes[i].Pipe(yaml.FieldClearer{Name: "metadata", IfEmpty: true}) + if err != nil { + return err + } + + if w.Style != 0 { + nodes[i].YNode().Style = w.Style + } + } + + // don't wrap the elements + if w.WrappingKind == "" { + for i := range nodes { + err := encoder.Encode(nodes[i].Document()) + if err != nil { + return err + } + } + return nil + } + // wrap the elements in a list + items := &yaml.Node{Kind: yaml.SequenceNode} + list := &yaml.Node{ + Kind: yaml.MappingNode, + Style: w.Style, + Content: []*yaml.Node{ + {Kind: yaml.ScalarNode, Value: "apiVersion"}, + {Kind: yaml.ScalarNode, Value: w.WrappingApiVersion}, + {Kind: yaml.ScalarNode, Value: "kind"}, + {Kind: yaml.ScalarNode, Value: w.WrappingKind}, + {Kind: yaml.ScalarNode, Value: "items"}, items, + }} + if w.FunctionConfig != nil { + list.Content = append(list.Content, + &yaml.Node{Kind: yaml.ScalarNode, Value: "functionConfig"}, + w.FunctionConfig.YNode()) + } + doc := &yaml.Node{ + Kind: yaml.DocumentNode, + Style: yaml.FoldedStyle, + Content: []*yaml.Node{list}} + for i := range nodes { + items.Content = append(items.Content, nodes[i].YNode()) + } + return encoder.Encode(doc) +} diff --git a/kyaml/kio/byteio_writer_test.go b/kyaml/kio/byteio_writer_test.go new file mode 100644 index 000000000..391d2ddae --- /dev/null +++ b/kyaml/kio/byteio_writer_test.go @@ -0,0 +1,284 @@ +// Copyright 2019 The Kubernetes Authors. +// SPDX-License-Identifier: Apache-2.0 + +package kio + +import ( + "bytes" + "testing" + + "github.com/stretchr/testify/assert" + "sigs.k8s.io/kustomize/kyaml/yaml" +) + +// TestByteWriter_Write_withoutAnnotations tests: +// - Resource Config ordering is preserved if no annotations are present +func TestByteWriter_Write_wrapped(t *testing.T) { + node1, err := yaml.Parse(`a: b #first +`) + if !assert.NoError(t, err) { + return + } + node2, err := yaml.Parse(`c: d # second +`) + if !assert.NoError(t, err) { + return + } + node3, err := yaml.Parse(`e: f +g: + h: + - i # has a list + - j +`) + if !assert.NoError(t, err) { + return + } + + buff := &bytes.Buffer{} + err = ByteWriter{ + Sort: true, + Writer: buff, + FunctionConfig: node3, + WrappingKind: ResourceListKind, + WrappingApiVersion: ResourceListApiVersion}. + Write([]*yaml.RNode{node2, node1}) + if !assert.NoError(t, err) { + return + } + assert.Equal(t, `apiVersion: kyaml.kustomize.dev/v1alpha1 +kind: ResourceList +items: +- c: d # second +- a: b #first +functionConfig: + e: f + g: + h: + - i # has a list + - j +`, buff.String()) +} + +// TestByteWriter_Write_withoutAnnotations tests: +// - Resource Config ordering is preserved if no annotations are present +func TestByteWriter_Write_withoutAnnotations(t *testing.T) { + node1, err := yaml.Parse(`a: b #first +`) + if !assert.NoError(t, err) { + return + } + node2, err := yaml.Parse(`c: d # second +`) + if !assert.NoError(t, err) { + return + } + node3, err := yaml.Parse(`e: f +g: + h: + - i # has a list + - j +`) + if !assert.NoError(t, err) { + return + } + + buff := &bytes.Buffer{} + err = ByteWriter{Writer: buff}. + Write([]*yaml.RNode{node2, node3, node1}) + if !assert.NoError(t, err) { + return + } + assert.Equal(t, `c: d # second +--- +e: f +g: + h: + - i # has a list + - j +--- +a: b #first +`, buff.String()) +} + +// TestByteWriter_Write_withAnnotationsKeepAnnotations tests: +// - Resource Config is sorted by annotations if present +// - IndexAnnotations are retained +func TestByteWriter_Write_withAnnotationsKeepAnnotations(t *testing.T) { + node1, err := yaml.Parse(`a: b #first +metadata: + annotations: + kyaml.kustomize.dev/kio/index: 0 + kyaml.kustomize.dev/kio/path: "a/b/a_test.yaml" +`) + if !assert.NoError(t, err) { + return + } + node2, err := yaml.Parse(`c: d # second +metadata: + annotations: + kyaml.kustomize.dev/kio/index: 1 + kyaml.kustomize.dev/kio/path: "a/b/a_test.yaml" +`) + if !assert.NoError(t, err) { + return + } + node3, err := yaml.Parse(`e: f +g: + h: + - i # has a list + - j +metadata: + annotations: + kyaml.kustomize.dev/kio/index: 0 + kyaml.kustomize.dev/kio/path: "a/b/b_test.yaml" +`) + if !assert.NoError(t, err) { + return + } + + buff := &bytes.Buffer{} + err = ByteWriter{Sort: true, Writer: buff, KeepReaderAnnotations: true}. + Write([]*yaml.RNode{node2, node3, node1}) + if !assert.NoError(t, err) { + return + } + assert.Equal(t, `a: b #first +metadata: + annotations: + kyaml.kustomize.dev/kio/index: 0 + kyaml.kustomize.dev/kio/path: "a/b/a_test.yaml" +--- +c: d # second +metadata: + annotations: + kyaml.kustomize.dev/kio/index: 1 + kyaml.kustomize.dev/kio/path: "a/b/a_test.yaml" +--- +e: f +g: + h: + - i # has a list + - j +metadata: + annotations: + kyaml.kustomize.dev/kio/index: 0 + kyaml.kustomize.dev/kio/path: "a/b/b_test.yaml" +`, buff.String()) +} + +// TestByteWriter_Write_withAnnotations tests: +// - Resource Config is sorted by annotations if present +// - IndexAnnotations are pruned +func TestByteWriter_Write_withAnnotations(t *testing.T) { + node1, err := yaml.Parse(`a: b #first +metadata: + annotations: + kyaml.kustomize.dev/kio/index: 0 + kyaml.kustomize.dev/kio/path: "a/b/a_test.yaml" +`) + if !assert.NoError(t, err) { + return + } + node2, err := yaml.Parse(`c: d # second +metadata: + annotations: + kyaml.kustomize.dev/kio/index: 1 + kyaml.kustomize.dev/kio/path: "a/b/a_test.yaml" +`) + if !assert.NoError(t, err) { + return + } + node3, err := yaml.Parse(`e: f +g: + h: + - i # has a list + - j +metadata: + annotations: + kyaml.kustomize.dev/kio/index: 0 + kyaml.kustomize.dev/kio/path: "a/b/b_test.yaml" +`) + if !assert.NoError(t, err) { + return + } + + buff := &bytes.Buffer{} + err = ByteWriter{Sort: true, Writer: buff}. + Write([]*yaml.RNode{node2, node3, node1}) + if !assert.NoError(t, err) { + return + } + assert.Equal(t, `a: b #first +metadata: + annotations: + kyaml.kustomize.dev/kio/path: "a/b/a_test.yaml" +--- +c: d # second +metadata: + annotations: + kyaml.kustomize.dev/kio/path: "a/b/a_test.yaml" +--- +e: f +g: + h: + - i # has a list + - j +metadata: + annotations: + kyaml.kustomize.dev/kio/path: "a/b/b_test.yaml" +`, buff.String()) +} + +// TestByteWriter_Write_partialValues tests: +// - Resource Config is sorted when annotations are present on some but not all ResourceNodes +func TestByteWriter_Write_partialAnnotations(t *testing.T) { + node1, err := yaml.Parse(`a: b #first +metadata: + annotations: + kyaml.kustomize.dev/kio/path: "a/b/a_test.yaml" +`) + if !assert.NoError(t, err) { + return + } + node2, err := yaml.Parse(`c: d # second +metadata: + annotations: + kyaml.kustomize.dev/kio/index: 1 + kyaml.kustomize.dev/kio/path: "a/b/a_test.yaml" +`) + if !assert.NoError(t, err) { + return + } + node3, err := yaml.Parse(`e: f +g: + h: + - i # has a list + - j +`) + if !assert.NoError(t, err) { + return + } + + buff := &bytes.Buffer{} + rw := ByteWriter{Sort: true, Writer: buff} + err = rw.Write([]*yaml.RNode{node2, node3, node1}) + if !assert.NoError(t, err) { + return + } + assert.Equal(t, `e: f +g: + h: + - i # has a list + - j +--- +a: b #first +metadata: + annotations: + kyaml.kustomize.dev/kio/path: "a/b/a_test.yaml" +--- +c: d # second +metadata: + annotations: + kyaml.kustomize.dev/kio/path: "a/b/a_test.yaml" +`, buff.String()) +} diff --git a/kyaml/kio/example_test.go b/kyaml/kio/example_test.go new file mode 100644 index 000000000..d0c27ec51 --- /dev/null +++ b/kyaml/kio/example_test.go @@ -0,0 +1,110 @@ +// Copyright 2019 The Kubernetes Authors. +// SPDX-License-Identifier: Apache-2.0 + +package kio_test + +import ( + "bytes" + "log" + "os" + + "sigs.k8s.io/kustomize/kyaml/kio" + "sigs.k8s.io/kustomize/kyaml/yaml" +) + +func Example() { + input := bytes.NewReader([]byte(`apiVersion: apps/v1 +kind: Deployment +metadata: + name: nginx + labels: + app: nginx +spec: + replicas: 3 + selector: + matchLabels: + app: nginx + template: + metadata: + labels: + app: nginx + spec: + containers: + - name: nginx + image: nginx:1.7.9 + ports: + - containerPort: 80 +--- +apiVersion: v1 +kind: Service +metadata: + name: nginx +spec: + selector: + app: nginx + ports: + - protocol: TCP + port: 80 + targetPort: 80 +`)) + + // setAnnotationFn + setAnnotationFn := kio.FilterFunc(func(operand []*yaml.RNode) ([]*yaml.RNode, error) { + for i := range operand { + resource := operand[i] + _, err := resource.Pipe(yaml.SetAnnotation("foo", "bar")) + if err != nil { + return nil, err + } + } + return operand, nil + }) + + err := kio.Pipeline{ + Inputs: []kio.Reader{&kio.ByteReader{Reader: input}}, + Filters: []kio.Filter{setAnnotationFn}, + Outputs: []kio.Writer{kio.ByteWriter{Writer: os.Stdout}}, + }.Execute() + if err != nil { + log.Fatal(err) + } + + // Output: + // apiVersion: apps/v1 + // kind: Deployment + // metadata: + // name: nginx + // labels: + // app: nginx + // annotations: + // foo: bar + // spec: + // replicas: 3 + // selector: + // matchLabels: + // app: nginx + // template: + // metadata: + // labels: + // app: nginx + // spec: + // containers: + // - name: nginx + // image: nginx:1.7.9 + // ports: + // - containerPort: 80 + // --- + // apiVersion: v1 + // kind: Service + // metadata: + // name: nginx + // annotations: + // foo: bar + // spec: + // selector: + // app: nginx + // ports: + // - protocol: TCP + // port: 80 + // targetPort: 80 +} diff --git a/kyaml/kio/filters/container.go b/kyaml/kio/filters/container.go new file mode 100644 index 000000000..9749a9deb --- /dev/null +++ b/kyaml/kio/filters/container.go @@ -0,0 +1,171 @@ +// Copyright 2019 The Kubernetes Authors. +// SPDX-License-Identifier: Apache-2.0 + +package filters + +import ( + "bytes" + "os" + "os/exec" + "regexp" + "strings" + + "sigs.k8s.io/kustomize/kyaml/kio" + + "sigs.k8s.io/kustomize/kyaml/yaml" +) + +// GrepFilter filters Resources using a container image. +// The container must start a process that reads the list of +// input Resources from stdin, reads the Configuration from the env +// API_CONFIG, and writes the filtered Resources to stdout. +// If there is a error or validation failure, the process must exit +// non-zero. +// The full set of environment variables from the parent process +// are passed to the container. +type ContainerFilter struct { + // Image is the container image to use to create a container. + Image string `yaml:"image,omitempty"` + + // Config is the API configuration for the container and passed through the + // API_CONFIG env var to the container. + // Typically a Kubernetes style Resource Config. + Config *yaml.RNode `yaml:"config,omitempty"` + + // args may be specified by tests to override how a container is spawned + args []string + + checkInput func(string) +} + +// GrepFilter implements kio.GrepFilter +func (c *ContainerFilter) Filter(input []*yaml.RNode) ([]*yaml.RNode, error) { + // get the command to filter the Resources + cmd, err := c.getCommand() + if err != nil { + return nil, err + } + + in := &bytes.Buffer{} + out := &bytes.Buffer{} + + // write the input + err = kio.ByteWriter{ + WrappingApiVersion: kio.ResourceListApiVersion, + WrappingKind: kio.ResourceListKind, + Writer: in, KeepReaderAnnotations: true, FunctionConfig: c.Config}.Write(input) + if err != nil { + return nil, err + } + + // capture the command stdout for the return value + r := &kio.ByteReader{Reader: out} + + // do the filtering + if c.checkInput != nil { + c.checkInput(in.String()) + } + cmd.Stdin = in + cmd.Stdout = out + if err := cmd.Run(); err != nil { + return nil, err + } + + return r.Read() +} + +// getArgs returns the command + args to run to spawn the container +func (c *ContainerFilter) getArgs() []string { + // run the container using docker. this is simpler than using the docker + // libraries, and ensures things like auth work the same as if the container + // was run from the cli. + args := []string{"docker", "run", + "--rm", // delete the container afterward + "-i", "-a", "STDIN", "-a", "STDOUT", "-a", "STDERR", // attach stdin, stdout, stderr + + // added security options + "--network", "none", // disable the network + "--user", "nobody", // run as nobody + // don't make fs readonly because things like heredoc rely on writing tmp files + "--security-opt=no-new-privileges", // don't allow the user to escalate privileges + } + + // export the local environment vars to the container + for _, pair := range os.Environ() { + args = append(args, "-e", strings.Split(pair, "=")[0]) + } + return append(args, c.Image) + +} + +// getCommand returns a command which will apply the GrepFilter using the container image +func (c *ContainerFilter) getCommand() (*exec.Cmd, error) { + // encode the filter command API configuration + cfg := &bytes.Buffer{} + if err := func() error { + e := yaml.NewEncoder(cfg) + defer e.Close() + // make it fit on a single line + c.Config.YNode().Style = yaml.FlowStyle + return e.Encode(c.Config.YNode()) + }(); err != nil { + return nil, err + } + + if len(c.args) == 0 { + c.args = c.getArgs() + } + + cmd := exec.Command(c.args[0], c.args[1:]...) + cmd.Stderr = os.Stderr + cmd.Env = os.Environ() + + // set stderr for err messaging + return cmd, nil +} + +// IsReconcilerFilter filters Resources based on whether or not they are Reconciler Resource. +// Resources with an apiVersion starting with '*.gcr.io', 'gcr.io' or 'docker.io' are considered +// Reconciler Resources. +type IsReconcilerFilter struct { + // ExcludeReconcilers if set to true, then Reconcilers will be excluded -- e.g. + // Resources with a reconcile container through the apiVersion (gcr.io prefix) or + // through the annotations + ExcludeReconcilers bool `yaml:"excludeReconcilers,omitempty"` + + // IncludeNonReconcilers if set to true, the NonReconciler will be included. + IncludeNonReconcilers bool `yaml:"includeNonReconcilers,omitempty"` +} + +// Filter implements kio.Filter +func (c *IsReconcilerFilter) Filter(inputs []*yaml.RNode) ([]*yaml.RNode, error) { + var out []*yaml.RNode + for i := range inputs { + isContainerResource := GetContainerName(inputs[i]) != "" + if isContainerResource && !c.ExcludeReconcilers { + out = append(out, inputs[i]) + } + if !isContainerResource && c.IncludeNonReconcilers { + out = append(out, inputs[i]) + } + } + return out, nil +} + +// GetContainerName returns the container image for an API if one exists +func GetContainerName(n *yaml.RNode) string { + meta, _ := n.GetMeta() + container := meta.Annotations["kyaml.kustomize.dev/container"] + if container != "" { + return container + } + + if match.MatchString(meta.ApiVersion) { + return meta.ApiVersion + } + + return "" +} + +// match specifies the set of apiVersions to recognize as being container images +var match = regexp.MustCompile(`(docker\.io|.*\.?gcr\.io)/.*(:.*)?`) diff --git a/kyaml/kio/filters/container_test.go b/kyaml/kio/filters/container_test.go new file mode 100644 index 000000000..0a294ee3a --- /dev/null +++ b/kyaml/kio/filters/container_test.go @@ -0,0 +1,281 @@ +// Copyright 2019 The Kubernetes Authors. +// SPDX-License-Identifier: Apache-2.0 + +package filters + +import ( + "bytes" + "os" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "sigs.k8s.io/kustomize/kyaml/kio" + "sigs.k8s.io/kustomize/kyaml/yaml" +) + +func TestFilter_command(t *testing.T) { + cfg, err := yaml.Parse(`apiversion: apps/v1 +kind: Deployment +metadata: + name: foo +`) + if !assert.NoError(t, err) { + return + } + instance := &ContainerFilter{ + Image: "example.com:version", + Config: cfg, + } + os.Setenv("KYAML_TEST", "FOO") + cmd, err := instance.getCommand() + if !assert.NoError(t, err) { + return + } + + expected := []string{ + "docker", "run", + "--rm", + "-i", "-a", "STDIN", "-a", "STDOUT", "-a", "STDERR", + "--network", "none", + "--user", "nobody", + "--security-opt=no-new-privileges", + } + for _, e := range os.Environ() { + // the process env + expected = append(expected, "-e", strings.Split(e, "=")[0]) + } + expected = append(expected, "example.com:version") + assert.Equal(t, expected, cmd.Args) + + foundKyaml := false + for _, e := range cmd.Env { + // verify the command has the right environment variables to pass to the container + split := strings.Split(e, "=") + if split[0] == "KYAML_TEST" { + assert.Equal(t, "FOO", split[1]) + foundKyaml = true + } + } + assert.True(t, foundKyaml) +} + +func TestFilter_Filter(t *testing.T) { + cfg, err := yaml.Parse(`apiVersion: apps/v1 +kind: Deployment +metadata: + name: foo +`) + if !assert.NoError(t, err) { + return + } + + input, err := (&kio.ByteReader{Reader: bytes.NewBufferString(` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: deployment-foo +--- +apiVersion: v1 +kind: Service +metadata: + name: service-foo +`)}).Read() + if !assert.NoError(t, err) { + return + } + + called := false + result, err := (&ContainerFilter{ + Image: "example.com:version", + Config: cfg, + args: []string{"sed", "s/Deployment/StatefulSet/g"}, + checkInput: func(s string) { + called = true + if !assert.Equal(t, `apiVersion: kyaml.kustomize.dev/v1alpha1 +kind: ResourceList +items: +- apiVersion: apps/v1 + kind: Deployment + metadata: + name: deployment-foo + annotations: + kyaml.kustomize.dev/kio/index: 0 +- apiVersion: v1 + kind: Service + metadata: + name: service-foo + annotations: + kyaml.kustomize.dev/kio/index: 1 +functionConfig: {apiVersion: apps/v1, kind: Deployment, metadata: {name: foo}} +`, s) { + t.FailNow() + } + }, + }).Filter(input) + if !assert.NoError(t, err) { + return + } + if !assert.True(t, called) { + return + } + + b := &bytes.Buffer{} + err = kio.ByteWriter{Writer: b, KeepReaderAnnotations: true}.Write(result) + if !assert.NoError(t, err) { + return + } + + assert.Equal(t, `apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: deployment-foo + annotations: + kyaml.kustomize.dev/kio/index: 0 +--- +apiVersion: v1 +kind: Service +metadata: + name: service-foo + annotations: + kyaml.kustomize.dev/kio/index: 1 +`, b.String()) +} + +func TestFilter_Filter_noChange(t *testing.T) { + cfg, err := yaml.Parse(`apiversion: apps/v1 +kind: Deployment +metadata: + name: foo +`) + if !assert.NoError(t, err) { + return + } + + input, err := (&kio.ByteReader{Reader: bytes.NewBufferString(` +apiversion: apps/v1 +kind: Deployment +metadata: + name: deployment-foo +--- +apiVersion: v1 +kind: Service +metadata: + name: service-foo +`)}).Read() + if !assert.NoError(t, err) { + return + } + + called := false + result, err := (&ContainerFilter{ + Image: "example.com:version", + Config: cfg, + args: []string{"sh", "-c", "cat <&0"}, + checkInput: func(s string) { + called = true + if !assert.Equal(t, `apiVersion: kyaml.kustomize.dev/v1alpha1 +kind: ResourceList +items: +- apiversion: apps/v1 + kind: Deployment + metadata: + name: deployment-foo + annotations: + kyaml.kustomize.dev/kio/index: 0 +- apiVersion: v1 + kind: Service + metadata: + name: service-foo + annotations: + kyaml.kustomize.dev/kio/index: 1 +functionConfig: {apiversion: apps/v1, kind: Deployment, metadata: {name: foo}} +`, s) { + t.FailNow() + } + }, + }).Filter(input) + if !assert.NoError(t, err) { + return + } + if !assert.True(t, called) { + return + } + + b := &bytes.Buffer{} + err = kio.ByteWriter{Writer: b, KeepReaderAnnotations: true}.Write(result) + if !assert.NoError(t, err) { + return + } + + assert.Equal(t, `apiversion: apps/v1 +kind: Deployment +metadata: + name: deployment-foo + annotations: + kyaml.kustomize.dev/kio/index: 0 +--- +apiVersion: v1 +kind: Service +metadata: + name: service-foo + annotations: + kyaml.kustomize.dev/kio/index: 1 +`, b.String()) +} + +func Test_GetContainerName(t *testing.T) { + // make sure gcr.io works + n, err := yaml.Parse(`apiVersion: gcr.io/foo/bar:something +kind: MyThing +`) + if !assert.NoError(t, err) { + return + } + c := GetContainerName(n) + assert.Equal(t, "gcr.io/foo/bar:something", c) + + // make sure regional gcr.io works + n, err = yaml.Parse(`apiVersion: us.gcr.io/foo/bar:something +kind: MyThing +`) + if !assert.NoError(t, err) { + return + } + c = GetContainerName(n) + assert.Equal(t, "us.gcr.io/foo/bar:something", c) + + // container from annotation + n, err = yaml.Parse(`apiVersion: v1 +kind: MyThing +metadata: + annotations: + kyaml.kustomize.dev/container: gcr.io/foo/bar:something +`) + if !assert.NoError(t, err) { + return + } + c = GetContainerName(n) + assert.Equal(t, "gcr.io/foo/bar:something", c) + + // doesn't have a container + n, err = yaml.Parse(`apiVersion: v1 +kind: MyThing +metadata: +`) + if !assert.NoError(t, err) { + return + } + c = GetContainerName(n) + assert.Equal(t, "", c) + + // make sure docker.io works + n, err = yaml.Parse(`apiVersion: docker.io/foo/bar:something +kind: MyThing +`) + if !assert.NoError(t, err) { + return + } + c = GetContainerName(n) + assert.Equal(t, "docker.io/foo/bar:something", c) +} diff --git a/kyaml/kio/filters/filters.go b/kyaml/kio/filters/filters.go new file mode 100644 index 000000000..5902d4e20 --- /dev/null +++ b/kyaml/kio/filters/filters.go @@ -0,0 +1,198 @@ +// Copyright 2019 The Kubernetes Authors. +// SPDX-License-Identifier: Apache-2.0 + +package filters + +import ( + "fmt" + "sort" + "strings" + + "sigs.k8s.io/kustomize/kyaml/kio" + "sigs.k8s.io/kustomize/kyaml/kio/kioutil" + "sigs.k8s.io/kustomize/kyaml/yaml" +) + +// Filters are the list of known filters for unmarshalling a filter into a concrete +// implementation. +var Filters = map[string]func() kio.Filter{ + "FileSetter": func() kio.Filter { return &FileSetter{} }, + "FormatFilter": func() kio.Filter { return &FormatFilter{} }, + "GrepFilter": func() kio.Filter { return GrepFilter{} }, + "MatchModifier": func() kio.Filter { return &MatchModifyFilter{} }, + "Modifier": func() kio.Filter { return &Modifier{} }, +} + +// filter wraps a kio.filter so that it can be unmarshalled from yaml. +type KFilter struct { + kio.Filter +} + +func (t KFilter) MarshalYAML() (interface{}, error) { + return t.Filter, nil +} + +func (t *KFilter) UnmarshalYAML(unmarshal func(interface{}) error) error { + i := map[string]interface{}{} + if err := unmarshal(i); err != nil { + return err + } + meta := &yaml.ResourceMeta{} + if err := unmarshal(meta); err != nil { + return err + } + if filter, found := Filters[meta.Kind]; !found { + var knownFilters []string + for k := range Filters { + knownFilters = append(knownFilters, k) + } + sort.Strings(knownFilters) + return fmt.Errorf("unsupported filter Kind %v: may be one of: [%s]", + meta, strings.Join(knownFilters, ",")) + } else { + t.Filter = filter() + } + return unmarshal(t.Filter) +} + +// Modifier modifies the input Resources by invoking the provided pipeline. +// Modifier will return any Resources for which the pipeline does not return an error. +type Modifier struct { + Kind string `yaml:"kind,omitempty"` + + Filters yaml.YFilters `yaml:"pipeline,omitempty"` +} + +var _ kio.Filter = &Modifier{} + +func (f Modifier) Filter(input []*yaml.RNode) ([]*yaml.RNode, error) { + for i := range input { + if _, err := input[i].Pipe(f.Filters.Filters()...); err != nil { + return nil, err + } + } + return input, nil +} + +type MatchModifyFilter struct { + Kind string `yaml:"kind,omitempty"` + + MatchFilters []yaml.YFilters `yaml:"match,omitempty"` + + ModifyFilters yaml.YFilters `yaml:"modify,omitempty"` +} + +var _ kio.Filter = &MatchModifyFilter{} + +func (f MatchModifyFilter) Filter(input []*yaml.RNode) ([]*yaml.RNode, error) { + var matches = input + var err error + for _, filter := range f.MatchFilters { + matches, err = MatchFilter{Filters: filter}.Filter(matches) + if err != nil { + return nil, err + } + } + _, err = Modifier{Filters: f.ModifyFilters}.Filter(matches) + if err != nil { + return nil, err + } + return input, nil +} + +type MatchFilter struct { + Kind string `yaml:"kind,omitempty"` + + Filters yaml.YFilters `yaml:"pipeline,omitempty"` +} + +var _ kio.Filter = &MatchFilter{} + +func (f MatchFilter) Filter(input []*yaml.RNode) ([]*yaml.RNode, error) { + var output []*yaml.RNode + for i := range input { + if v, err := input[i].Pipe(f.Filters.Filters()...); err != nil { + return nil, err + } else if v == nil { + continue + } + output = append(output, input[i]) + } + return output, nil +} + +type FilenameFmtVerb string + +const ( + // KindFmt substitutes kind + KindFmt FilenameFmtVerb = "%k" + + // NameFmt substitutes metadata.name + NameFmt FilenameFmtVerb = "%n" + + // NamespaceFmt substitutes metdata.namespace + NamespaceFmt FilenameFmtVerb = "%s" +) + +// FileSetter sets the file name and mode annotations on Resources. +type FileSetter struct { + Kind string `yaml:"kind,omitempty"` + + // FilenamePattern is the pattern to use for generating filenames. FilenameFmtVerb + // FielnameFmtVerbs may be specified to substitute Resource metadata into the filename. + FilenamePattern string `yaml:"filenamePattern,omitempty"` + + // Mode is the filemode to write. + Mode string `yaml:"mode,omitempty"` + + // Override will override the existing filename if it is set on the pattern. + // Otherwise the existing filename is kept. + Override bool `yaml:"override,omitempty"` +} + +var _ kio.Filter = &FileSetter{} + +const DefaultFilenamePattern = "%n_%k.yaml" + +func (f *FileSetter) Filter(input []*yaml.RNode) ([]*yaml.RNode, error) { + if f.Mode == "" { + f.Mode = fmt.Sprintf("%d", 0600) + } + if f.FilenamePattern == "" { + f.FilenamePattern = DefaultFilenamePattern + } + + resources := map[string][]*yaml.RNode{} + for i := range input { + m, err := input[i].GetMeta() + if err != nil { + return nil, err + } + file := f.FilenamePattern + file = strings.Replace(file, string(KindFmt), strings.ToLower(m.Kind), -1) + file = strings.Replace(file, string(NameFmt), strings.ToLower(m.Name), -1) + file = strings.Replace(file, string(NamespaceFmt), strings.ToLower(m.Namespace), -1) + + if _, found := m.Annotations[kioutil.PathAnnotation]; !found || f.Override { + if _, err := input[i].Pipe(yaml.SetAnnotation(kioutil.PathAnnotation, file)); err != nil { + return nil, err + } + } + resources[file] = append(resources[file], input[i]) + } + + var output []*yaml.RNode + for i := range resources { + if err := kioutil.SortNodes(resources[i]); err != nil { + return nil, err + } + for j := range resources[i] { + if _, err := resources[i][j].Pipe( + yaml.SetAnnotation(kioutil.IndexAnnotation, fmt.Sprintf("%d", j))); err != nil { + return nil, err + } + output = append(output, resources[i][j]) + } + } + return output, nil +} diff --git a/kyaml/kio/filters/filters_test.go b/kyaml/kio/filters/filters_test.go new file mode 100644 index 000000000..d4e2eb4e6 --- /dev/null +++ b/kyaml/kio/filters/filters_test.go @@ -0,0 +1,170 @@ +// Copyright 2019 The Kubernetes Authors. +// SPDX-License-Identifier: Apache-2.0 + +package filters_test + +import ( + "bytes" + "testing" + + "github.com/stretchr/testify/assert" + . "sigs.k8s.io/kustomize/kyaml/kio" + . "sigs.k8s.io/kustomize/kyaml/kio/filters" +) + +var r = ` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: foo1 + namespace: bar +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: foo2 +--- +apiVersion: v1 +kind: Service +metadata: + name: foo2 + namespace: bar +--- +apiVersion: v1 +kind: Service +metadata: + name: foo1 +` + +func TestFileSetter_Filter(t *testing.T) { + in := bytes.NewBufferString(r) + out := &bytes.Buffer{} + err := Pipeline{ + Inputs: []Reader{&ByteReader{Reader: in}}, + Filters: []Filter{&FileSetter{}}, + Outputs: []Writer{ByteWriter{Sort: true, Writer: out}}, + }.Execute() + if !assert.NoError(t, err) { + return + } + assert.Equal(t, `apiVersion: apps/v1 +kind: Deployment +metadata: + name: foo1 + namespace: bar + annotations: + kyaml.kustomize.dev/kio/path: foo1_deployment.yaml +--- +apiVersion: v1 +kind: Service +metadata: + name: foo1 + annotations: + kyaml.kustomize.dev/kio/path: foo1_service.yaml +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: foo2 + annotations: + kyaml.kustomize.dev/kio/path: foo2_deployment.yaml +--- +apiVersion: v1 +kind: Service +metadata: + name: foo2 + namespace: bar + annotations: + kyaml.kustomize.dev/kio/path: foo2_service.yaml +`, out.String()) +} + +func TestFileSetter_Filter_pattern(t *testing.T) { + in := bytes.NewBufferString(r) + out := &bytes.Buffer{} + err := Pipeline{ + Inputs: []Reader{&ByteReader{Reader: in}}, + Filters: []Filter{&FileSetter{ + FilenamePattern: "%n_%s_%k.yaml", + }}, + Outputs: []Writer{ByteWriter{Sort: true, Writer: out}}, + }.Execute() + if !assert.NoError(t, err) { + return + } + assert.Equal(t, `apiVersion: v1 +kind: Service +metadata: + name: foo1 + annotations: + kyaml.kustomize.dev/kio/path: foo1__service.yaml +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: foo1 + namespace: bar + annotations: + kyaml.kustomize.dev/kio/path: foo1_bar_deployment.yaml +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: foo2 + annotations: + kyaml.kustomize.dev/kio/path: foo2__deployment.yaml +--- +apiVersion: v1 +kind: Service +metadata: + name: foo2 + namespace: bar + annotations: + kyaml.kustomize.dev/kio/path: foo2_bar_service.yaml +`, out.String()) +} + +func TestFileSetter_Filter_empty(t *testing.T) { + in := bytes.NewBufferString(r) + out := &bytes.Buffer{} + err := Pipeline{ + Inputs: []Reader{&ByteReader{Reader: in}}, + Filters: []Filter{&FileSetter{ + FilenamePattern: "resource.yaml", + }}, + Outputs: []Writer{ByteWriter{Writer: out}}, + }.Execute() + if !assert.NoError(t, err) { + return + } + assert.Equal(t, `apiVersion: apps/v1 +kind: Deployment +metadata: + name: foo1 + namespace: bar + annotations: + kyaml.kustomize.dev/kio/path: resource.yaml +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: foo2 + annotations: + kyaml.kustomize.dev/kio/path: resource.yaml +--- +apiVersion: v1 +kind: Service +metadata: + name: foo2 + namespace: bar + annotations: + kyaml.kustomize.dev/kio/path: resource.yaml +--- +apiVersion: v1 +kind: Service +metadata: + name: foo1 + annotations: + kyaml.kustomize.dev/kio/path: resource.yaml +`, out.String()) +} diff --git a/kyaml/kio/filters/fmtr.go b/kyaml/kio/filters/fmtr.go new file mode 100644 index 000000000..201c74f0d --- /dev/null +++ b/kyaml/kio/filters/fmtr.go @@ -0,0 +1,220 @@ +// Copyright 2019 The Kubernetes Authors. +// SPDX-License-Identifier: Apache-2.0 + +// Package yamlfmt contains libraries for formatting yaml files containing +// Kubernetes Resource configuration. +// +// Yaml files are formatted by: +// - Sorting fields and map values +// - Sorting unordered lists for whitelisted types +// - Applying a canonical yaml Style +// +// Fields are ordered using a relative ordering applied to commonly +// encountered Resource fields. All Resources, including non-builtin +// Resources such as CRDs, share the same field precedence. +// +// Fields that do not appear in the explicit ordering are ordered +// lexicographically. +// +// A subset of well known known unordered lists are sorted by element field +// values. +package filters + +import ( + "bytes" + "fmt" + "io" + "sort" + + "sigs.k8s.io/kustomize/kyaml/kio" + "sigs.k8s.io/kustomize/kyaml/yaml" +) + +// FormatInput returns the formatted input. +func FormatInput(input io.Reader) (*bytes.Buffer, error) { + buff := &bytes.Buffer{} + err := kio.Pipeline{ + Inputs: []kio.Reader{&kio.ByteReader{Reader: input}}, + Filters: []kio.Filter{FormatFilter{}}, + Outputs: []kio.Writer{kio.ByteWriter{Writer: buff}}, + }.Execute() + + return buff, err +} + +// FormatFileOrDirectory reads the file or directory and formats each file's +// contents by writing it back to the file. +func FormatFileOrDirectory(path string) error { + return kio.Pipeline{ + Inputs: []kio.Reader{kio.LocalPackageReader{ + PackagePath: path, + }}, + Filters: []kio.Filter{FormatFilter{}}, + Outputs: []kio.Writer{kio.LocalPackageWriter{PackagePath: path}}, + }.Execute() +} + +type FormatFilter struct{} + +var _ kio.Filter = FormatFilter{} + +func (f FormatFilter) Filter(slice []*yaml.RNode) ([]*yaml.RNode, error) { + for i := range slice { + kindNode, err := slice[i].Pipe(yaml.Get("kind")) + if err != nil { + return nil, err + } + if kindNode == nil { + continue + } + apiVersionNode, err := slice[i].Pipe(yaml.Get("apiVersion")) + if err != nil { + return nil, err + } + if apiVersionNode == nil { + continue + } + kind, apiVersion := kindNode.YNode().Value, apiVersionNode.YNode().Value + err = (&formatter{apiVersion: apiVersion, kind: kind}).fmtNode(slice[i].YNode(), "") + if err != nil { + return nil, err + } + } + return slice, nil +} + +type formatter struct { + apiVersion string + kind string +} + +// fmtNode recursively formats the Document Contents. +func (f *formatter) fmtNode(n *yaml.Node, path string) error { + n.Style = 0 + // sort the order of mapping fields + if n.Kind == yaml.MappingNode { + sort.Sort(sortedMapContents(*n)) + } + + // sort the order of sequence elements if it is whitelisted + if n.Kind == yaml.SequenceNode { + if yaml.WhitelistedListSortKinds.Has(f.kind) && + yaml.WhitelistedListSortApis.Has(f.apiVersion) { + if sortField, found := yaml.WhitelistedListSortFields[path]; found { + sort.Sort(sortedSeqContents{Node: *n, sortField: sortField}) + } + } + } + for i := range n.Content { + p := path + if n.Kind == yaml.MappingNode && i%2 == 1 { + p = fmt.Sprintf("%s.%s", path, n.Content[i-1].Value) + } + err := f.fmtNode(n.Content[i], p) + if err != nil { + return err + } + } + return nil +} + +// sortedMapContents sorts the Contents field of a MappingNode by the field names using a statically +// defined field precedence, and falling back on lexicographical sorting +type sortedMapContents yaml.Node + +func (s sortedMapContents) Len() int { + return len(s.Content) / 2 +} +func (s sortedMapContents) Swap(i, j int) { + // yaml MappingNode Contents are a list of field names followed by + // field values, rather than a list of field pairs. + // increment. + // + // e.g. ["field1Name", "field1Value", "field2Name", "field2Value"] + iFieldNameIndex := i * 2 + jFieldNameIndex := j * 2 + iFieldValueIndex := iFieldNameIndex + 1 + jFieldValueIndex := jFieldNameIndex + 1 + + // swap field names + s.Content[iFieldNameIndex], s.Content[jFieldNameIndex] = + s.Content[jFieldNameIndex], s.Content[iFieldNameIndex] + + // swap field values + s.Content[iFieldValueIndex], s.Content[jFieldValueIndex] = s. + Content[jFieldValueIndex], s.Content[iFieldValueIndex] +} +func (s sortedMapContents) Less(i, j int) bool { + iFieldNameIndex := i * 2 + jFieldNameIndex := j * 2 + iFieldName := s.Content[iFieldNameIndex].Value + jFieldName := s.Content[jFieldNameIndex].Value + + // order by their precedence values looked up from the index + iOrder, foundI := yaml.FieldOrder[iFieldName] + jOrder, foundJ := yaml.FieldOrder[jFieldName] + if foundI && foundJ { + return iOrder < jOrder + } + + // known fields come before unknown fields + if foundI { + return true + } + if foundJ { + return false + } + + // neither field is known, sort them lexicographically + return iFieldName < jFieldName +} + +// sortedSeqContents sorts the Contents field of a SequenceNode by the value of +// the elements sortField. +// e.g. it will sort spec.template.spec.containers by the value of the container `name` field +type sortedSeqContents struct { + yaml.Node + sortField string +} + +func (s sortedSeqContents) Len() int { + return len(s.Content) +} +func (s sortedSeqContents) Swap(i, j int) { + s.Content[i], s.Content[j] = s.Content[j], s.Content[i] +} +func (s sortedSeqContents) Less(i, j int) bool { + // primitive lists -- sort by the element's primitive values + if s.sortField == "" { + iValue := s.Content[i].Value + jValue := s.Content[j].Value + return iValue < jValue + } + + // map lists -- sort by the element's sortField values + var iValue, jValue string + for a := range s.Content[i].Content { + if a%2 != 0 { + continue // not a fieldNameIndex + } + // locate the index of the sortField field + if s.Content[i].Content[a].Value == s.sortField { + // a is the yaml node for the field key, a+1 is the node for the field value + iValue = s.Content[i].Content[a+1].Value + } + } + for a := range s.Content[j].Content { + if a%2 != 0 { + continue // not a fieldNameIndex + } + + // locate the index of the sortField field + if s.Content[j].Content[a].Value == s.sortField { + // a is the yaml node for the field key, a+1 is the node for the field value + jValue = s.Content[j].Content[a+1].Value + } + } + + // compare the field values + return iValue < jValue +} diff --git a/kyaml/kio/filters/fmtr_test.go b/kyaml/kio/filters/fmtr_test.go new file mode 100644 index 000000000..cbfe115b0 --- /dev/null +++ b/kyaml/kio/filters/fmtr_test.go @@ -0,0 +1,623 @@ +// Copyright 2019 The Kubernetes Authors. +// SPDX-License-Identifier: Apache-2.0 + +package filters_test + +import ( + "bytes" + "io" + "io/ioutil" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + . "sigs.k8s.io/kustomize/kyaml/kio/filters" + "sigs.k8s.io/kustomize/kyaml/kio/filters/testyaml" +) + +// TestFormatInput_configMap verifies a ConfigMap yaml is formatted correctly +func TestFormatInput_configMap(t *testing.T) { + y := ` + + +# this formatting is intentionally weird + +apiVersion: v1 +# this is data +data: + # this is color + color: purple + # that was color + + # this is textmode + textmode: "true" + # this is how + how: fairlyNice + + + +kind: ConfigMap + + +metadata: + selfLink: /api/v1/namespaces/default/configmaps/config-multi-env-files + namespace: default + creationTimestamp: 2017-12-27T18:38:34Z + name: config-multi-env-files + resourceVersion: "810136" + uid: 252c4572-eb35-11e7-887b-42010a8002b8 # keep no trailing linefeed` + + expected := `# this formatting is intentionally weird + +apiVersion: v1 +kind: ConfigMap +metadata: + name: config-multi-env-files + namespace: default + creationTimestamp: 2017-12-27T18:38:34Z + resourceVersion: "810136" + selfLink: /api/v1/namespaces/default/configmaps/config-multi-env-files + uid: 252c4572-eb35-11e7-887b-42010a8002b8 # keep no trailing linefeed +# this is data +data: + # this is color + color: purple + # that was color + + # this is how + how: fairlyNice + # this is textmode + textmode: "true" +` + + s, err := FormatInput(strings.NewReader(y)) + assert.NoError(t, err) + assert.Equal(t, expected, s.String()) +} + +// TestFormatInput_deployment verifies a Deployment yaml is formatted correctly +func TestFormatInput_deployment(t *testing.T) { + y := ` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nginx-deployment + labels: + app: nginx +spec: + selector: + matchLabels: + app: nginx + replicas: 3 + template: + metadata: + labels: + app: nginx + spec: + containers: + # this is a container + - ports: + # this is a port + - containerPort: 80 + name: b-nginx + image: nginx:1.7.9 + # this is another container + - name: a-nginx + image: nginx:1.7.9 + ports: + - containerPort: 80 +` + expected := `apiVersion: apps/v1 +kind: Deployment +metadata: + name: nginx-deployment + labels: + app: nginx +spec: + replicas: 3 + selector: + matchLabels: + app: nginx + template: + metadata: + labels: + app: nginx + spec: + containers: + - # this is another container + name: a-nginx + image: nginx:1.7.9 + ports: + - containerPort: 80 + - name: b-nginx + image: nginx:1.7.9 + # this is a container + ports: + - # this is a port + containerPort: 80 +` + s, err := FormatInput(strings.NewReader(y)) + assert.NoError(t, err) + assert.Equal(t, expected, s.String()) +} + +// TestFormatInput_service verifies a Service yaml is formatted correctly +func TestFormatInput_service(t *testing.T) { + + y := ` +apiVersion: v1 +kind: Service +metadata: + name: my-service +spec: + selector: + app: MyApp + ports: + - protocol: TCP + port: 80 + targetPort: 9376 +` + expected := `apiVersion: v1 +kind: Service +metadata: + name: my-service +spec: + selector: + app: MyApp + ports: + - protocol: TCP + port: 80 + targetPort: 9376 +` + s, err := FormatInput(strings.NewReader(y)) + assert.NoError(t, err) + assert.Equal(t, expected, s.String()) +} + +// TestFormatInput_service verifies a Service yaml is formatted correctly +func TestFormatInput_validatingWebhookConfiguration(t *testing.T) { + + y := ` +apiVersion: admissionregistration.k8s.io/v1beta1 +kind: ValidatingWebhookConfiguration +metadata: + name: +webhooks: +- name: + rules: + - apiGroups: + - "" + apiVersions: + - v1 + operations: + - UPDATE # this list is indented by 2 + - CREATE + - CONNECT + resources: + - pods # this list is not indented by 2 + scope: "Namespaced" + clientConfig: + service: + namespace: + name: + caBundle: + admissionReviewVersions: + - v1beta1 + timeoutSeconds: 1 +` + expected := `apiVersion: admissionregistration.k8s.io/v1beta1 +kind: ValidatingWebhookConfiguration +metadata: + name: +webhooks: +- name: + admissionReviewVersions: + - v1beta1 + clientConfig: + service: + name: + namespace: + caBundle: + rules: + - resources: + - pods # this list is not indented by 2 + apiGroups: + - "" + apiVersions: + - v1 + operations: + - CONNECT + - CREATE + - UPDATE # this list is indented by 2 + scope: Namespaced + timeoutSeconds: 1 +` + s, err := FormatInput(strings.NewReader(y)) + assert.NoError(t, err) + assert.Equal(t, expected, s.String()) +} + +// TestFormatInput_unKnownType verifies an unknown type yaml is formatted correctly +func TestFormatInput_unKnownType(t *testing.T) { + y := ` +spec: + template: + spec: + # these shouldn't be sorted because the type isn't whitelisted + containers: + - name: b + - name: a + replicas: 1 +status: + conditions: + - 3 + - 1 + - 2 +other: + b: a1 + a: b1 +apiVersion: example.com/v1beta1 +kind: MyType +` + + expected := `apiVersion: example.com/v1beta1 +kind: MyType +spec: + replicas: 1 + template: + spec: + # these shouldn't be sorted because the type isn't whitelisted + containers: + - name: b + - name: a +status: + conditions: + - 3 + - 1 + - 2 +other: + a: b1 + b: a1 +` + s, err := FormatInput(strings.NewReader(y)) + assert.NoError(t, err) + assert.Equal(t, expected, s.String()) +} + +// TestFormatInput_deployment verifies a Deployment yaml is formatted correctly +func TestFormatInput_resources(t *testing.T) { + input := &bytes.Buffer{} + _, err := io.Copy(input, bytes.NewReader(testyaml.UnformattedYaml1)) + assert.NoError(t, err) + _, err = io.Copy(input, strings.NewReader("---\n")) + assert.NoError(t, err) + _, err = io.Copy(input, bytes.NewReader(testyaml.UnformattedYaml2)) + assert.NoError(t, err) + + expectedOutput := &bytes.Buffer{} + _, err = io.Copy(expectedOutput, bytes.NewReader(testyaml.FormattedYaml1)) + assert.NoError(t, err) + _, err = io.Copy(expectedOutput, strings.NewReader("---\n")) + assert.NoError(t, err) + _, err = io.Copy(expectedOutput, bytes.NewReader(testyaml.FormattedYaml2)) + assert.NoError(t, err) + + s, err := FormatInput(input) + assert.NoError(t, err) + assert.Equal(t, expectedOutput.String(), s.String()) +} + +// +func TestFormatInput_failMissingKind(t *testing.T) { + y := ` +spec: + template: + spec: + containers: + - b + - a + replicas: 1 +status: + conditions: + - 3 + - 1 + - 2 +other: + b: a1 + a: b1 +apiVersion: example.com/v1beta1 +` + + b, err := FormatInput(strings.NewReader(y)) + assert.NoError(t, err) + assert.Equal(t, strings.TrimLeft(y, "\n"), b.String()) +} + +func TestFormatInput_failMissingApiVersion(t *testing.T) { + y := ` +spec: + template: + spec: + containers: + - a + - b + replicas: 1 +status: + conditions: + - 3 + - 1 + - 2 +other: + b: a1 + a: b1 +kind: MyKind +` + + b, err := FormatInput(strings.NewReader(y)) + assert.NoError(t, err) + assert.Equal(t, strings.TrimLeft(y, "\n"), b.String()) +} + +func TestFormatInput_failUnmarshal(t *testing.T) { + y := ` +spec: + template: + spec: + containers: + - a + - b + replicas: 1 +status: + conditions: + - 3 + - 1 + - 2 +other: + b: a1 + a: b1 +kind: MyKind +apiVersion: example.com/v1beta1 +` + + _, err := FormatInput(strings.NewReader(y)) + assert.EqualError(t, err, "yaml: line 15: found character that cannot start any token") +} + +// TestFormatFileOrDirectory_yamlExtFile verifies that FormatFileOrDirectory will format a file +// with a .yaml extension. +func TestFormatFileOrDirectory_yamlExtFile(t *testing.T) { + // write the unformatted file + f, err := ioutil.TempFile("", "yamlfmt*.yaml") + if !assert.NoError(t, err) { + return + } + defer os.Remove(f.Name()) + err = ioutil.WriteFile(f.Name(), testyaml.UnformattedYaml1, 0600) + if !assert.NoError(t, err) { + return + } + + // format the file + err = FormatFileOrDirectory(f.Name()) + if !assert.NoError(t, err) { + return + } + + // check the result is formatted + b, err := ioutil.ReadFile(f.Name()) + if !assert.NoError(t, err) { + return + } + assert.Equal(t, string(testyaml.FormattedYaml1), string(b)) +} + +func TestFormatFileOrDirectory_multipleYamlEntries(t *testing.T) { + // write the unformatted file + f, err := ioutil.TempFile("", "yamlfmt*.yaml") + assert.NoError(t, err) + defer os.Remove(f.Name()) + err = ioutil.WriteFile(f.Name(), + []byte(string(testyaml.UnformattedYaml1)+"---\n"+string(testyaml.UnformattedYaml2)), 0600) + assert.NoError(t, err) + + // format the file + err = FormatFileOrDirectory(f.Name()) + assert.NoError(t, err) + + // check the result is formatted + b, err := ioutil.ReadFile(f.Name()) + assert.NoError(t, err) + assert.Equal(t, string(testyaml.FormattedYaml1)+"---\n"+string(testyaml.FormattedYaml2), string(b)) +} + +// TestFormatFileOrDirectory_ymlExtFile verifies that FormatFileOrDirectory will format a file +// with a .yml extension. +func TestFormatFileOrDirectory_ymlExtFile(t *testing.T) { + // write the unformatted file + f, err := ioutil.TempFile("", "yamlfmt*.yml") + assert.NoError(t, err) + defer os.Remove(f.Name()) + err = ioutil.WriteFile(f.Name(), testyaml.UnformattedYaml1, 0600) + assert.NoError(t, err) + + // format the file + err = FormatFileOrDirectory(f.Name()) + assert.NoError(t, err) + + // check the result is formatted + b, err := ioutil.ReadFile(f.Name()) + assert.NoError(t, err) + assert.Equal(t, string(testyaml.FormattedYaml1), string(b)) +} + +// TestFormatFileOrDirectory_skipYamlExtFileWithJson verifies that the json content is formatted +// as yaml +func TestFormatFileOrDirectory_YamlExtFileWithJson(t *testing.T) { + // write the unformatted JSON file contents + f, err := ioutil.TempFile("", "yamlfmt*.yaml") + assert.NoError(t, err) + defer os.Remove(f.Name()) + err = ioutil.WriteFile(f.Name(), testyaml.UnformattedJson1, 0600) + assert.NoError(t, err) + + // format the file + err = FormatFileOrDirectory(f.Name()) + assert.NoError(t, err) + + // check the result is formatted as yaml + b, err := ioutil.ReadFile(f.Name()) + assert.NoError(t, err) + assert.Equal(t, string(testyaml.FormattedYaml1), string(b)) +} + +// TestFormatFileOrDirectory_partialKubernetesYamlFile verifies that if a yaml file contains both +// Kubernetes and non-Kubernetes documents, it will only format the Kubernetes documents +func TestFormatFileOrDirectory_partialKubernetesYamlFile(t *testing.T) { + // write the unformatted file + f, err := ioutil.TempFile("", "yamlfmt*.yaml") + assert.NoError(t, err) + defer os.Remove(f.Name()) + err = ioutil.WriteFile(f.Name(), []byte(string(testyaml.UnformattedYaml1)+`--- +status: + conditions: + - 3 + - 1 + - 2 +spec: a +--- +`+string(testyaml.UnformattedYaml2)), 0600) + assert.NoError(t, err) + + // format the file + err = FormatFileOrDirectory(f.Name()) + assert.NoError(t, err) + + // check the result is NOT formatted + b, err := ioutil.ReadFile(f.Name()) + assert.NoError(t, err) + assert.Equal(t, string(testyaml.FormattedYaml1)+`--- +status: + conditions: + - 3 + - 1 + - 2 +spec: a +--- +`+string(testyaml.FormattedYaml2), string(b)) +} + +// TestFormatFileOrDirectory_nonKubernetesYamlFile verifies that if a yaml file does not contain +// kubernetes +func TestFormatFileOrDirectory_skipNonKubernetesYamlFile(t *testing.T) { + // write the unformatted JSON file contents + f, err := ioutil.TempFile("", "yamlfmt*.yaml") + assert.NoError(t, err) + defer os.Remove(f.Name()) + err = ioutil.WriteFile(f.Name(), []byte(` +status: + conditions: + - 3 + - 1 + - 2 +spec: a +`), 0600) + assert.NoError(t, err) + + // format the file + err = FormatFileOrDirectory(f.Name()) + assert.NoError(t, err) + + // check the result is formatted as yaml + b, err := ioutil.ReadFile(f.Name()) + assert.NoError(t, err) + assert.Equal(t, `status: + conditions: + - 3 + - 1 + - 2 +spec: a +`, string(b)) +} + +// TestFormatFileOrDirectory_jsonFile should not fmt the file even though it contains yaml. +func TestFormatFileOrDirectory_skipJsonExtFile(t *testing.T) { + f, err := ioutil.TempFile("", "yamlfmt*.json") + assert.NoError(t, err) + defer os.Remove(f.Name()) + err = ioutil.WriteFile(f.Name(), testyaml.UnformattedYaml1, 0600) + assert.NoError(t, err) + + err = FormatFileOrDirectory(f.Name()) + assert.NoError(t, err) + + b, err := ioutil.ReadFile(f.Name()) + assert.NoError(t, err) + + assert.Equal(t, string(testyaml.UnformattedYaml1), string(b)) +} + +// TestFormatFileOrDirectory_directory verifies that yaml files will be formatted, +// and other files will be ignored +func TestFormatFileOrDirectory_directory(t *testing.T) { + d, err := ioutil.TempDir("", "yamlfmt") + assert.NoError(t, err) + + err = os.Mkdir(filepath.Join(d, "config"), 0700) + assert.NoError(t, err) + + err = ioutil.WriteFile(filepath.Join(d, "c1.yaml"), testyaml.UnformattedYaml1, 0600) + assert.NoError(t, err) + + err = ioutil.WriteFile(filepath.Join(d, "config", "c2.yaml"), testyaml.UnformattedYaml2, 0600) + assert.NoError(t, err) + + err = ioutil.WriteFile(filepath.Join(d, "README.md"), []byte(`# Markdown`), 0600) + assert.NoError(t, err) + + err = FormatFileOrDirectory(d) + assert.NoError(t, err) + + b, err := ioutil.ReadFile(filepath.Join(d, "c1.yaml")) + assert.NoError(t, err) + assert.Equal(t, string(testyaml.FormattedYaml1), string(b)) + + b, err = ioutil.ReadFile(filepath.Join(d, "config", "c2.yaml")) + assert.NoError(t, err) + assert.Equal(t, string(testyaml.FormattedYaml2), string(b)) + + b, err = ioutil.ReadFile(filepath.Join(d, "README.md")) + assert.NoError(t, err) + assert.Equal(t, `# Markdown`, string(b)) + + // verify no additional files were created + files := []string{ + ".", "c1.yaml", "README.md", "config", filepath.Join("config", "c2.yaml")} + err = filepath.Walk(d, func(path string, info os.FileInfo, err error) error { + assert.NoError(t, err) + path, err = filepath.Rel(d, path) + assert.NoError(t, err) + assert.Contains(t, files, path) + return nil + }) + assert.NoError(t, err) +} + +// TestFormatFileOrDirectory_trimWhiteSpace verifies that trailling and leading whitespace is +// trimmed +func TestFormatFileOrDirectory_trimWhiteSpace(t *testing.T) { + f, err := ioutil.TempFile("", "yamlfmt*.yaml") + assert.NoError(t, err) + defer os.Remove(f.Name()) + err = ioutil.WriteFile(f.Name(), []byte("\n\n"+string(testyaml.UnformattedYaml1)+"\n\n"), 0600) + assert.NoError(t, err) + + err = FormatFileOrDirectory(f.Name()) + assert.NoError(t, err) + + b, err := ioutil.ReadFile(f.Name()) + assert.NoError(t, err) + + assert.Equal(t, string(testyaml.FormattedYaml1), string(b)) +} diff --git a/kyaml/kio/filters/grep.go b/kyaml/kio/filters/grep.go new file mode 100644 index 000000000..e9874f07c --- /dev/null +++ b/kyaml/kio/filters/grep.go @@ -0,0 +1,117 @@ +// Copyright 2019 The Kubernetes Authors. +// SPDX-License-Identifier: Apache-2.0 + +package filters + +import ( + "regexp" + "strings" + + "sigs.k8s.io/kustomize/kyaml/kio" + "sigs.k8s.io/kustomize/kyaml/yaml" +) + +type GrepType int + +const ( + Regexp GrepType = 1 << iota + GreaterThanEq + GreaterThan + LessThan + LessThanEq +) + +// GrepFilter filters RNodes with a matching field +type GrepFilter struct { + Path []string `yaml:"path,omitempty"` + Value string `yaml:"value,omitempty"` + MatchType GrepType `yaml:"matchType,omitempty"` + InvertMatch bool `yaml:"invertMatch,omitempty"` + Compare func(a, b string) (int, error) +} + +var _ kio.Filter = GrepFilter{} + +func (f GrepFilter) Filter(input []*yaml.RNode) ([]*yaml.RNode, error) { + // compile the regular expression 1 time if we are matching using regex + var reg *regexp.Regexp + var err error + if f.MatchType == Regexp || f.MatchType == 0 { + reg, err = regexp.Compile(f.Value) + if err != nil { + return nil, err + } + } + + var output kio.ResourceNodeSlice + for i := range input { + node := input[i] + val, err := node.Pipe(&yaml.PathMatcher{Path: f.Path}) + if err != nil { + return nil, err + } + if val == nil || len(val.Content()) == 0 { + if f.InvertMatch { + output = append(output, input[i]) + } + continue + } + found := false + err = val.VisitElements(func(elem *yaml.RNode) error { + // get the value + var str string + if f.MatchType == Regexp { + style := elem.YNode().Style + defer func() { elem.YNode().Style = style }() + elem.YNode().Style = yaml.FlowStyle + str, err = elem.String() + if err != nil { + return err + } + str = strings.TrimSpace(strings.Replace(str, `"`, "", -1)) + } else { + // if not regexp, then it needs to parse into a quantity and comments will + // break that + str = elem.YNode().Value + if str == "" { + return nil + } + } + + if f.MatchType == Regexp || f.MatchType == 0 { + if reg.MatchString(str) { + found = true + } + return nil + } + + comp, err := f.Compare(str, f.Value) + if err != nil { + return err + } + + if f.MatchType == GreaterThan && comp > 0 { + found = true + } + if f.MatchType == GreaterThanEq && comp >= 0 { + found = true + } + if f.MatchType == LessThan && comp < 0 { + found = true + } + if f.MatchType == LessThanEq && comp <= 0 { + found = true + } + return nil + }) + if err != nil { + return nil, err + } + if found == f.InvertMatch { + continue + } + + output = append(output, input[i]) + } + return output, nil +} diff --git a/kyaml/kio/filters/grep_test.go b/kyaml/kio/filters/grep_test.go new file mode 100644 index 000000000..29e56a2a2 --- /dev/null +++ b/kyaml/kio/filters/grep_test.go @@ -0,0 +1,157 @@ +// Copyright 2019 The Kubernetes Authors. +// SPDX-License-Identifier: Apache-2.0 + +package filters_test + +import ( + "bytes" + "testing" + + "github.com/stretchr/testify/assert" + "sigs.k8s.io/kustomize/kyaml/kio" + . "sigs.k8s.io/kustomize/kyaml/kio/filters" + "sigs.k8s.io/kustomize/kyaml/yaml" +) + +func TestGrepFilter_Filter(t *testing.T) { + in := `kind: Deployment +metadata: + labels: + app: nginx2 + name: foo + annotations: + app: nginx2 +spec: + replicas: 1 +--- +kind: Deployment +metadata: + labels: + app: nginx + annotations: + app: nginx + name: bar +spec: + replicas: 3 +--- +kind: Service +metadata: + name: foo + annotations: + app: nginx +spec: + selector: + app: nginx +` + out := &bytes.Buffer{} + err := kio.Pipeline{ + Inputs: []kio.Reader{&kio.ByteReader{Reader: bytes.NewBufferString(in)}}, + Filters: []kio.Filter{GrepFilter{Path: []string{"metadata", "name"}, Value: "foo"}}, + Outputs: []kio.Writer{kio.ByteWriter{Writer: out}}, + }.Execute() + if !assert.NoError(t, err) { + t.FailNow() + } + + if !assert.Equal(t, `kind: Deployment +metadata: + labels: + app: nginx2 + name: foo + annotations: + app: nginx2 +spec: + replicas: 1 +--- +kind: Service +metadata: + name: foo + annotations: + app: nginx +spec: + selector: + app: nginx +`, out.String()) { + t.FailNow() + } + + out = &bytes.Buffer{} + err = kio.Pipeline{ + Inputs: []kio.Reader{&kio.ByteReader{Reader: bytes.NewBufferString(in)}}, + Filters: []kio.Filter{GrepFilter{Path: []string{"kind"}, Value: "Deployment"}}, + Outputs: []kio.Writer{kio.ByteWriter{Writer: out}}, + }.Execute() + if !assert.NoError(t, err) { + t.FailNow() + } + if !assert.Equal(t, `kind: Deployment +metadata: + labels: + app: nginx2 + name: foo + annotations: + app: nginx2 +spec: + replicas: 1 +--- +kind: Deployment +metadata: + labels: + app: nginx + annotations: + app: nginx + name: bar +spec: + replicas: 3 +`, out.String()) { + t.FailNow() + } + + out = &bytes.Buffer{} + err = kio.Pipeline{ + Inputs: []kio.Reader{&kio.ByteReader{Reader: bytes.NewBufferString(in)}}, + Filters: []kio.Filter{GrepFilter{Path: []string{"spec", "replicas"}, Value: "3"}}, + Outputs: []kio.Writer{kio.ByteWriter{Writer: out}}, + }.Execute() + if !assert.NoError(t, err) { + t.FailNow() + } + if !assert.Equal(t, `kind: Deployment +metadata: + labels: + app: nginx + annotations: + app: nginx + name: bar +spec: + replicas: 3 +`, out.String()) { + t.FailNow() + } + + out = &bytes.Buffer{} + err = kio.Pipeline{ + Inputs: []kio.Reader{&kio.ByteReader{Reader: bytes.NewBufferString(in)}}, + Filters: []kio.Filter{GrepFilter{Path: []string{"spec", "not-present"}, Value: "3"}}, + Outputs: []kio.Writer{kio.ByteWriter{Writer: out}}, + }.Execute() + if !assert.NoError(t, err) { + t.FailNow() + } + if !assert.Equal(t, ``, out.String()) { + t.FailNow() + } +} + +func TestGrepFilter_init(t *testing.T) { + assert.Equal(t, GrepFilter{}, Filters["GrepFilter"]()) +} + +func TestGrepFilter_error(t *testing.T) { + v, err := GrepFilter{Path: []string{"metadata", "name"}, + Value: "foo"}.Filter([]*yaml.RNode{{}}) + if !assert.NoError(t, err) { + t.FailNow() + } + assert.Nil(t, v) +} diff --git a/kyaml/kio/filters/merge.go b/kyaml/kio/filters/merge.go new file mode 100644 index 000000000..65726e2b2 --- /dev/null +++ b/kyaml/kio/filters/merge.go @@ -0,0 +1,76 @@ +// Copyright 2019 The Kubernetes Authors. +// SPDX-License-Identifier: Apache-2.0 + +// Package merge contains libraries for merging Resources and Patches +package filters + +import ( + "sigs.k8s.io/kustomize/kyaml/yaml" + "sigs.k8s.io/kustomize/kyaml/yaml/merge2" +) + +// GrepFilter merges Resources with the Group/Version/Kind/Namespace/Name together using +// a 2-way merge strategy. +// +// - Fields set to null in the source will be cleared from the destination +// - Fields with matching keys will be merged recursively +// - Lists with an associative key (e.g. name) will have their elements merged using the key +// - List without an associative key will have the dest list replaced by the source list +type MergeFilter struct { + Reverse bool +} + +type mergeKey struct { + apiVersion string + kind string + namespace string + name string +} + +// GrepFilter implements kio.GrepFilter by merge Resources with the same G/V/K/NS/N +func (c MergeFilter) Filter(input []*yaml.RNode) ([]*yaml.RNode, error) { + // invert the merge precedence + if c.Reverse { + for i, j := 0, len(input)-1; i < j; i, j = i+1, j-1 { + input[i], input[j] = input[j], input[i] + } + } + + // index the Resources by G/V/K/NS/N + index := map[mergeKey][]*yaml.RNode{} + for i := range input { + meta, err := input[i].GetMeta() + if err != nil { + return nil, err + } + key := mergeKey{ + apiVersion: meta.ApiVersion, + kind: meta.Kind, + namespace: meta.Namespace, + name: meta.Name, + } + index[key] = append(index[key], input[i]) + } + + // merge each of the G/V/K/NS/N lists + var output []*yaml.RNode + var err error + for k := range index { + var merged *yaml.RNode + resources := index[k] + for i := range resources { + patch := resources[i] + if merged == nil { + // first resources, don't merge it + merged = resources[i] + } else { + merged, err = merge2.Merge(patch, merged) + if err != nil { + return nil, err + } + } + } + output = append(output, merged) + } + return output, nil +} diff --git a/kyaml/kio/filters/merge3.go b/kyaml/kio/filters/merge3.go new file mode 100644 index 000000000..80d29edf2 --- /dev/null +++ b/kyaml/kio/filters/merge3.go @@ -0,0 +1,167 @@ +// Copyright 2019 The Kubernetes Authors. +// SPDX-License-Identifier: Apache-2.0 + +package filters + +import ( + "fmt" + + "sigs.k8s.io/kustomize/kyaml/kio" + "sigs.k8s.io/kustomize/kyaml/yaml" + "sigs.k8s.io/kustomize/kyaml/yaml/merge3" +) + +const ( + mergeSourceAnnotation = "kyaml.kustomize.dev/merge-source" + mergeSourceOriginal = "original" + mergeSourceUpdated = "updated" + mergeSourceDest = "dest" +) + +// Merge3 performs a 3-way merge on the original, updated, and destination packages. +type Merge3 struct { + OriginalPath string + UpdatedPath string + DestPath string + MatchFilesGlob []string +} + +func (m Merge3) Merge() error { + // Read the destination package. The ReadWriter will take take of deleting files + // for removed resources. + var inputs []kio.Reader + dest := &kio.LocalPackageReadWriter{ + PackagePath: m.DestPath, + MatchFilesGlob: m.MatchFilesGlob, + SetAnnotations: map[string]string{mergeSourceAnnotation: mergeSourceDest}, + } + inputs = append(inputs, dest) + + // Read the original package + inputs = append(inputs, kio.LocalPackageReader{ + PackagePath: m.OriginalPath, + MatchFilesGlob: m.MatchFilesGlob, + SetAnnotations: map[string]string{mergeSourceAnnotation: mergeSourceOriginal}, + }) + + // Read the updated package + inputs = append(inputs, kio.LocalPackageReader{ + PackagePath: m.UpdatedPath, + MatchFilesGlob: m.MatchFilesGlob, + SetAnnotations: map[string]string{mergeSourceAnnotation: mergeSourceUpdated}, + }) + + return kio.Pipeline{ + Inputs: inputs, + Filters: []kio.Filter{m, FormatFilter{}}, // format the merged output + Outputs: []kio.Writer{dest}, + }.Execute() +} + +// Filter combines Resources with the same GVK + N + NS into tuples, and then merges them +func (m Merge3) Filter(nodes []*yaml.RNode) ([]*yaml.RNode, error) { + // index the nodes by their identity + tl := tuples{} + for i := range nodes { + if err := tl.add(nodes[i]); err != nil { + return nil, err + } + } + + // iterate over the inputs, merging as needed + var output []*yaml.RNode + for i := range tl.list { + t := tl.list[i] + switch { + case t.original == nil && t.updated == nil && t.dest != nil: + // added locally -- keep dest + output = append(output, t.dest) + case t.original == nil && t.updated != nil && t.dest == nil: + // added in the update -- add update + output = append(output, t.updated) + case t.original != nil && t.updated == nil: + // deleted in the update + // don't include the resource in the output + case t.original != nil && t.dest == nil: + // deleted locally + // don't include the resource in the output + default: + // dest and updated are non-nil -- merge them + node, err := t.merge() + if err != nil { + return nil, err + } + if node != nil { + output = append(output, node) + } + } + } + return output, nil +} + +// tuples combines nodes with the same GVK + N + NS +type tuples struct { + list []*tuple +} + +// add adds a node to the list, combining it with an existing matching Resource if found +func (ts *tuples) add(node *yaml.RNode) error { + nodeMeta, err := node.GetMeta() + if err != nil { + return err + } + for i := range ts.list { + t := ts.list[i] + if t.meta.Name == nodeMeta.Name && t.meta.Namespace == nodeMeta.Namespace && + t.meta.ApiVersion == nodeMeta.ApiVersion && t.meta.Kind == nodeMeta.Kind { + return t.add(node) + } + } + t := &tuple{meta: nodeMeta} + if err := t.add(node); err != nil { + return err + } + ts.list = append(ts.list, t) + return nil +} + +// tuple wraps an original, updated, and dest tuple for a given Resource +type tuple struct { + meta yaml.ResourceMeta + original *yaml.RNode + updated *yaml.RNode + dest *yaml.RNode +} + +// add sets the corresponding tuple field for the node +func (t *tuple) add(node *yaml.RNode) error { + meta, err := node.GetMeta() + if err != nil { + return err + } + switch meta.Annotations[mergeSourceAnnotation] { + case mergeSourceDest: + if t.dest != nil { + return fmt.Errorf("dest source already specified") + } + t.dest = node + case mergeSourceOriginal: + if t.original != nil { + return fmt.Errorf("original source already specified") + } + t.original = node + case mergeSourceUpdated: + if t.updated != nil { + return fmt.Errorf("updated source already specified") + } + t.updated = node + default: + return fmt.Errorf("no source annotation for Resource") + } + return nil +} + +// merge performs a 3-way merge on the tuple +func (t *tuple) merge() (*yaml.RNode, error) { + return merge3.Merge(t.dest, t.original, t.updated) +} diff --git a/kyaml/kio/filters/merge3_test.go b/kyaml/kio/filters/merge3_test.go new file mode 100644 index 000000000..d8615b33d --- /dev/null +++ b/kyaml/kio/filters/merge3_test.go @@ -0,0 +1,56 @@ +// Copyright 2019 The Kubernetes Authors. +// SPDX-License-Identifier: Apache-2.0 + +package filters_test + +import ( + "io/ioutil" + "os" + "path/filepath" + "runtime" + "testing" + + "github.com/stretchr/testify/assert" + "sigs.k8s.io/kustomize/kyaml/copyutil" + "sigs.k8s.io/kustomize/kyaml/kio/filters" +) + +func TestMerge3_Merge(t *testing.T) { + _, datadir, _, ok := runtime.Caller(0) + if !assert.True(t, ok) { + t.FailNow() + } + datadir = filepath.Join(filepath.Dir(datadir), "testdata") + + // setup the local directory + dir, err := ioutil.TempDir("", "kyaml-test") + if !assert.NoError(t, err) { + t.FailNow() + } + defer os.RemoveAll(dir) + + if !assert.NoError(t, copyutil.CopyDir( + filepath.Join(datadir, "dataset1-localupdates"), + filepath.Join(dir, "dataset1"))) { + t.FailNow() + } + + err = filters.Merge3{ + OriginalPath: filepath.Join(datadir, "dataset1"), + UpdatedPath: filepath.Join(datadir, "dataset1-remoteupdates"), + DestPath: filepath.Join(dir, "dataset1"), + }.Merge() + if !assert.NoError(t, err) { + t.FailNow() + } + + diffs, err := copyutil.Diff( + filepath.Join(dir, "dataset1"), + filepath.Join(datadir, "dataset1-expected")) + if !assert.NoError(t, err) { + t.FailNow() + } + if !assert.Empty(t, diffs.List()) { + t.FailNow() + } +} diff --git a/kyaml/kio/filters/modify.go b/kyaml/kio/filters/modify.go new file mode 100644 index 000000000..b1090302a --- /dev/null +++ b/kyaml/kio/filters/modify.go @@ -0,0 +1,4 @@ +// Copyright 2019 The Kubernetes Authors. +// SPDX-License-Identifier: Apache-2.0 + +package filters diff --git a/kyaml/kio/filters/stripcomments.go b/kyaml/kio/filters/stripcomments.go new file mode 100644 index 000000000..23e1a4541 --- /dev/null +++ b/kyaml/kio/filters/stripcomments.go @@ -0,0 +1,43 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package filters + +import ( + "sigs.k8s.io/kustomize/kyaml/kio" + "sigs.k8s.io/kustomize/kyaml/yaml" +) + +type StripCommentsFilter struct{} + +var _ kio.Filter = StripCommentsFilter{} + +func (f StripCommentsFilter) Filter(slice []*yaml.RNode) ([]*yaml.RNode, error) { + for i := range slice { + stripComments(slice[i].YNode()) + } + return slice, nil +} + +func stripComments(node *yaml.Node) { + if node == nil { + return + } + node.HeadComment = "" + node.LineComment = "" + node.FootComment = "" + for i := range node.Content { + stripComments(node.Content[i]) + } +} diff --git a/kyaml/kio/filters/testdata/dataset1-expected/java/java-configmap.resource.yaml b/kyaml/kio/filters/testdata/dataset1-expected/java/java-configmap.resource.yaml new file mode 100644 index 000000000..442c5ce51 --- /dev/null +++ b/kyaml/kio/filters/testdata/dataset1-expected/java/java-configmap.resource.yaml @@ -0,0 +1,11 @@ +# Copyright 2019 The Kubernetes Authors. +# SPDX-License-Identifier: Apache-2.0 +# +apiVersion: v1 +kind: ConfigMap +metadata: + name: app-config + labels: + app.kubernetes.io/component: undefined + app.kubernetes.io/instance: undefined +data: {} diff --git a/kyaml/kio/filters/testdata/dataset1-expected/java/java-deployment.resource.yaml b/kyaml/kio/filters/testdata/dataset1-expected/java/java-deployment.resource.yaml new file mode 100644 index 000000000..7ca9554fa --- /dev/null +++ b/kyaml/kio/filters/testdata/dataset1-expected/java/java-deployment.resource.yaml @@ -0,0 +1,43 @@ +# Copyright 2019 The Kubernetes Authors. +# SPDX-License-Identifier: Apache-2.0 +# +apiVersion: apps/v1 +kind: Deployment +metadata: + name: app + labels: + app: java + new-local: label + new-remote: label +spec: + replicas: 3 + selector: + matchLabels: + app: java + template: + metadata: + labels: + app: java + spec: + restartPolicy: Always + containers: + - name: app + image: gcr.io/project/app:version + command: + - java + - -jar + - /app.jar + - otherstuff + args: + - foo + ports: + - containerPort: 8080 + envFrom: + - configMapRef: + name: app-config + env: + - name: JAVA_OPTS + value: -XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap + -Djava.security.egd=file:/dev/./urandom + imagePullPolicy: Always + minReadySeconds: 20 diff --git a/kyaml/kio/filters/testdata/dataset1-expected/java/java-service.resource.yaml b/kyaml/kio/filters/testdata/dataset1-expected/java/java-service.resource.yaml new file mode 100644 index 000000000..051996295 --- /dev/null +++ b/kyaml/kio/filters/testdata/dataset1-expected/java/java-service.resource.yaml @@ -0,0 +1,16 @@ +# Copyright 2019 The Kubernetes Authors. +# SPDX-License-Identifier: Apache-2.0 +# +apiVersion: v1 +kind: Service +metadata: + name: app + labels: + app: java +spec: + selector: + app: java + ports: + - name: "8080" + port: 8080 + targetPort: 8080 diff --git a/kyaml/kio/filters/testdata/dataset1-expected/mysql/mysql-configmap.resource.yaml b/kyaml/kio/filters/testdata/dataset1-expected/mysql/mysql-configmap.resource.yaml new file mode 100644 index 000000000..714af7037 --- /dev/null +++ b/kyaml/kio/filters/testdata/dataset1-expected/mysql/mysql-configmap.resource.yaml @@ -0,0 +1,23 @@ +# Copyright 2019 The Kubernetes Authors. +# SPDX-License-Identifier: Apache-2.0 +# +apiVersion: v1 +kind: ConfigMap +metadata: + name: mysql + labels: + app: mysql + annotations: + file/index: "0" + file/path: mysql-configmap.resource.yaml + package/name: mysql + package/original-name: mysql +data: + master.cnf: | + # Apply this config only on the master. + [mysqld] + log-bin + slave.cnf: | + # Apply this config only on slaves. + [mysqld] + super-read-only diff --git a/kyaml/kio/filters/testdata/dataset1-expected/mysql/mysql-service.resource.yaml b/kyaml/kio/filters/testdata/dataset1-expected/mysql/mysql-service.resource.yaml new file mode 100644 index 000000000..667544d15 --- /dev/null +++ b/kyaml/kio/filters/testdata/dataset1-expected/mysql/mysql-service.resource.yaml @@ -0,0 +1,29 @@ +# Copyright 2019 The Kubernetes Authors. +# SPDX-License-Identifier: Apache-2.0 +# +apiVersion: v1 +kind: Service +metadata: + name: mysql + labels: + app: mysql +spec: + selector: + app: mysql + ports: + - name: mysql + port: 3306 + clusterIP: None +--- +apiVersion: v1 +kind: Service +metadata: + name: mysql-read + labels: + app: mysql +spec: + selector: + app: mysql + ports: + - name: mysql + port: 3306 diff --git a/kyaml/kio/filters/testdata/dataset1-expected/mysql/mysql-statefulset.resource.yaml b/kyaml/kio/filters/testdata/dataset1-expected/mysql/mysql-statefulset.resource.yaml new file mode 100644 index 000000000..681a806bd --- /dev/null +++ b/kyaml/kio/filters/testdata/dataset1-expected/mysql/mysql-statefulset.resource.yaml @@ -0,0 +1,173 @@ +# Copyright 2019 The Kubernetes Authors. +# SPDX-License-Identifier: Apache-2.0 +# +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: mysql +spec: + replicas: 3 + selector: + matchLabels: + app: mysql + template: + metadata: + labels: + app: mysql + spec: + initContainers: + - name: init-mysql + image: mysql:5.7 + command: + - bash + - -c + - | + set -ex + # Generate mysql server-id from pod ordinal index. + [[ `hostname` =~ -([0-9]+)$ ]] || exit 1 + ordinal=${BASH_REMATCH[1]} + echo [mysqld] > /mnt/conf.d/server-id.cnf + # Add an offset to avoid reserved server-id=0 value. + echo server-id=$((100 + $ordinal)) >> /mnt/conf.d/server-id.cnf + # Copy appropriate conf.d files from config-map to emptyDir. + if [[ $ordinal -eq 0 ]]; then + cp /mnt/config-map/master.cnf /mnt/conf.d/ + else + cp /mnt/config-map/slave.cnf /mnt/conf.d/ + fi + volumeMounts: + - name: conf + mountPath: /mnt/conf.d + - name: config-map + mountPath: /mnt/config-map + - name: clone-mysql + image: gcr.io/google-samples/xtrabackup:1.0 + command: + - bash + - -c + - | + set -ex + # Skip the clone if data already exists. + [[ -d /var/lib/mysql/mysql ]] && exit 0 + # Skip the clone on master (ordinal index 0). + [[ `hostname` =~ -([0-9]+)$ ]] || exit 1 + ordinal=${BASH_REMATCH[1]} + [[ $ordinal -eq 0 ]] && exit 0 + # Clone data from previous peer. + ncat --recv-only mysql-$(($ordinal-1)).mysql 3307 | xbstream -x -C /var/lib/mysql + # Prepare the backup. + xtrabackup --prepare --target-dir=/var/lib/mysql + volumeMounts: + - name: data + mountPath: /var/lib/mysql + subPath: mysql + - name: conf + mountPath: /etc/mysql/conf.d + containers: + - name: mysql + image: mysql:5.7 + ports: + - name: mysql + containerPort: 3306 + env: + - name: MYSQL_ALLOW_EMPTY_PASSWORD + value: "1" + resources: + requests: + cpu: 500m + memory: 1Gi + volumeMounts: + - name: data + mountPath: /var/lib/mysql + subPath: mysql + - name: conf + mountPath: /etc/mysql/conf.d + livenessProbe: + exec: + command: + - mysqladmin + - ping + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 5 + readinessProbe: + exec: + command: + - mysql + - -h + - 127.0.0.1 + - -e + - SELECT 1 + initialDelaySeconds: 5 + periodSeconds: 2 + timeoutSeconds: 1 + - name: xtrabackup + image: gcr.io/google-samples/xtrabackup:1.0 + command: + - bash + - -c + - | + set -ex + cd /var/lib/mysql + # Determine binlog position of cloned data, if any. + if [[ -f xtrabackup_slave_info ]]; then + # XtraBackup already generated a partial "CHANGE MASTER TO" query + # because we're cloning from an existing slave. + mv xtrabackup_slave_info change_master_to.sql.in + # Ignore xtrabackup_binlog_info in this case (it's useless). + rm -f xtrabackup_binlog_info + elif [[ -f xtrabackup_binlog_info ]]; then + # We're cloning directly from master. Parse binlog position. + [[ `cat xtrabackup_binlog_info` =~ ^(.*?)[[:space:]]+(.*?)$ ]] || exit 1 + rm xtrabackup_binlog_info + echo "CHANGE MASTER TO MASTER_LOG_FILE='${BASH_REMATCH[1]}',\ + MASTER_LOG_POS=${BASH_REMATCH[2]}" > change_master_to.sql.in + fi + # Check if we need to complete a clone by starting replication. + if [[ -f change_master_to.sql.in ]]; then + echo "Waiting for mysqld to be ready (accepting connections)" + until mysql -h 127.0.0.1 -e "SELECT 1"; do sleep 1; done + echo "Initializing replication from clone position" + # In case of container restart, attempt this at-most-once. + mv change_master_to.sql.in change_master_to.sql.orig + mysql -h 127.0.0.1 < /mnt/conf.d/server-id.cnf + # Add an offset to avoid reserved server-id=0 value. + echo server-id=$((100 + $ordinal)) >> /mnt/conf.d/server-id.cnf + # Copy appropriate conf.d files from config-map to emptyDir. + if [[ $ordinal -eq 0 ]]; then + cp /mnt/config-map/master.cnf /mnt/conf.d/ + else + cp /mnt/config-map/slave.cnf /mnt/conf.d/ + fi + volumeMounts: + - name: conf + mountPath: /mnt/conf.d + - name: config-map + mountPath: /mnt/config-map + - name: clone-mysql + image: gcr.io/google-samples/xtrabackup:1.0 + command: + - bash + - -c + - | + set -ex + # Skip the clone if data already exists. + [[ -d /var/lib/mysql/mysql ]] && exit 0 + # Skip the clone on master (ordinal index 0). + [[ `hostname` =~ -([0-9]+)$ ]] || exit 1 + ordinal=${BASH_REMATCH[1]} + [[ $ordinal -eq 0 ]] && exit 0 + # Clone data from previous peer. + ncat --recv-only mysql-$(($ordinal-1)).mysql 3307 | xbstream -x -C /var/lib/mysql + # Prepare the backup. + xtrabackup --prepare --target-dir=/var/lib/mysql + volumeMounts: + - name: data + mountPath: /var/lib/mysql + subPath: mysql + - name: conf + mountPath: /etc/mysql/conf.d + containers: + - name: mysql + image: mysql:5.7 + ports: + - name: mysql + containerPort: 3306 + env: + - name: MYSQL_ALLOW_EMPTY_PASSWORD + value: "1" + resources: + requests: + cpu: 500m + memory: 1Gi + volumeMounts: + - name: data + mountPath: /var/lib/mysql + subPath: mysql + - name: conf + mountPath: /etc/mysql/conf.d + livenessProbe: + exec: + command: + - mysqladmin + - ping + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 5 + readinessProbe: + exec: + command: + - mysql + - -h + - 127.0.0.1 + - -e + - SELECT 1 + initialDelaySeconds: 5 + periodSeconds: 2 + timeoutSeconds: 1 + - name: xtrabackup + image: gcr.io/google-samples/xtrabackup:1.0 + command: + - bash + - -c + - | + set -ex + cd /var/lib/mysql + # Determine binlog position of cloned data, if any. + if [[ -f xtrabackup_slave_info ]]; then + # XtraBackup already generated a partial "CHANGE MASTER TO" query + # because we're cloning from an existing slave. + mv xtrabackup_slave_info change_master_to.sql.in + # Ignore xtrabackup_binlog_info in this case (it's useless). + rm -f xtrabackup_binlog_info + elif [[ -f xtrabackup_binlog_info ]]; then + # We're cloning directly from master. Parse binlog position. + [[ `cat xtrabackup_binlog_info` =~ ^(.*?)[[:space:]]+(.*?)$ ]] || exit 1 + rm xtrabackup_binlog_info + echo "CHANGE MASTER TO MASTER_LOG_FILE='${BASH_REMATCH[1]}',\ + MASTER_LOG_POS=${BASH_REMATCH[2]}" > change_master_to.sql.in + fi + # Check if we need to complete a clone by starting replication. + if [[ -f change_master_to.sql.in ]]; then + echo "Waiting for mysqld to be ready (accepting connections)" + until mysql -h 127.0.0.1 -e "SELECT 1"; do sleep 1; done + echo "Initializing replication from clone position" + # In case of container restart, attempt this at-most-once. + mv change_master_to.sql.in change_master_to.sql.orig + mysql -h 127.0.0.1 < 0 { + value = fmt.Sprintf("%s %s/%s", meta.Kind, meta.Namespace, meta.Name) + } + + fields, err := p.getFields(leaf) + if err != nil { + return nil, err + } + + n := branch.AddMetaBranch(metaString, value) + for i := range fields { + field := fields[i] + + // do leaf node + if len(field.matchingElementsAndFields) == 0 { + n.AddNode(fmt.Sprintf("%s: %s", field.name, field.value)) + continue + } + + // do nested nodes + b := n.AddBranch(field.name) + for j := range field.matchingElementsAndFields { + elem := field.matchingElementsAndFields[j] + b := b.AddBranch(elem.name) + for k := range elem.matchingElementsAndFields { + field := elem.matchingElementsAndFields[k] + b.AddNode(fmt.Sprintf("%s: %s", field.name, field.value)) + } + } + } + + return n, nil +} + +// getFields looks up p.Fields from leaf and structures them into treeFields. +// TODO(pwittrock): simplify this function +func (p TreeWriter) getFields(leaf *yaml.RNode) (treeFields, error) { + fieldsByName := map[string]*treeField{} + + // index nested and non-nested fields + for i := range p.Fields { + f := p.Fields[i] + seq, err := leaf.Pipe(&f) + if err != nil { + return nil, err + } + if seq == nil { + continue + } + + if fieldsByName[f.Name] == nil { + fieldsByName[f.Name] = &treeField{name: f.Name} + } + + // non-nested field -- add directly to the treeFields list + if f.SubName == "" { + // non-nested field -- only 1 element + val, err := yaml.String(seq.Content()[0], yaml.Trim, yaml.Flow) + if err != nil { + return nil, err + } + fieldsByName[f.Name].value = val + continue + } + + // nested-field -- create a parent elem, and index by the 'match' value + if fieldsByName[f.Name].subFieldByMatch == nil { + fieldsByName[f.Name].subFieldByMatch = map[string]treeFields{} + } + index := fieldsByName[f.Name].subFieldByMatch + for j := range seq.Content() { + elem := seq.Content()[j] + matches := f.Matches[elem] + str, err := yaml.String(elem, yaml.Trim, yaml.Flow) + if err != nil { + return nil, err + } + + // map the field by the name of the element + // index the subfields by the matching element so we can put all the fields for the + // same element under the same branch + matchKey := strings.Join(matches, "/") + index[matchKey] = append(index[matchKey], &treeField{name: f.SubName, value: str}) + } + } + + // iterate over collection of all queried fields in the Resource + for _, field := range fieldsByName { + // iterate over collection of elements under the field -- indexed by element name + for match, subFields := range field.subFieldByMatch { + // create a new element for this collection of fields + // note: we will convert name to an index later, but keep the match for sorting + elem := &treeField{name: match} + field.matchingElementsAndFields = append(field.matchingElementsAndFields, elem) + + // iterate over collection of queried fields for the element + for i := range subFields { + // add to the list of fields for this element + elem.matchingElementsAndFields = append(elem.matchingElementsAndFields, subFields[i]) + } + } + // clear this cached data + field.subFieldByMatch = nil + } + + // put the fields in a list so they are ordered + fieldList := treeFields{} + for _, v := range fieldsByName { + fieldList = append(fieldList, v) + } + + // sort the fields + sort.Sort(fieldList) + for i := range fieldList { + field := fieldList[i] + // sort the elements under this field + sort.Sort(field.matchingElementsAndFields) + + for i := range field.matchingElementsAndFields { + element := field.matchingElementsAndFields[i] + // sort the elements under a list field by their name + sort.Sort(element.matchingElementsAndFields) + // set the name of the element to its index + element.name = fmt.Sprintf("%d", i) + } + } + + return fieldList, nil +} + +// treeField wraps a field node +type treeField struct { + // name is the name of the node + name string + + // value is the value of the node -- may be empty + value string + + // matchingElementsAndFields is a slice of fields that go under this as a branch + matchingElementsAndFields treeFields + + // subFieldByMatch caches matchingElementsAndFields indexed by the name of the matching elem + subFieldByMatch map[string]treeFields +} + +// treeFields wraps a slice of treeField so they can be sorted +type treeFields []*treeField + +func (nodes treeFields) Len() int { return len(nodes) } + +func (nodes treeFields) Less(i, j int) bool { + iIndex, iFound := yaml.FieldOrder[nodes[i].name] + jIndex, jFound := yaml.FieldOrder[nodes[j].name] + if iFound && jFound { + return iIndex < jIndex + } + if iFound { + return true + } + if jFound { + return false + } + + if nodes[i].name != nodes[j].name { + return nodes[i].name < nodes[j].name + } + if nodes[i].value != nodes[j].value { + return nodes[i].value < nodes[j].value + } + return false +} + +func (nodes treeFields) Swap(i, j int) { nodes[i], nodes[j] = nodes[j], nodes[i] } diff --git a/kyaml/kio/tree_test.go b/kyaml/kio/tree_test.go new file mode 100644 index 000000000..e1eec5f68 --- /dev/null +++ b/kyaml/kio/tree_test.go @@ -0,0 +1,386 @@ +// Copyright 2019 The Kubernetes Authors. +// SPDX-License-Identifier: Apache-2.0 + +package kio_test + +import ( + "bytes" + "testing" + + "github.com/stretchr/testify/assert" + . "sigs.k8s.io/kustomize/kyaml/kio" + "sigs.k8s.io/kustomize/kyaml/yaml" +) + +func TestPrinter_Write(t *testing.T) { + in := `kind: Deployment +metadata: + labels: + app: nginx3 + name: foo + namespace: default + annotations: + app: nginx3 + kyaml.kustomize.dev/kio/package: foo-package/3 + kyaml.kustomize.dev/kio/path: f3.yaml +spec: + replicas: 1 +--- +kind: Deployment +metadata: + labels: + app: nginx2 + name: foo + namespace: default + annotations: + app: nginx2 + kyaml.kustomize.dev/kio/package: foo-package + kyaml.kustomize.dev/kio/path: f1.yaml +spec: + replicas: 1 +--- +kind: Deployment +metadata: + labels: + app: nginx + annotations: + app: nginx + kyaml.kustomize.dev/kio/package: bar-package + kyaml.kustomize.dev/kio/path: f2.yaml + name: bar +spec: + replicas: 3 +--- +kind: Service +metadata: + name: foo + namespace: default + annotations: + app: nginx + kyaml.kustomize.dev/kio/package: foo-package + kyaml.kustomize.dev/kio/path: f1.yaml +spec: + selector: + app: nginx +` + out := &bytes.Buffer{} + err := Pipeline{ + Inputs: []Reader{&ByteReader{Reader: bytes.NewBufferString(in)}}, + Outputs: []Writer{TreeWriter{Writer: out}}, + }.Execute() + if !assert.NoError(t, err) { + t.FailNow() + } + + if !assert.Equal(t, ` +├── bar-package +│   └── [f2.yaml] Deployment bar +└── foo-package + ├── [f1.yaml] Deployment default/foo + ├── [f1.yaml] Service default/foo + └── foo-package/3 + └── [f3.yaml] Deployment default/foo +`, out.String()) { + t.FailNow() + } +} + +func TestPrinter_Write_base(t *testing.T) { + in := `kind: Deployment +metadata: + labels: + app: nginx3 + name: foo + namespace: default + annotations: + app: nginx3 + kyaml.kustomize.dev/kio/package: . + kyaml.kustomize.dev/kio/path: f3.yaml +spec: + replicas: 1 +--- +kind: Deployment +metadata: + labels: + app: nginx2 + name: foo + namespace: default + annotations: + app: nginx2 + kyaml.kustomize.dev/kio/package: foo-package + kyaml.kustomize.dev/kio/path: f1.yaml +spec: + replicas: 1 +--- +kind: Deployment +metadata: + labels: + app: nginx + annotations: + app: nginx + kyaml.kustomize.dev/kio/package: bar-package + kyaml.kustomize.dev/kio/path: f2.yaml + name: bar +spec: + replicas: 3 +--- +kind: Service +metadata: + name: foo + namespace: default + annotations: + app: nginx + kyaml.kustomize.dev/kio/package: . + kyaml.kustomize.dev/kio/path: f1.yaml +spec: + selector: + app: nginx +` + out := &bytes.Buffer{} + err := Pipeline{ + Inputs: []Reader{&ByteReader{Reader: bytes.NewBufferString(in)}}, + Outputs: []Writer{TreeWriter{Writer: out}}, + }.Execute() + if !assert.NoError(t, err) { + t.FailNow() + } + + if !assert.Equal(t, ` +├── [f1.yaml] Service default/foo +├── [f3.yaml] Deployment default/foo +├── bar-package +│   └── [f2.yaml] Deployment bar +└── foo-package + └── [f1.yaml] Deployment default/foo +`, out.String()) { + t.FailNow() + } +} + +func TestPrinter_Write_sort(t *testing.T) { + in := `apiVersion: extensions/v1 +kind: Deployment +metadata: + labels: + app: nginx2 + name: foo3 + namespace: default + annotations: + app: nginx2 + kyaml.kustomize.dev/kio/package: . + kyaml.kustomize.dev/kio/path: f1.yaml +spec: + replicas: 1 +--- +apiVersion: extensions/v1 +kind: Deployment +metadata: + labels: + app: nginx2 + name: foo3 + namespace: default + annotations: + app: nginx2 + kyaml.kustomize.dev/kio/package: . + kyaml.kustomize.dev/kio/path: f1.yaml +spec: + replicas: 1 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: nginx2 + name: foo3 + namespace: default + annotations: + app: nginx2 + kyaml.kustomize.dev/kio/package: . + kyaml.kustomize.dev/kio/path: f1.yaml +spec: + replicas: 1 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: nginx2 + name: foo2 + namespace: default2 + annotations: + app: nginx2 + kyaml.kustomize.dev/kio/package: . + kyaml.kustomize.dev/kio/path: f1.yaml +spec: + replicas: 1 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: nginx3 + name: foo + namespace: default + annotations: + app: nginx3 + kyaml.kustomize.dev/kio/package: . + kyaml.kustomize.dev/kio/path: f1.yaml +spec: + replicas: 1 +--- +kind: Deployment +metadata: + labels: + app: nginx + annotations: + app: nginx + kyaml.kustomize.dev/kio/package: bar-package + kyaml.kustomize.dev/kio/path: f2.yaml + name: bar +spec: + replicas: 3 +--- +kind: Service +metadata: + name: foo + namespace: default + annotations: + app: nginx + kyaml.kustomize.dev/kio/package: . + kyaml.kustomize.dev/kio/path: f1.yaml +spec: + selector: + app: nginx +` + out := &bytes.Buffer{} + err := Pipeline{ + Inputs: []Reader{&ByteReader{Reader: bytes.NewBufferString(in)}}, + Outputs: []Writer{TreeWriter{Writer: out}}, + }.Execute() + if !assert.NoError(t, err) { + t.FailNow() + } + + if !assert.Equal(t, ` +├── [f1.yaml] Deployment default/foo +├── [f1.yaml] Service default/foo +├── [f1.yaml] Deployment default/foo3 +├── [f1.yaml] Deployment default/foo3 +├── [f1.yaml] Deployment default/foo3 +├── [f1.yaml] Deployment default2/foo2 +└── bar-package + └── [f2.yaml] Deployment bar +`, out.String()) { + t.FailNow() + } +} + +func TestPrinter_metaError(t *testing.T) { + out := &bytes.Buffer{} + err := TreeWriter{Writer: out}.Write([]*yaml.RNode{{}}) + if !assert.NoError(t, err) { + t.FailNow() + } + if !assert.Equal(t, ` +`, out.String()) { + t.FailNow() + } +} + +func TestPrinter_Write_owners(t *testing.T) { + in := ` +apiVersion: v1 +kind: Pod +metadata: + name: cockroachdb-0 + namespace: myapp-staging + ownerReferences: + - apiVersion: apps/v1 + kind: StatefulSet + name: cockroachdb +spec: + containers: + - name: cockroachdb + image: cockraochdb:1.1.1 +--- +apiVersion: v1 +kind: Pod +metadata: + name: cockroachdb-1 + namespace: myapp-staging + ownerReferences: + - apiVersion: apps/v1 + kind: StatefulSet + name: cockroachdb +spec: + containers: + - name: cockroachdb + image: cockraochdb:1.1.1 +--- +apiVersion: v1 +kind: Pod +metadata: + name: cockroachdb-2 + namespace: myapp-staging + ownerReferences: + - apiVersion: apps/v1 + kind: StatefulSet + name: cockroachdb +spec: + containers: + - name: cockroachdb + image: cockraochdb:1.1.0 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: cockroachdb + namespace: myapp-staging + ownerReferences: + - apiVersion: app.k8s.io/v1beta1 + kind: Application + name: myapp +spec: + replicas: 3 + containers: + - name: cockroachdb + image: cockraochdb:1.1.1 +--- +apiVersion: v1 +kind: Service +metadata: + name: cockroachdb + namespace: myapp-staging + ownerReferences: + - apiVersion: app.k8s.io/v1beta1 + kind: Application + name: myapp +--- +apiVersion: app.k8s.io/v1beta1 +kind: Application +metadata: + labels: + app.kubernetes.io/name: myapp + name: myapp + namespace: myapp-staging +` + out := &bytes.Buffer{} + err := Pipeline{ + Inputs: []Reader{&ByteReader{Reader: bytes.NewBufferString(in)}}, + Outputs: []Writer{TreeWriter{Writer: out, Structure: TreeStructureGraph}}, + }.Execute() + if !assert.NoError(t, err) { + t.FailNow() + } + + if !assert.Equal(t, `. +└── [Resource] Application myapp-staging/myapp + ├── [Resource] Service myapp-staging/cockroachdb + └── [Resource] StatefulSet myapp-staging/cockroachdb + ├── [Resource] Pod myapp-staging/cockroachdb-0 + ├── [Resource] Pod myapp-staging/cockroachdb-1 + └── [Resource] Pod myapp-staging/cockroachdb-2 +`, out.String()) { + t.FailNow() + } +} diff --git a/kyaml/sets/sets.go b/kyaml/sets/sets.go new file mode 100644 index 000000000..07f02afb0 --- /dev/null +++ b/kyaml/sets/sets.go @@ -0,0 +1,64 @@ +// Copyright 2019 The Kubernetes Authors. +// SPDX-License-Identifier: Apache-2.0 + +package sets + +type String map[string]interface{} + +func (s String) Len() int { + return len(s) +} + +func (s String) List() []string { + var val []string + for k := range s { + val = append(val, k) + } + return val +} + +func (s String) Has(val string) bool { + _, found := s[val] + return found +} + +func (s String) Insert(vals ...string) { + for _, val := range vals { + s[val] = nil + } +} + +func (s String) Difference(s2 String) String { + s3 := String{} + for k := range s { + if _, found := s2[k]; !found { + s3.Insert(k) + } + } + return s3 +} + +func (s String) SymmetricDifference(s2 String) String { + s3 := String{} + for k := range s { + if _, found := s2[k]; !found { + s3.Insert(k) + } + } + for k := range s2 { + if _, found := s[k]; !found { + s3.Insert(k) + } + } + return s3 +} + +func (s String) Intersection(s2 String) String { + s3 := String{} + for k := range s { + if _, found := s2[k]; found { + s3.Insert(k) + } + } + return s3 +} diff --git a/kyaml/yaml/alias.go b/kyaml/yaml/alias.go new file mode 100644 index 000000000..175775287 --- /dev/null +++ b/kyaml/yaml/alias.go @@ -0,0 +1,44 @@ +// Copyright 2019 The Kubernetes Authors. +// SPDX-License-Identifier: Apache-2.0 + +package yaml + +import ( + "io" + + "gopkg.in/yaml.v3" +) + +// Expose the yaml.v3 functions so this package can be used as a replacement + +type Decoder = yaml.Decoder +type Encoder = yaml.Encoder +type IsZeroer = yaml.IsZeroer +type Kind = yaml.Kind +type Marshaler = yaml.Marshaler +type Node = yaml.Node +type Style = yaml.Style +type TypeError = yaml.TypeError +type Unmarshaler = yaml.Unmarshaler + +var Marshal = yaml.Marshal +var UnMarshal = yaml.Unmarshal +var NewDecoder = yaml.NewDecoder +var NewEncoder = func(w io.Writer) *yaml.Encoder { + e := yaml.NewEncoder(w) + e.SetIndent(2) + return e +} + +var AliasNode yaml.Kind = yaml.AliasNode +var DocumentNode yaml.Kind = yaml.DocumentNode +var MappingNode yaml.Kind = yaml.MappingNode +var ScalarNode yaml.Kind = yaml.ScalarNode +var SequenceNode yaml.Kind = yaml.SequenceNode + +var DoubleQuotedStyle yaml.Style = yaml.DoubleQuotedStyle +var FlowStyle yaml.Style = yaml.FlowStyle +var FoldedStyle yaml.Style = yaml.FoldedStyle +var LiteralStyle yaml.Style = yaml.LiteralStyle +var SingleQuotedStyle yaml.Style = yaml.SingleQuotedStyle +var TaggedStyle yaml.Style = yaml.TaggedStyle diff --git a/kyaml/yaml/doc.go b/kyaml/yaml/doc.go new file mode 100644 index 000000000..1f57e499b --- /dev/null +++ b/kyaml/yaml/doc.go @@ -0,0 +1,12 @@ +// Copyright 2019 The Kubernetes Authors. +// SPDX-License-Identifier: Apache-2.0 + +// Package yaml contains low-level libraries for manipulating individual Kubernetes Resource +// Configuration yaml. +// +// It exports the public pieces of "gopkg.in/yaml.v3", so can be used as a drop in replacement. +// +// This package should be used over sigs.k8s.io/yaml: +// - If retaining or modifying yaml comments, structure, formatting +// - If Resources should be round tripped without dropping fields +package yaml diff --git a/kyaml/yaml/example_test.go b/kyaml/yaml/example_test.go new file mode 100644 index 000000000..136add0a1 --- /dev/null +++ b/kyaml/yaml/example_test.go @@ -0,0 +1,841 @@ +// Copyright 2019 The Kubernetes Authors. +// SPDX-License-Identifier: Apache-2.0 + +package yaml_test + +import ( + "fmt" + "log" + + . "sigs.k8s.io/kustomize/kyaml/yaml" +) + +func Example() { + obj, err := Parse(`apiVersion: apps/v1 +kind: Deployment +metadata: + name: app + labels: + app: java + annotations: + a.b.c: d.e.f +`) + if err != nil { + log.Fatal(err) + } + + containers, err := Parse(` +- name: nginx # first container + image: nginx +- name: nginx2 # second container + image: nginx2 +`) + if err != nil { + log.Fatal(err) + } + + node, err := obj.Pipe( + LookupCreate(SequenceNode, "spec", "template", "spec", "containers"), + Append(containers.YNode().Content...)) + if err != nil { + log.Fatal(err) + } + + fmt.Println(node.String()) + fmt.Println(obj.String()) + // Output: + // + // apiVersion: apps/v1 + // kind: Deployment + // metadata: + // name: app + // labels: + // app: java + // annotations: + // a.b.c: d.e.f + // spec: + // template: + // spec: + // containers: + // - name: nginx # first container + // image: nginx + // - name: nginx2 # second container + // image: nginx2 + // +} + +func ExampleAppend_appendScalars() { + obj, err := Parse(` +- a +- b +`) + if err != nil { + log.Fatal(err) + } + _, err = obj.Pipe(Append(&Node{Value: "c", Kind: ScalarNode})) + if err != nil { + log.Fatal(err) + } + node, err := obj.Pipe(Append( + &Node{Value: "c", Kind: ScalarNode}, + &Node{Value: "d", Kind: ScalarNode}, + )) + if err != nil { + log.Fatal(err) + } + fmt.Println(node.String()) + fmt.Println(obj.String()) + // Output: + // + // - a + // - b + // - c + // - c + // - d + // +} + +func ExampleAppend_appendMap() { + obj, err := Parse(` +- name: foo +- name: bar +`) + if err != nil { + log.Fatal(err) + } + elem, err := Parse("name: baz") + if err != nil { + log.Fatal(err) + } + node, err := obj.Pipe(Append(elem.YNode())) + if err != nil { + log.Fatal(err) + } + + // Expect the node to contain the appended element because only + // 1 element was appended + fmt.Println(node.String()) + fmt.Println(obj.String()) + // Output: + // name: baz + // + // - name: foo + // - name: bar + // - name: baz + // +} + +func ExampleClear() { + obj, err := Parse(` +kind: Deployment +metadata: + name: app + annotations: + a.b.c: d.e.f + g: h +spec: + template: {} +`) + if err != nil { + log.Fatal(err) + } + node, err := obj.Pipe(Clear("metadata")) + if err != nil { + log.Fatal(err) + } + fmt.Println(node.String()) + fmt.Println(obj.String()) + // Output: + // name: app + // annotations: + // a.b.c: d.e.f + // g: h + // + // kind: Deployment + // spec: + // template: {} + // +} + +func ExampleGet() { + obj, err := Parse(`apiVersion: apps/v1 +kind: Deployment +metadata: + name: app + annotations: + a.b.c: d.e.f + g: h +spec: + template: {} +`) + if err != nil { + log.Fatal(err) + } + node, err := obj.Pipe(Get("metadata")) + if err != nil { + log.Fatal(err) + } + fmt.Println(node.String()) + fmt.Println(obj.String()) + // Output: + // name: app + // annotations: + // a.b.c: d.e.f + // g: h + // + // apiVersion: apps/v1 + // kind: Deployment + // metadata: + // name: app + // annotations: + // a.b.c: d.e.f + // g: h + // spec: + // template: {} + // +} + +func ExampleGet_notFound() { + obj, err := Parse(`apiVersion: apps/v1 +kind: Deployment +spec: + template: {} +`) + if err != nil { + log.Fatal(err) + } + node, err := obj.Pipe(FieldMatcher{Name: "metadata"}) + if err != nil { + log.Fatal(err) + } + fmt.Println(node.String()) + fmt.Println(obj.String()) + // Output: + // + // apiVersion: apps/v1 + // kind: Deployment + // spec: + // template: {} + // +} + +func ExampleElementMatcher_Filter() { + obj, err := Parse(` +- a +- b +`) + if err != nil { + log.Fatal(err) + } + elem, err := obj.Pipe(ElementMatcher{ + FieldValue: "c", Create: NewScalarRNode("c"), + }) + if err != nil { + log.Fatal(err) + } + + fmt.Println(elem.String()) + fmt.Println(obj.String()) + // Output: + // c + // + // - a + // - b + // - c + // +} + +func ExampleElementMatcher_Filter_primitiveFound() { + obj, err := Parse(` +- a +- b +- c +`) + if err != nil { + log.Fatal(err) + } + elem, err := obj.Pipe(ElementMatcher{ + FieldValue: "c", Create: NewScalarRNode("c"), + }) + if err != nil { + log.Fatal(err) + } + fmt.Println(elem.String()) + fmt.Println(obj.String()) + // Output: + // c + // + // - a + // - b + // - c + // +} + +func ExampleElementMatcher_Filter_objectNotFound() { + obj, err := Parse(` +- name: foo +- name: bar +`) + if err != nil { + log.Fatal(err) + } + append, err := Parse(` +name: baz +image: nginx +`) + if err != nil { + log.Fatal(err) + } + elem, err := obj.Pipe(ElementMatcher{ + FieldName: "name", FieldValue: "baz", Create: append}) + if err != nil { + log.Fatal(err) + } + fmt.Println(elem.String()) + fmt.Println(obj.String()) + // Output: + // name: baz + // image: nginx + // + // - name: foo + // - name: bar + // - name: baz + // image: nginx + // +} + +func ExampleElementMatcher_Filter_objectFound() { + obj, err := Parse(` +- name: foo +- name: bar +- name: baz +`) + if err != nil { + log.Fatal(err) + } + append, err := Parse(` +name: baz +image: nginx +`) + if err != nil { + log.Fatal(err) + } + elem, err := obj.Pipe(ElementMatcher{ + FieldName: "name", FieldValue: "baz", Create: append}) + if err != nil { + log.Fatal(err) + } + fmt.Println(elem.String()) + fmt.Println(obj.String()) + // Output: + // name: baz + // + // - name: foo + // - name: bar + // - name: baz + // +} + +func ExampleFieldMatcher_Filter() { + obj, err := Parse(` +kind: Deployment +spec: + template: {} +`) + if err != nil { + log.Fatal(err) + } + value, err := Parse(` +name: app +annotations: + a.b.c: d.e.f + g: h +`) + if err != nil { + log.Fatal(err) + } + elem, err := obj.Pipe(FieldMatcher{ + Name: "metadata", Value: value, Create: value}) + if err != nil { + log.Fatal(err) + } + fmt.Println(elem.String()) + fmt.Println(obj.String()) + // Output: + // name: app + // annotations: + // a.b.c: d.e.f + // g: h + // + // kind: Deployment + // spec: + // template: {} + // metadata: + // name: app + // annotations: + // a.b.c: d.e.f + // g: h + // +} + +func ExampleLookup_element() { + obj, err := Parse(`apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: java + annotations: + a.b.c: d.e.f + name: app +spec: + templates: + spec: + containers: + - name: nginx + image: nginx:latest +`) + if err != nil { + log.Fatal(err) + } + value, err := obj.Pipe(Lookup( + "spec", "templates", "spec", "containers", "[name=nginx]")) + if err != nil { + log.Fatal(err) + } + + fmt.Println(value.String()) + // Output: + // name: nginx + // image: nginx:latest + // +} + +func ExampleLookup_sequence() { + obj, err := Parse(`apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: java + annotations: + a.b.c: d.e.f + name: app +spec: + templates: + spec: + containers: + - name: nginx + image: nginx:latest +`) + if err != nil { + log.Fatal(err) + } + value, err := obj.Pipe(Lookup( + "spec", "templates", "spec", "containers")) + if err != nil { + log.Fatal(err) + } + + fmt.Println(value.String()) + // Output: + // - name: nginx + // image: nginx:latest + // +} + +func ExampleLookup_scalar() { + obj, err := Parse(`apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: java + annotations: + a.b.c: d.e.f + name: app +spec: + templates: + spec: + containers: + - name: nginx + image: nginx:latest +`) + if err != nil { + log.Fatal(err) + } + value, err := obj.Pipe(Lookup( + "spec", "templates", "spec", "containers", "[name=nginx]", "image")) + if err != nil { + log.Fatal(err) + } + + fmt.Println(value.String()) + // Output: + // nginx:latest + // +} + +func ExampleLookupCreate_element() { + obj, err := Parse(` +kind: Deployment +metadata: + labels: + app: java + annotations: + a.b.c: d.e.f + name: app +`) + if err != nil { + log.Fatal(err) + } + rs, err := obj.Pipe(LookupCreate( + MappingNode, "spec", "templates", "spec", "containers", "[name=nginx]")) + if err != nil { + log.Fatal(err) + } + + fmt.Println(rs.String()) + fmt.Println("---") + fmt.Println(obj.String()) + // Output: + // name: nginx + // + // --- + // kind: Deployment + // metadata: + // labels: + // app: java + // annotations: + // a.b.c: d.e.f + // name: app + // spec: + // templates: + // spec: + // containers: + // - name: nginx + // +} + +func ExampleLookupCreate_sequence() { + obj, err := Parse(` +kind: Deployment +metadata: + labels: + app: java + annotations: + a.b.c: d.e.f + name: app +`) + if err != nil { + log.Fatal(err) + } + rs, err := obj.Pipe(LookupCreate( + SequenceNode, "spec", "templates", "spec", "containers")) + if err != nil { + log.Fatal(err) + } + + fmt.Println(rs.String()) + fmt.Println("---") + fmt.Println(obj.String()) + // Output: + // [] + // + // --- + // kind: Deployment + // metadata: + // labels: + // app: java + // annotations: + // a.b.c: d.e.f + // name: app + // spec: + // templates: + // spec: + // containers: [] + // +} + +func ExamplePathGetter_Filter() { + obj, err := Parse(` +kind: Deployment +metadata: + labels: + app: java + annotations: + a.b.c: d.e.f + name: app +`) + if err != nil { + log.Fatal(err) + } + rs, err := obj.Pipe(PathGetter{ + Path: []string{"spec", "templates", "spec", "containers", "[name=nginx]", "image"}, + Create: ScalarNode, + }) + if err != nil { + log.Fatal(err) + } + rs.Document().Style = SingleQuotedStyle + + fmt.Println(rs.String()) + fmt.Println("---") + fmt.Println(obj.String()) + // Output: + // '' + // + // --- + // kind: Deployment + // metadata: + // labels: + // app: java + // annotations: + // a.b.c: d.e.f + // name: app + // spec: + // templates: + // spec: + // containers: + // - name: nginx + // image: '' + // +} + +func ExampleLookupCreate_object() { + obj, err := Parse(` +kind: Deployment +metadata: + labels: + app: java + annotations: + a.b.c: d.e.f + name: app +`) + if err != nil { + log.Fatal(err) + } + rs, err := obj.Pipe(LookupCreate( + MappingNode, "spec", "templates", "spec")) + if err != nil { + log.Fatal(err) + } + + fmt.Println(rs.String()) + fmt.Println("---") + fmt.Println(obj.String()) + // Output: + // {} + // + // --- + // kind: Deployment + // metadata: + // labels: + // app: java + // annotations: + // a.b.c: d.e.f + // name: app + // spec: + // templates: + // spec: {} + // +} + +func ExampleLookup_notFound() { + obj, err := Parse(` +kind: Deployment +metadata: + labels: + app: java + annotations: + a.b.c: d.e.f + name: app +`) + if err != nil { + log.Fatal(err) + } + rs, err := obj.Pipe(Lookup("spec", "templates", "spec")) + if err != nil { + log.Fatal(err) + } + + fmt.Println(rs) + fmt.Println("---") + fmt.Println(obj.String()) + // Output: + // + // --- + // kind: Deployment + // metadata: + // labels: + // app: java + // annotations: + // a.b.c: d.e.f + // name: app + // +} + +func ExampleSetField_stringValue() { + obj, err := Parse(`apiVersion: apps/v1 +kind: Deployment +metadata: + name: app +`) + if err != nil { + log.Fatal(err) + } + _, err = obj.Pipe(SetField("foo", NewScalarRNode("bar"))) + if err != nil { + log.Fatal(err) + } + + fmt.Println(obj.String()) + // Output: + // apiVersion: apps/v1 + // kind: Deployment + // metadata: + // name: app + // foo: bar + // +} + +func ExampleSetField_stringValueOverwrite() { + obj, err := Parse(`apiVersion: apps/v1 +kind: Deployment +metadata: + name: app +foo: baz +`) + if err != nil { + // handle error + } + // set metadata.annotations.foo = bar + _, err = obj.Pipe(SetField("foo", NewScalarRNode("bar"))) + if err != nil { + // handle error + } + + fmt.Println(obj.String()) + // Output: + // apiVersion: apps/v1 + // kind: Deployment + // metadata: + // name: app + // foo: bar + // +} + +func ExampleSetField() { + obj, err := Parse(`apiVersion: apps/v1 +kind: Deployment +metadata: + name: app + labels: + app: java + annotations: + a.b.c: d.e.f +`) + if err != nil { + log.Fatal(err) + } + + containers, err := Parse(` +- name: nginx # first container + image: nginx +- name: nginx2 # second container + image: nginx2 +`) + if err != nil { + log.Fatal(err) + } + + _, err = obj.Pipe( + LookupCreate(MappingNode, "spec", "template", "spec"), + SetField("containers", containers)) + if err != nil { + log.Fatal(err) + } + + fmt.Println(obj.String()) + // Output: + // apiVersion: apps/v1 + // kind: Deployment + // metadata: + // name: app + // labels: + // app: java + // annotations: + // a.b.c: d.e.f + // spec: + // template: + // spec: + // containers: + // - name: nginx # first container + // image: nginx + // - name: nginx2 # second container + // image: nginx2 + // +} + +func ExampleTee() { + obj, err := Parse(`apiVersion: apps/v1 +kind: Deployment +metadata: + name: nginx-deployment + labels: + app: nginx +spec: + replicas: 3 + selector: + matchLabels: + app: nginx + template: + metadata: + labels: + app: nginx + spec: + containers: + - name: nginx + image: nginx:1.7.9 + ports: + - containerPort: 80 +`) + if err != nil { + // handle error + } + // set metadata.annotations.foo = bar + _, err = obj.Pipe( + Lookup("spec", "template", "spec", "containers", "[name=nginx]"), + Tee(SetField("filter", NewListRNode("foo"))), + SetField("args", NewListRNode("baz", "bar"))) + if err != nil { + // handle error + } + + fmt.Println(obj.String()) + // Output: + // apiVersion: apps/v1 + // kind: Deployment + // metadata: + // name: nginx-deployment + // labels: + // app: nginx + // spec: + // replicas: 3 + // selector: + // matchLabels: + // app: nginx + // template: + // metadata: + // labels: + // app: nginx + // spec: + // containers: + // - name: nginx + // image: nginx:1.7.9 + // ports: + // - containerPort: 80 + // filter: + // - foo + // args: + // - baz + // - bar + // +} diff --git a/kyaml/yaml/filters.go b/kyaml/yaml/filters.go new file mode 100644 index 000000000..2fa76eb34 --- /dev/null +++ b/kyaml/yaml/filters.go @@ -0,0 +1,140 @@ +// Copyright 2019 The Kubernetes Authors. +// SPDX-License-Identifier: Apache-2.0 + +package yaml + +import ( + "fmt" + "regexp" + "sort" + "strings" +) + +var Filters = map[string]func() Filter{ + "AnnotationClearer": func() Filter { return &AnnotationClearer{} }, + "AnnotationGetter": func() Filter { return &AnnotationGetter{} }, + "AnnotationSetter": func() Filter { return &AnnotationSetter{} }, + "ElementAppender": func() Filter { return &ElementAppender{} }, + "ElementMatcher": func() Filter { return &ElementMatcher{} }, + "FieldClearer": func() Filter { return &FieldClearer{} }, + "FilterMatcher": func() Filter { return &FilterMatcher{} }, + "FieldMatcher": func() Filter { return &FieldMatcher{} }, + "FieldSetter": func() Filter { return &FieldSetter{} }, + "PathGetter": func() Filter { return &PathGetter{} }, + "PathMatcher": func() Filter { return &PathMatcher{} }, + "Parser": func() Filter { return &Parser{} }, + "PrefixSetter": func() Filter { return &PrefixSetter{} }, + "ValueReplacer": func() Filter { return &ValueReplacer{} }, + "SuffixSetter": func() Filter { return &SuffixSetter{} }, + "TeePiper": func() Filter { return &TeePiper{} }, +} + +// YFilter wraps the GrepFilter interface so it can be unmarshalled into a struct. +type YFilter struct { + Filter +} + +func (y YFilter) MarshalYAML() (interface{}, error) { + return y.Filter, nil +} + +func (y *YFilter) UnmarshalYAML(unmarshal func(interface{}) error) error { + meta := &ResourceMeta{} + if err := unmarshal(meta); err != nil { + return err + } + if filter, found := Filters[meta.Kind]; !found { + var knownFilters []string + for k := range Filters { + knownFilters = append(knownFilters, k) + } + sort.Strings(knownFilters) + return fmt.Errorf("unsupported GrepFilter Kind %s: may be one of: [%s]", + meta.Kind, strings.Join(knownFilters, ",")) + } else { + y.Filter = filter() + } + if err := unmarshal(y.Filter); err != nil { + return err + } + return nil +} + +type YFilters []YFilter + +func (y YFilters) Filters() []Filter { + var f []Filter + for i := range y { + f = append(f, y[i].Filter) + } + return f +} + +type FilterMatcher struct { + Kind string `yaml:"kind"` + + // Filters are the set of Filters run by TeePiper. + Filters YFilters `yaml:"pipeline,omitempty"` +} + +func (t FilterMatcher) Filter(rn *RNode) (*RNode, error) { + v, err := rn.Pipe(t.Filters.Filters()...) + if v == nil || err != nil { + return nil, err + } + // return the original input if the pipeline resolves to true + return rn, err +} + +type ValueReplacer struct { + Kind string `yaml:"kind"` + + StringMatch string `yaml:"stringMatch"` + RegexMatch string `yaml:"regexMatch"` + Replace string `yaml:"replace"` + Count int `yaml:"count"` +} + +func (s ValueReplacer) Filter(object *RNode) (*RNode, error) { + if s.Count == 0 { + s.Count = -1 + } + if s.StringMatch != "" { + object.value.Value = strings.Replace(object.value.Value, s.StringMatch, s.Replace, s.Count) + } else if s.RegexMatch != "" { + r, err := regexp.Compile(s.RegexMatch) + if err != nil { + return nil, fmt.Errorf("ValueReplacer RegexMatch does not compile: %v", err) + } + object.value.Value = r.ReplaceAllString(object.value.Value, s.Replace) + } else { + return nil, fmt.Errorf("ValueReplacer missing StringMatch and RegexMatch") + } + return object, nil +} + +type PrefixSetter struct { + Kind string `yaml:"kind"` + + Value string `yaml:"value"` +} + +func (s PrefixSetter) Filter(object *RNode) (*RNode, error) { + if !strings.HasPrefix(object.value.Value, s.Value) { + object.value.Value = s.Value + object.value.Value + } + return object, nil +} + +type SuffixSetter struct { + Kind string `yaml:"kind"` + + Value string `yaml:"value"` +} + +func (s SuffixSetter) Filter(object *RNode) (*RNode, error) { + if !strings.HasSuffix(object.value.Value, s.Value) { + object.value.Value = object.value.Value + s.Value + } + return object, nil +} diff --git a/kyaml/yaml/fns.go b/kyaml/yaml/fns.go new file mode 100644 index 000000000..392f9f487 --- /dev/null +++ b/kyaml/yaml/fns.go @@ -0,0 +1,585 @@ +// Copyright 2019 The Kubernetes Authors. +// SPDX-License-Identifier: Apache-2.0 + +package yaml + +import ( + "fmt" + "regexp" + "strings" + + "github.com/davecgh/go-spew/spew" + "github.com/go-errors/errors" + "gopkg.in/yaml.v3" +) + +// Append creates an ElementAppender +func Append(elements ...*yaml.Node) ElementAppender { + return ElementAppender{Elements: elements} +} + +// ElementAppender adds all element to a SequenceNode's Content. +// Returns Elements[0] if len(Elements) == 1, otherwise returns nil. +type ElementAppender struct { + Kind string `yaml:"kind,omitempty"` + + // Elem is the value to append. + Elements []*yaml.Node `yaml:"elements,omitempty"` +} + +func (a ElementAppender) Filter(rn *RNode) (*RNode, error) { + if err := ErrorIfInvalid(rn, yaml.SequenceNode); err != nil { + return nil, err + } + for i := range a.Elements { + rn.YNode().Content = append(rn.Content(), a.Elements[i]) + } + if len(a.Elements) == 1 { + return NewRNode(a.Elements[0]), nil + } + return nil, nil +} + +// ElementSetter sets the value for an Element in an associative list. ElementSetter +// will remove any elements which are empty. +type ElementSetter struct { + Kind string `yaml:"kind,omitempty"` + + // Element is the new value to set -- remove the existing element if nil + Element *Node + + // Key is a field on the elements. It is used to find the matching element to + // update / delete. + Key string `yaml:"key,omitempty"` + + // Value is a field value on the elements. It is used to find matching elements to + // update / delete. + Value string `yaml:"value,omitempty"` +} + +func (e ElementSetter) Filter(rn *RNode) (*RNode, error) { + if err := ErrorIfInvalid(rn, SequenceNode); err != nil { + return nil, err + } + + // build the new Content slice + var newContent []*yaml.Node + matchingElementFound := false + for i := range rn.YNode().Content { + elem := rn.Content()[i] + + // empty elements are not valid -- they at least need an associative key + if IsEmpty(NewRNode(elem)) { + continue + } + + // check if this is the element we are matching + val, err := NewRNode(elem).Pipe(FieldMatcher{Name: e.Key, StringValue: e.Value}) + if err != nil { + return nil, err + } + if val == nil { + // not the element we are looking for, keep it in the Content + newContent = append(newContent, elem) + continue + } + matchingElementFound = true + + // deletion operation -- remove the element from the new Content + if e.Element == nil { + continue + } + // replace operation -- replace the element in the Content + newContent = append(newContent, e.Element) + } + rn.YNode().Content = newContent + + // deletion operation -- return nil + if IsMissingOrNull(NewRNode(e.Element)) { + return nil, nil + } + + // append operation -- add the element to the Content + if !matchingElementFound { + rn.YNode().Content = append(rn.YNode().Content, e.Element) + } + + return NewRNode(e.Element), nil +} + +// Clear returns a FieldClearer +func Clear(name string) FieldClearer { + return FieldClearer{Name: name} +} + +// FieldClearer removes the field or map key. +// Returns a RNode with the removed field or map entry. +type FieldClearer struct { + Kind string `yaml:"kind,omitempty"` + + // Name is the name of the field or key in the map. + Name string `yaml:"name,omitempty"` + + IfEmpty bool `yaml:"ifEmpty,omitempty"` +} + +func (c FieldClearer) Filter(rn *RNode) (*RNode, error) { + if err := ErrorIfInvalid(rn, yaml.MappingNode); err != nil { + return nil, err + } + + for i := 0; i < len(rn.Content()); i += 2 { + + // if name matches, remove these 2 elements from the list because + // they are treated as a fieldName/fieldValue pair. + if rn.Content()[i].Value == c.Name { + if c.IfEmpty { + if len(rn.Content()[i+1].Content) > 0 { + continue + } + } + + // save the item we are about to remove + removed := NewRNode(rn.Content()[i+1]) + if len(rn.YNode().Content) > i+2 { + // remove from the middle of the list + rn.YNode().Content = append( + rn.Content()[:i], + rn.Content()[i+2:len(rn.YNode().Content)]...) + } else { + // remove from the end of the list + rn.YNode().Content = rn.Content()[:i] + } + + // return the removed field name and value + return removed, nil + } + } + // nothing removed + return nil, nil +} + +func MatchElement(field, value string) ElementMatcher { + return ElementMatcher{FieldName: field, FieldValue: value} +} + +// ElementMatcher returns the first element from a Sequence matching the +// specified field's value. +type ElementMatcher struct { + Kind string `yaml:"kind,omitempty"` + + // FieldName will attempt to match this field in each list element. + // Optional. Leave empty for lists of primitives (ScalarNode). + FieldName string `yaml:"name,omitempty"` + + // FieldValue will attempt to match each element field to this value. + // For lists of primitives, this will be used to match the primitive value. + FieldValue string `yaml:"value,omitempty"` + + // Create will create the Element if it is not found + Create *RNode `yaml:"create,omitempty"` +} + +func (e ElementMatcher) Filter(rn *RNode) (*RNode, error) { + if err := ErrorIfInvalid(rn, yaml.SequenceNode); err != nil { + return nil, err + } + + // SequenceNode Content is a slice of ScalarNodes. Each ScalarNode has a + // YNode containing the primitive data. + if len(e.FieldName) == 0 { + for i := range rn.Content() { + if rn.Content()[i].Value == e.FieldValue { + return &RNode{value: rn.Content()[i]}, nil + } + } + if e.Create != nil { + return rn.Pipe(Append(e.Create.YNode())) + } + return nil, nil + } + + // SequenceNode Content is a slice of MappingNodes. Each MappingNode has Content + // with a slice of key-value pairs containing the fields. + for i := range rn.Content() { + // cast the entry to a RNode so we can operate on it + elem := NewRNode(rn.Content()[i]) + + field, err := elem.Pipe(MatchField(e.FieldName, e.FieldValue)) + if IsFoundOrError(field, err) { + return elem, err + } + } + + // create the element + if e.Create != nil { + return rn.Pipe(Append(e.Create.YNode())) + } + + return nil, nil +} + +func Get(name string) FieldMatcher { + return FieldMatcher{Name: name} +} + +func MatchField(name, value string) FieldMatcher { + return FieldMatcher{Name: name, Value: NewScalarRNode(value)} +} + +func Match(value string) FieldMatcher { + return FieldMatcher{Value: NewScalarRNode(value)} +} + +// FieldMatcher returns the value of a named field or map entry. +type FieldMatcher struct { + Kind string `yaml:"kind,omitempty"` + + // Name of the field to return + Name string `yaml:"name,omitempty"` + + // YNode of the field to return. + // Optional. Will only need to match field name if unset. + Value *RNode `yaml:"value,omitempty"` + + StringValue string `yaml:"stringValue,omitempty"` + + StringRegexValue string `yaml:"stringRegexValue,omitempty"` + + // Create will cause the field to be created with this value + // if it is set. + Create *RNode `yaml:"create,omitempty"` +} + +func (f FieldMatcher) Filter(rn *RNode) (*RNode, error) { + if f.StringValue != "" && f.Value == nil { + f.Value = NewScalarRNode(f.StringValue) + } + + // never match nil or null fields + if IsMissingOrNull(rn) { + return nil, nil + } + + if f.Name == "" { + if err := ErrorIfInvalid(rn, yaml.ScalarNode); err != nil { + return nil, err + } + if f.StringRegexValue != "" { + // TODO(pwittrock): pre-compile this when unmarshalling and cache to a field + rg, err := regexp.Compile(f.StringRegexValue) + if err != nil { + return nil, err + } + if match := rg.MatchString(rn.value.Value); match { + return rn, nil + } + return nil, nil + } else if rn.value.Value == f.Value.YNode().Value { + return rn, nil + } else { + return nil, nil + } + } + + if err := ErrorIfInvalid(rn, yaml.MappingNode); err != nil { + return nil, err + } + + for i := 0; i < len(rn.Content()); IncrementFieldIndex(&i) { + isMatchingField := rn.Content()[i].Value == f.Name + if isMatchingField { + requireMatchFieldValue := f.Value != nil + if !requireMatchFieldValue || rn.Content()[i+1].Value == f.Value.YNode().Value { + return NewRNode(rn.Content()[i+1]), nil + } + } + } + + if f.Create != nil { + return rn.Pipe(SetField(f.Name, f.Create)) + } + + return nil, nil +} + +func Lookup(path ...string) PathGetter { + return PathGetter{Path: path} +} + +func LookupCreate(kind yaml.Kind, path ...string) PathGetter { + return PathGetter{Path: path, Create: kind} +} + +// PathGetter returns the RNode under Path. +type PathGetter struct { + Kind string `yaml:"kind,omitempty"` + + // Path is a slice of parts leading to the RNode to lookup. + // Each path part may be one of: + // * FieldMatcher -- e.g. "spec" + // * Map Key -- e.g. "app.k8s.io/version" + // * List Entry -- e.g. "[name=nginx]" or "[=-jar]" + // + // Map Keys and Fields are equivalent. + // See FieldMatcher for more on Fields and Map Keys. + // + // List Entries are specified as map entry to match [fieldName=fieldValue]. + // See Elem for more on List Entries. + // + // Examples: + // * spec.template.spec.container with matching name: [name=nginx] + // * spec.template.spec.container.argument matching a value: [=-jar] + Path []string `yaml:"path,omitempty"` + + // Create will cause missing path parts to be created as they are walked. + // + // * The leaf Node (final path) will be created with a Kind matching Create + // * Intermediary Nodes will be created as either a MappingNodes or + // SequenceNodes as appropriate for each's Path location. + Create yaml.Kind `yaml:"create,omitempty"` + + // Style is the style to apply to created value Nodes. + // Created key Nodes keep an unspecified Style. + Style yaml.Style `yaml:"style,omitempty"` +} + +func (l PathGetter) Filter(rn *RNode) (*RNode, error) { + var err error + fieldPath := append([]string{}, rn.FieldPath()...) + match := rn + + // iterate over path until encountering an error or missing value + l.Path = cleanPath(l.Path) + for i := range l.Path { + var part, nextPart string + part = l.Path[i] + if len(l.Path) > i+1 { + nextPart = l.Path[i+1] + } + if IsListIndex(part) { + match, err = l.doElem(match, part) + } else { + fieldPath = append(fieldPath, part) + match, err = l.doField(match, part, l.getKind(nextPart)) + } + if IsMissingOrError(match, err) { + return nil, err + } + match.AppendToFieldPath(fieldPath...) + } + return match, nil +} + +func (l PathGetter) doElem(rn *RNode, part string) (*RNode, error) { + var match *RNode + name, value, err := SplitIndexNameValue(part) + if err != nil { + return nil, err + } + if !IsCreate(l.Create) { + return rn.Pipe(MatchElement(name, value)) + } + + var elem *RNode + primitiveElement := len(name) == 0 + if primitiveElement { + // append a ScalarNode + elem = NewScalarRNode(value) + elem.YNode().Style = l.Style + match = elem + } else { + // append a MappingNode + match = NewRNode(&yaml.Node{Kind: yaml.ScalarNode, Value: value, Style: l.Style}) + elem = NewRNode(&yaml.Node{ + Kind: yaml.MappingNode, + Content: []*yaml.Node{{Kind: yaml.ScalarNode, Value: name}, match.YNode()}, + Style: l.Style, + }) + } + // Append the Node + return rn.Pipe(ElementMatcher{FieldName: name, FieldValue: value, Create: elem}) +} + +func (l PathGetter) doField( + rn *RNode, name string, kind yaml.Kind) (*RNode, error) { + if !IsCreate(l.Create) { + return rn.Pipe(Get(name)) + } + return rn.Pipe(FieldMatcher{Name: name, Create: &RNode{value: &yaml.Node{Kind: kind, Style: l.Style}}}) +} + +func (l PathGetter) getKind(nextPart string) yaml.Kind { + if IsListIndex(nextPart) { + // if nextPart is of the form [a=b], then it is an index into a Sequence + // so the current part must be a SequenceNode + return yaml.SequenceNode + } + if nextPart == "" { + // final name in the path, use the l.Create defined Kind + return l.Create + } + + // non-sequence intermediate Node + return yaml.MappingNode +} + +func SetField(name string, value *RNode) FieldSetter { + return FieldSetter{Name: name, Value: value} +} + +func Set(value *RNode) FieldSetter { + return FieldSetter{Value: value} +} + +// FieldSetter sets a field or map entry to a value. +type FieldSetter struct { + Kind string `yaml:"kind,omitempty"` + + // Name is the name of the field or key to lookup in a MappingNode. + // If Name is unspecified, and the input is a ScalarNode, FieldSetter will set the + // value on the ScalarNode. + Name string `yaml:"name,omitempty"` + + // YNode is the value to set. + // Optional if Kind is set. + Value *RNode `yaml:"value,omitempty"` + + StringValue string `yaml:"stringValue,omitempty"` +} + +// FieldSetter returns an GrepFilter that sets the named field to the given value. +func (s FieldSetter) Filter(rn *RNode) (*RNode, error) { + if s.StringValue != "" && s.Value == nil { + s.Value = NewScalarRNode(s.StringValue) + } + + if s.Name == "" { + if err := ErrorIfInvalid(rn, yaml.ScalarNode); err != nil { + return rn, err + } + rn.SetYNode(s.Value.YNode()) + return rn, nil + } + + // Clear the field if it is empty, or explicitly null + if s.Value == nil || IsNull(s.Value) { + return rn.Pipe(Clear(s.Name)) + } + + field, err := rn.Pipe(FieldMatcher{Name: s.Name}) + if err != nil { + return nil, err + } + if field != nil { + // need to def ref the Node since field is ephemeral + field.SetYNode(s.Value.YNode()) + return field, nil + } + + // create the field + rn.YNode().Content = append(rn.YNode().Content, + &yaml.Node{Kind: yaml.ScalarNode, Value: s.Name}, + s.Value.YNode()) + return s.Value, nil +} + +// Tee calls the provided Filters, and returns its argument rather than the result +// of the filters. +// May be used to fork sub-filters from a call. +// e.g. locate field, set value; locate another field, set another value +func Tee(filters ...Filter) Filter { + return TeePiper{Filters: filters} +} + +// TeePiper Calls a slice of Filters and returns its input. +// May be used to fork sub-filters from a call. +// e.g. locate field, set value; locate another field, set another value +type TeePiper struct { + Kind string `yaml:"kind,omitempty"` + + // Filters are the set of Filters run by TeePiper. + Filters []Filter `yaml:"filters,omitempty"` +} + +func (t TeePiper) Filter(rn *RNode) (*RNode, error) { + _, err := rn.Pipe(t.Filters...) + return rn, err +} + +// IsCreate returns true if kind is specified +func IsCreate(kind yaml.Kind) bool { + return kind != 0 +} + +// IsMissingOrError returns true if rn is NOT found or err is non-nil +func IsMissingOrError(rn *RNode, err error) bool { + return rn == nil || err != nil +} + +// IsFoundOrError returns true if rn is found or err is non-nil +func IsFoundOrError(rn *RNode, err error) bool { + return rn != nil || err != nil +} + +func ErrorIfAnyInvalidAndNonNull(kind yaml.Kind, rn ...*RNode) error { + for i := range rn { + if IsEmpty(rn[i]) { + continue + } + if err := ErrorIfInvalid(rn[i], kind); err != nil { + return err + } + } + return nil +} + +func ErrorIfInvalid(rn *RNode, kind yaml.Kind) error { + if rn == nil || rn.YNode() == nil || IsNull(rn) { + // node has no type, pass validation + return nil + } + + if rn.YNode().Kind != kind { + s, _ := rn.String() + return errors.Errorf( + "wrong Node Kind for %s expected: %v was %v: value: {%s}", + strings.Join(rn.FieldPath(), "."), + kind, rn.YNode().Kind, strings.TrimSpace(s)) + } + + if kind == yaml.MappingNode { + if len(rn.YNode().Content)%2 != 0 { + return fmt.Errorf("yaml MappingNodes must have even length contents: %v", spew.Sdump(rn)) + } + } + + return nil +} + +// IsListIndex returns true if p is an index into a Val. +// e.g. [fieldName=fieldValue] +// e.g. [=primitiveValue] +func IsListIndex(p string) bool { + return strings.HasPrefix(p, "[") && strings.HasSuffix(p, "]") +} + +// SplitIndexNameValue splits a lookup part Val index into the field name +// and field value to match. +// e.g. splits [name=nginx] into (name, nginx) +// e.g. splits [=-jar] into ("", jar) +func SplitIndexNameValue(p string) (string, string, error) { + elem := strings.TrimSuffix(p, "]") + elem = strings.TrimPrefix(elem, "[") + parts := strings.SplitN(elem, "=", 2) + if len(parts) == 1 { + return "", "", fmt.Errorf("list path element must contain fieldName=fieldValue for element to match") + } + return parts[0], parts[1], nil +} + +// IncrementFieldIndex increments i to point to the next field name element in +// a slice of Contents. +func IncrementFieldIndex(i *int) { + *i = *i + 2 +} diff --git a/kyaml/yaml/fns_test.go b/kyaml/yaml/fns_test.go new file mode 100644 index 000000000..620692dea --- /dev/null +++ b/kyaml/yaml/fns_test.go @@ -0,0 +1,704 @@ +// Copyright 2019 The Kubernetes Authors. +// SPDX-License-Identifier: Apache-2.0 + +package yaml_test + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + "gopkg.in/yaml.v3" + . "sigs.k8s.io/kustomize/kyaml/yaml" +) + +func TestResourceNode_SetValue(t *testing.T) { + instance := *NewScalarRNode("foo") + copy := instance + instance.SetYNode(&yaml.Node{Kind: yaml.ScalarNode, Value: "bar"}) + assert.Equal(t, `bar +`, assertNoErrorString(t)(copy.String())) + assert.Equal(t, `bar +`, assertNoErrorString(t)(instance.String())) + + instance = *NewScalarRNode("foo") + copy = instance + instance.SetYNode(nil) + instance.SetYNode(&yaml.Node{Kind: yaml.ScalarNode, Value: "bar"}) + assert.Equal(t, `foo +`, assertNoErrorString(t)(copy.String())) + assert.Equal(t, `bar +`, assertNoErrorString(t)(instance.String())) +} + +func TestAppend(t *testing.T) { + s := `n: o +a: b +c: d +` + node, err := Parse(s) + assert.NoError(t, err) + rn, err := node.Pipe(Append(NewScalarRNode("").YNode())) + if assert.Error(t, err) { + assert.Contains(t, err.Error(), "wrong Node Kind") + } + assert.Nil(t, rn) + + s = `- a +- b +` + node, err = Parse(s) + assert.NoError(t, err) + rn, err = node.Pipe(Append()) + assert.NoError(t, err) + assert.Nil(t, rn) +} + +func TestClearField_Fn(t *testing.T) { + s := `n: o +a: b +c: d +` + node, err := Parse(s) + assert.NoError(t, err) + rn, err := node.Pipe(FieldClearer{Name: "a"}) + assert.NoError(t, err) + assert.Equal(t, "n: o\nc: d\n", assertNoErrorString(t)(node.String())) + assert.Equal(t, "b\n", assertNoErrorString(t)(rn.String())) + + s = `n: o +a: b +c: d +` + node, err = Parse(s) + assert.NoError(t, err) + rn, err = node.Pipe(FieldClearer{Name: "n"}) + assert.NoError(t, err) + assert.Equal(t, "a: b\nc: d\n", assertNoErrorString(t)(node.String())) + assert.Equal(t, "o\n", assertNoErrorString(t)(rn.String())) + + node, err = Parse(s) + assert.NoError(t, err) + rn, err = node.Pipe(FieldClearer{Name: "c"}) + assert.NoError(t, err) + assert.Equal(t, "n: o\na: b\n", assertNoErrorString(t)(node.String())) + assert.Equal(t, "d\n", assertNoErrorString(t)(rn.String())) + + s = `n: o +a: b +` + node, err = Parse(s) + assert.NoError(t, err) + rn, err = node.Pipe(FieldClearer{Name: "o"}) + assert.NoError(t, err) + assert.Nil(t, rn) + assert.Equal(t, "n: o\na: b\n", assertNoErrorString(t)(node.String())) + + s = `- a +- b +` + node, err = Parse(s) + assert.NoError(t, err) + rn, err = node.Pipe(FieldClearer{Name: "a"}) + if assert.Error(t, err) { + assert.Contains(t, err.Error(), "wrong Node Kind") + } + assert.Nil(t, rn) + assert.Equal(t, s, assertNoErrorString(t)(node.String())) + + // should not clear n because it is not empty + s = `n: + k: v +a: b +c: d +` + node, err = Parse(s) + assert.NoError(t, err) + rn, err = node.Pipe(FieldClearer{Name: "n", IfEmpty: true}) + assert.NoError(t, err) + assert.Equal(t, "n:\n k: v\na: b\nc: d\n", assertNoErrorString(t)(node.String())) + assert.Equal(t, "", assertNoErrorString(t)(rn.String())) + + // should clear n because it is empty + s = `n: {} +a: b +c: d +` + node, err = Parse(s) + assert.NoError(t, err) + rn, err = node.Pipe(FieldClearer{Name: "n", IfEmpty: true}) + assert.NoError(t, err) + assert.Equal(t, "a: b\nc: d\n", assertNoErrorString(t)(node.String())) + assert.Equal(t, "{}\n", assertNoErrorString(t)(rn.String())) +} + +var s = `n: o +a: + l: m + b: + - f: g + - c: e + - h: i +r: s +` + +func TestLookup_Fn_create(t *testing.T) { + + // primitive + node, err := Parse(s) + assert.NoError(t, err) + rn, err := node.Pipe(PathGetter{ + Path: []string{"a", "b", "[c=d]", "t", "f", "[=h]"}, + Create: yaml.ScalarNode, + }) + assert.NoError(t, err) + assert.Equal(t, `n: o +a: + l: m + b: + - f: g + - c: e + - h: i + - c: d + t: + f: + - h +r: s +`, assertNoErrorString(t)(node.String())) + assert.Equal(t, `h +`, assertNoErrorString(t)(rn.String())) + +} + +func TestLookup_Fn_create2(t *testing.T) { + node, err := Parse(s) + assert.NoError(t, err) + rn, err := node.Pipe(PathGetter{ + Path: []string{"a", "b", "[c=d]", "t", "f"}, + Create: yaml.SequenceNode, + }) + assert.NoError(t, err) + assert.Equal(t, `n: o +a: + l: m + b: + - f: g + - c: e + - h: i + - c: d + t: + f: [] +r: s +`, assertNoErrorString(t)(node.String())) + assert.Equal(t, `[] +`, assertNoErrorString(t)(rn.String())) +} + +func TestLookup_Fn_create3(t *testing.T) { + node, err := Parse(s) + assert.NoError(t, err) + rn, err := node.Pipe(LookupCreate(yaml.MappingNode, "a", "b", "[c=d]", "t")) + assert.NoError(t, err) + assert.Equal(t, `n: o +a: + l: m + b: + - f: g + - c: e + - h: i + - c: d + t: {} +r: s +`, assertNoErrorString(t)(node.String())) + assert.Equal(t, `{} +`, assertNoErrorString(t)(rn.String())) +} + +func TestLookupCreate_4(t *testing.T) { + node, err := Parse(` +a: {} +`) + assert.NoError(t, err) + rn, err := node.Pipe( + LookupCreate(yaml.MappingNode, "a", "b", "[c=d]", "t", "f", "[=h]")) + + node.YNode().Style = yaml.FlowStyle + assert.NoError(t, err) + assert.Equal(t, "{a: {b: [{c: d, t: {f: [h]}}]}}\n", assertNoErrorString(t)(node.String())) + assert.Equal(t, "h\n", assertNoErrorString(t)(rn.String())) +} + +func TestLookup(t *testing.T) { + s := `n: o +a: + l: m + b: + - f: g + - c: e + - c: d + t: + u: v + f: + - g + - h + - i + j: k + - h: i + p: q +r: s +` + node, err := Parse(s) + assert.NoError(t, err) + + // primitive + rn, err := node.Pipe(Lookup("a", "b", "[c=d]", "t", "f", "[=h]")) + assert.NoError(t, err) + assert.Equal(t, s, assertNoErrorString(t)(node.String())) + assert.Equal(t, `h +`, assertNoErrorString(t)(rn.String())) + + // seq + rn, err = node.Pipe(Lookup("a", "b", "[c=d]", "t", "f")) + assert.NoError(t, err) + assert.Equal(t, s, assertNoErrorString(t)(node.String())) + assert.Equal(t, `- g +- h +- i +`, assertNoErrorString(t)(rn.String())) + + rn, err = node.Pipe(Lookup("a", "b", "[c=d]", "t")) + assert.NoError(t, err) + assert.Equal(t, s, assertNoErrorString(t)(node.String())) + assert.Equal(t, `u: v +f: +- g +- h +- i +`, assertNoErrorString(t)(rn.String())) + + rn, err = node.Pipe(Lookup("a", "b", "[c=d]")) + assert.NoError(t, err) + assert.Equal(t, s, assertNoErrorString(t)(node.String())) + assert.Equal(t, `c: d +t: + u: v + f: + - g + - h + - i +j: k +`, assertNoErrorString(t)(rn.String())) + + rn, err = node.Pipe(Lookup("a", "b")) + assert.NoError(t, err) + assert.Equal(t, s, assertNoErrorString(t)(node.String())) + assert.Equal(t, `- f: g +- c: e +- c: d + t: + u: v + f: + - g + - h + - i + j: k +- h: i + p: q +`, assertNoErrorString(t)(rn.String())) + + rn, err = node.Pipe(Lookup("l")) + assert.NoError(t, err) + assert.Equal(t, s, assertNoErrorString(t)(node.String())) + assert.Nil(t, rn) + + rn, err = node.Pipe(Lookup("zzz")) + assert.NoError(t, err) + assert.Equal(t, s, assertNoErrorString(t)(node.String())) + assert.Nil(t, rn) + + rn, err = node.Pipe(Lookup("[a=b]")) + assert.Error(t, err) + assert.Equal(t, s, assertNoErrorString(t)(node.String())) + assert.Nil(t, rn) + + rn, err = node.Pipe(Lookup("a", "b", "f")) + assert.Error(t, err) + assert.Equal(t, s, assertNoErrorString(t)(node.String())) + assert.Nil(t, rn) + + rn, err = node.Pipe(Lookup("a", "b", "c=zzz")) + assert.Error(t, err) + assert.Equal(t, s, assertNoErrorString(t)(node.String())) + assert.Nil(t, rn) + + rn, err = node.Pipe(Lookup(" ", "a", "", "b", " ", "[c=d]", "\n", "t", "\t", "f", " ", "[=h]", " ")) + assert.NoError(t, err) + assert.Equal(t, s, assertNoErrorString(t)(node.String())) + assert.Equal(t, `h +`, assertNoErrorString(t)(rn.String())) + + rn, err = node.Pipe(Lookup(" ", "a", "", "b", " ", "[]")) + assert.Error(t, err) + assert.Nil(t, rn) + + rn, err = node.Pipe(Lookup("a", "b", "[c=d]", "t", "f", "[=c]")) + assert.NoError(t, err) + assert.Equal(t, s, assertNoErrorString(t)(node.String())) + assert.Nil(t, rn) + + rn, err = node.Pipe(Lookup("a", "b", "[z=z]")) + assert.NoError(t, err) + assert.Equal(t, s, assertNoErrorString(t)(node.String())) + assert.Nil(t, rn) +} + +func TestSetField_Fn(t *testing.T) { + // Change field + node, err := Parse(` +foo: baz +`) + assert.NoError(t, err) + instance := FieldSetter{ + Name: "foo", + Value: NewScalarRNode("bar"), + } + k, err := instance.Filter(node) + assert.NoError(t, err) + assert.Equal(t, `foo: bar +`, assertNoErrorString(t)(node.String())) + assert.Equal(t, `bar +`, assertNoErrorString(t)(k.String())) + + // Add field + node, err = Parse(` +foo: baz +`) + assert.NoError(t, err) + instance = FieldSetter{ + Name: "bar", + Value: NewScalarRNode("buz"), + } + k, err = instance.Filter(node) + assert.NoError(t, err) + assert.Equal(t, `foo: baz +bar: buz +`, assertNoErrorString(t)(node.String())) + assert.Equal(t, `buz +`, assertNoErrorString(t)(k.String())) + + // Clear field + node, err = Parse(` +foo: baz +bar: buz +`) + assert.NoError(t, err) + instance = FieldSetter{ + Name: "foo", + } + k, err = instance.Filter(node) + assert.NoError(t, err) + assert.Equal(t, `bar: buz +`, assertNoErrorString(t)(node.String())) + assert.Equal(t, `baz +`, assertNoErrorString(t)(k.String())) + + // Encounter error + node, err = Parse(` +-a +-b +`) + assert.NoError(t, err) + instance = FieldSetter{ + Name: "foo", + Value: NewScalarRNode("v"), + } + k, err = instance.Filter(node) + if assert.Error(t, err) { + assert.Contains(t, err.Error(), "wrong Node Kind") + } + assert.Nil(t, k) +} + +func TestSet_Fn(t *testing.T) { + node, err := Parse(` +foo: baz +`) + assert.NoError(t, err) + k, err := node.Pipe(Get("foo"), Set(NewScalarRNode("bar"))) + assert.NoError(t, err) + assert.Equal(t, `foo: bar +`, assertNoErrorString(t)(node.String())) + assert.Equal(t, `bar +`, assertNoErrorString(t)(k.String())) + + node, err = Parse(` +foo: baz +`) + assert.NoError(t, err) + _, err = node.Pipe(Set(NewScalarRNode("bar"))) + if !assert.Error(t, err) { + return + } + assert.Contains(t, err.Error(), "wrong Node Kind") + assert.Equal(t, `foo: baz +`, assertNoErrorString(t)(node.String())) + +} + +func TestErrorIfInvalid(t *testing.T) { + err := ErrorIfInvalid( + NewRNode(&yaml.Node{Kind: yaml.SequenceNode}), yaml.SequenceNode) + assert.NoError(t, err) + + // nil values should pass validation -- they were not specified + err = ErrorIfInvalid(&RNode{}, yaml.SequenceNode) + if !assert.NoError(t, err) { + t.FailNow() + } + + err = ErrorIfInvalid(NewRNode(&Node{Content: []*yaml.Node{{Value: "hello"}}}), yaml.SequenceNode) + if !assert.Error(t, err) { + t.FailNow() + } + assert.Contains(t, err.Error(), "wrong Node Kind") + + err = ErrorIfInvalid(NewRNode(&yaml.Node{}), yaml.SequenceNode) + if assert.Error(t, err) { + assert.Contains(t, err.Error(), "wrong Node Kind") + } + err = ErrorIfInvalid(NewRNode(&yaml.Node{}), yaml.MappingNode) + if assert.Error(t, err) { + assert.Contains(t, err.Error(), "wrong Node Kind") + } + + err = ErrorIfInvalid(NewRNode(&yaml.Node{ + Kind: yaml.MappingNode, + Content: []*yaml.Node{{}, {}}, + }), yaml.MappingNode) + assert.NoError(t, err) + + err = ErrorIfInvalid(NewRNode(&yaml.Node{}), yaml.SequenceNode) + if assert.Error(t, err) { + assert.Contains(t, err.Error(), "wrong Node Kind") + } + + err = ErrorIfInvalid(NewRNode(&yaml.Node{ + Kind: yaml.MappingNode, + Content: []*yaml.Node{{}}, + }), yaml.MappingNode) + if assert.Error(t, err) { + assert.Contains(t, err.Error(), "even length") + } +} + +func TestSplitIndexNameValue(t *testing.T) { + k, v, err := SplitIndexNameValue("") + if assert.Error(t, err) { + assert.Contains(t, err.Error(), "fieldName=fieldValue") + } + assert.Equal(t, "", k) + assert.Equal(t, "", v) + + k, v, err = SplitIndexNameValue("a=b") + assert.NoError(t, err) + assert.Equal(t, "a", k) + assert.Equal(t, "b", v) + + k, v, err = SplitIndexNameValue("=b") + assert.NoError(t, err) + assert.Equal(t, "", k) + assert.Equal(t, "b", v) + + k, v, err = SplitIndexNameValue("a=b=c") + assert.NoError(t, err) + assert.Equal(t, "a", k) + assert.Equal(t, "b=c", v) +} + +type filter struct { + fn func(object *RNode) (*RNode, error) +} + +func (c filter) Filter(object *RNode) (*RNode, error) { + return c.fn(object) +} + +func TestResourceNode_Pipe(t *testing.T) { + r0, r1, r2, r3 := &RNode{}, &RNode{}, &RNode{}, &RNode{} + var called []string + + // check the nil value case + r0 = nil + _, err := r0.Pipe(FieldMatcher{Name: "foo"}) + assert.NoError(t, err) + + r0 = &RNode{} + // all filters successful + v, err := r0.Pipe( + filter{fn: func(object *RNode) (*RNode, error) { + assert.True(t, r0 == object) + called = append(called, "a") + return r1, nil + }}, + filter{fn: func(object *RNode) (*RNode, error) { + assert.True(t, object == r1, "function arg doesn't match last function output") + called = append(called, "b") + return r2, nil + }}, + filter{fn: func(object *RNode) (*RNode, error) { + assert.True(t, object == r2, "function arg doesn't match last function output") + return r3, nil + }}, + ) + assert.True(t, v == r3, "expected r3") + assert.Nil(t, err) + assert.Equal(t, called, []string{"a", "b"}) + + // filter returns nil + called = []string{} + v, err = r0.Pipe( + filter{fn: func(object *RNode) (*RNode, error) { + assert.True(t, r0 == object) + called = append(called, "a") + return r1, nil + }}, + filter{fn: func(object *RNode) (*RNode, error) { + assert.True(t, object == r1, "function arg doesn't match last function output") + called = append(called, "b") + return nil, nil + }}, + filter{fn: func(object *RNode) (*RNode, error) { + assert.Fail(t, "function should be run after error") + return nil, nil + }}, + ) + assert.Nil(t, v) + assert.Nil(t, err) + assert.Equal(t, called, []string{"a", "b"}) + + // filter returns an error + called = []string{} + v, err = r0.Pipe( + filter{fn: func(object *RNode) (*RNode, error) { + assert.True(t, r0 == object) + called = append(called, "a") + return r1, nil + }}, + filter{fn: func(object *RNode) (*RNode, error) { + assert.True(t, object == r1, "function arg doesn't match last function output") + called = append(called, "b") + return r1, fmt.Errorf("expected-error") + }}, + filter{fn: func(object *RNode) (*RNode, error) { + assert.Fail(t, "function should be run after error") + return nil, nil + }}, + ) + assert.True(t, v == r1, "expected r1 as value") + assert.EqualError(t, err, "expected-error") + assert.Equal(t, called, []string{"a", "b"}) +} + +func TestClearAnnotation(t *testing.T) { + // create metadata.annotations field + r0 := assertNoError(t)(Parse(`apiVersion: apps/v1 +kind: Deployment +metadata: + annotations: + z: y + a.b.c: d.e.f + s: t +`)) + + rn := assertNoError(t)(r0.Pipe(ClearAnnotation("a.b.c"))) + assert.Equal(t, "d.e.f\n", assertNoErrorString(t)(rn.String())) + assert.Equal(t, `apiVersion: apps/v1 +kind: Deployment +metadata: + annotations: + z: y + s: t +`, assertNoErrorString(t)(r0.String())) +} + +func TestGetAnnotation(t *testing.T) { + r0 := assertNoError(t)(Parse(`apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: java + annotations: + a.b.c: d.e.f + g: h + i: j + k: l + name: app`)) + + rn := assertNoError(t)( + r0.Pipe(GetAnnotation("a.b.c"))) + assert.Equal(t, "d.e.f\n", assertNoErrorString(t)(rn.String())) +} + +func TestSetAnnotation_Fn(t *testing.T) { + // create metadata.annotations field + r0 := assertNoError(t)(Parse(`apiVersion: apps/v1 +kind: Deployment`)) + + rn := assertNoError(t)(r0.Pipe(SetAnnotation("a.b.c", "d.e.f"))) + assert.Equal(t, "d.e.f\n", assertNoErrorString(t)(rn.String())) + assert.Equal(t, `apiVersion: apps/v1 +kind: Deployment +metadata: + annotations: + a.b.c: d.e.f +`, assertNoErrorString(t)(r0.String())) +} + +func TestRNode_GetMeta(t *testing.T) { + s := `apiVersion: v1/apps +kind: Deployment +metadata: + name: foo + namespace: bar + labels: + kl: vl + annotations: + ka: va +` + node, err := Parse(s) + if !assert.NoError(t, err) { + return + } + meta, err := node.GetMeta() + if !assert.NoError(t, err) { + return + } + assert.Equal(t, ResourceMeta{ + Kind: "Deployment", + ApiVersion: "v1/apps", + ObjectMeta: ObjectMeta{ + Name: "foo", + Namespace: "bar", + Annotations: map[string]string{ + "ka": "va", + }, + Labels: map[string]string{ + "kl": "vl", + }, + }, + }, meta) +} + +func assertNoError(t *testing.T) func(o *RNode, err error) *RNode { + return func(o *RNode, err error) *RNode { + assert.NoError(t, err) + return o + } +} + +func assertNoErrorString(t *testing.T) func(string, error) string { + return func(s string, err error) string { + assert.NoError(t, err) + return s + } +} diff --git a/kyaml/yaml/kfns.go b/kyaml/yaml/kfns.go new file mode 100644 index 000000000..60d28b09a --- /dev/null +++ b/kyaml/yaml/kfns.go @@ -0,0 +1,66 @@ +// Copyright 2019 The Kubernetes Authors. +// SPDX-License-Identifier: Apache-2.0 + +package yaml + +import "gopkg.in/yaml.v3" + +// AnnotationClearer removes an annotation at metadata.annotations. +// Returns nil if the annotation or field does not exist. +type AnnotationClearer struct { + Kind string `yaml:"kind,omitempty"` + Key string `yaml:"key,omitempty"` +} + +func (c AnnotationClearer) Filter(rn *RNode) (*RNode, error) { + return rn.Pipe( + PathGetter{Path: []string{"metadata", "annotations"}}, + FieldClearer{Name: c.Key}) +} + +func ClearAnnotation(key string) AnnotationClearer { + return AnnotationClearer{Key: key} +} + +// AnnotationSetter sets an annotation at metadata.annotations. +// Creates metadata.annotations if does not exist. +type AnnotationSetter struct { + Kind string `yaml:"kind,omitempty"` + Key string `yaml:"key,omitempty"` + Value string `yaml:"value,omitempty"` +} + +func (s AnnotationSetter) Filter(rn *RNode) (*RNode, error) { + return rn.Pipe( + PathGetter{Path: []string{"metadata", "annotations"}, Create: yaml.MappingNode}, + FieldSetter{Name: s.Key, Value: NewScalarRNode(s.Value)}) +} + +func SetAnnotation(key, value string) AnnotationSetter { + return AnnotationSetter{Key: key, Value: value} +} + +// AnnotationGetter gets an annotation at metadata.annotations. +// Returns nil if metadata.annotations does not exist. +type AnnotationGetter struct { + Kind string `yaml:"kind,omitempty"` + Key string `yaml:"key,omitempty"` + Value string `yaml:"value,omitempty"` +} + +// AnnotationGetter returns the annotation value. +// Returns "", nil if the annotation does not exist. +func (g AnnotationGetter) Filter(rn *RNode) (*RNode, error) { + v, err := rn.Pipe(PathGetter{Path: []string{"metadata", "annotations", g.Key}}) + if v == nil || err != nil { + return v, err + } + if g.Value == "" || v.value.Value == g.Value { + return v, err + } + return nil, err +} + +func GetAnnotation(key string) AnnotationGetter { + return AnnotationGetter{Key: key} +} diff --git a/kyaml/yaml/match.go b/kyaml/yaml/match.go new file mode 100644 index 000000000..3ba66c010 --- /dev/null +++ b/kyaml/yaml/match.go @@ -0,0 +1,208 @@ +// Copyright 2019 The Kubernetes Authors. +// SPDX-License-Identifier: Apache-2.0 + +package yaml + +import ( + "regexp" + "strings" +) + +// PathMatcher returns all RNodes matching the path wrapped in a SequenceNode. +// Lists may have multiple elements matching the path, and each matching element +// is added to the return result. +// If Path points to a SequenceNode, the SequenceNode is wrapped in another SequenceNode +// If Path does not contain any lists, the result is still wrapped in a SequenceNode of len == 1 +type PathMatcher struct { + Kind string `yaml:"kind,omitempty"` + + // Path is a slice of parts leading to the RNode to lookup. + // Each path part may be one of: + // * FieldMatcher -- e.g. "spec" + // * Map Key -- e.g. "app.k8s.io/version" + // * List Entry -- e.g. "[name=nginx]" or "[=-jar]" + // + // Map Keys and Fields are equivalent. + // See FieldMatcher for more on Fields and Map Keys. + // + // List Entries are specified as map entry to match [fieldName=fieldValue]. + // See Elem for more on List Entries. + // + // Examples: + // * spec.template.spec.container with matching name: [name=nginx] -- match 'name': 'nginx' + // * spec.template.spec.container.argument matching a value: [=-jar] -- match '-jar' + Path []string `yaml:"path,omitempty"` + + // Matches is set by PathMatch to publish the matched element values for each node. + // After running PathMatcher.Filter, each node from the SequenceNode result may be + // looked up in Matches to find the field values that were matched. + Matches map[*Node][]string + + // StripComments may be set to remove the comments on the matching Nodes. + // This is useful for if the nodes are to be printed in FlowStyle. + StripComments bool + + val *RNode + field string + matchRegex string +} + +func (p *PathMatcher) stripComments(n *Node) { + if n == nil { + return + } + if p.StripComments { + n.LineComment = "" + n.HeadComment = "" + n.FootComment = "" + for i := range n.Content { + p.stripComments(n.Content[i]) + } + } +} + +func (p *PathMatcher) Filter(rn *RNode) (*RNode, error) { + val, err := p.filter(rn) + if err != nil { + return nil, err + } + p.stripComments(val.YNode()) + return val, err +} + +func (p *PathMatcher) filter(rn *RNode) (*RNode, error) { + p.Matches = map[*Node][]string{} + + if len(p.Path) == 0 { + // return the element wrapped in a SequenceNode + p.appendRNode("", rn) + return p.val, nil + } + + if IsListIndex(p.Path[0]) { + // match seq elements + return p.doSeq(rn) + } else { + // match a field + return p.doField(rn) + } +} + +func (p *PathMatcher) doField(rn *RNode) (*RNode, error) { + // lookup the field + field, err := rn.Pipe(Get(p.Path[0])) + if err != nil || field == nil { + // if the field doesn't exist, return nil + return nil, err + } + + // recurse on the field, removing the first element of the path + pm := &PathMatcher{Path: p.Path[1:]} + p.val, err = pm.filter(field) + p.Matches = pm.Matches + return p.val, err +} + +// doSeq iterates over a sequence and appends elements matching the path regex to p.Val +func (p *PathMatcher) doSeq(rn *RNode) (*RNode, error) { + + // parse the field + match pair + var err error + p.field, p.matchRegex, err = SplitIndexNameValue(p.Path[0]) + if err != nil { + return nil, err + } + + if p.field == "" { + err = rn.VisitElements(p.visitPrimitiveElem) + } else { + err = rn.VisitElements(p.visitElem) + } + if err != nil || p.val == nil || len(p.val.YNode().Content) == 0 { + return nil, err + } + + return p.val, nil +} + +func (p *PathMatcher) visitPrimitiveElem(elem *RNode) error { + r, err := regexp.Compile(p.matchRegex) + if err != nil { + return err + } + + str, err := elem.String() + if err != nil { + return err + } + str = strings.TrimSpace(str) + if !r.MatchString(str) { + return nil + } + + p.appendRNode("", elem) + return nil +} + +func (p *PathMatcher) visitElem(elem *RNode) error { + r, err := regexp.Compile(p.matchRegex) + if err != nil { + return err + } + + // check if this elements field matches the regex + val := elem.Field(p.field) + if val == nil || val.Value == nil { + return nil + } + str, err := val.Value.String() + if err != nil { + return err + } + str = strings.TrimSpace(str) + if !r.MatchString(str) { + return nil + } + + // recurse on the matching element + pm := &PathMatcher{Path: p.Path[1:]} + add, err := pm.filter(elem) + for k, v := range pm.Matches { + p.Matches[k] = v + } + if err != nil || add == nil { + return err + } + p.append(str, add.Content()...) + return nil +} + +func (p *PathMatcher) appendRNode(path string, node *RNode) { + p.append(path, node.YNode()) +} + +func (p *PathMatcher) append(path string, nodes ...*Node) { + if p.val == nil { + p.val = NewRNode(&Node{Kind: SequenceNode}) + } + for i := range nodes { + node := nodes[i] + p.val.YNode().Content = append(p.val.YNode().Content, node) + // record the path if specified + if path != "" { + p.Matches[node] = append(p.Matches[node], path) + } + } +} + +func cleanPath(path []string) []string { + var p []string + for _, elem := range path { + elem = strings.TrimSpace(elem) + if len(elem) == 0 { + continue + } + p = append(p, elem) + } + return p +} diff --git a/kyaml/yaml/match_test.go b/kyaml/yaml/match_test.go new file mode 100644 index 000000000..70852398b --- /dev/null +++ b/kyaml/yaml/match_test.go @@ -0,0 +1,88 @@ +// Copyright 2019 The Kubernetes Authors. +// SPDX-License-Identifier: Apache-2.0 + +package yaml + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestPathMatcher_Filter(t *testing.T) { + node := MustParse(`apiVersion: apps/v1 +kind: Deployment +metadata: + name: nginx-deployment + labels: + app: nginx +spec: + replicas: 3 + selector: + matchLabels: + app: nginx + template: + metadata: + labels: + app: nginx + spec: + containers: + - name: nginx + image: nginx:1.7.9 + ports: + - containerPort: 80 + - name: sidecar + image: sidecar:1.0.0 + ports: + - containerPort: 8081 + - containerPort: 9090 +`) + + updates := []struct { + path []string + value string + }{ + {[]string{ + "spec", "template", "spec", "containers", "[name=.*]"}, + "- name: nginx\n image: nginx:1.7.9\n ports:\n - containerPort: 80\n" + + "- name: sidecar\n image: sidecar:1.0.0\n ports:\n - containerPort: 8081\n - containerPort: 9090\n"}, + {[]string{ + "spec", "template", "spec", "containers", "[name=.*]", "image"}, + "- nginx:1.7.9\n- sidecar:1.0.0\n"}, + {[]string{ + "spec", "template", "spec", "containers", "[name=n.*]", "image"}, + "- nginx:1.7.9\n"}, + {[]string{ + "spec", "template", "spec", "containers", "[name=s.*]", "image"}, + "- sidecar:1.0.0\n"}, + {[]string{ + "spec", "template", "spec", "containers", "[name=.*x]", "image"}, + "- nginx:1.7.9\n"}, + {[]string{ + "spec", "template", "spec", "containers", "[name=.*]", "ports"}, + "- - containerPort: 80\n- - containerPort: 8081\n - containerPort: 9090\n"}, + {[]string{ + "spec", "template", "spec", "containers", "[name=.*]", "ports", "[containerPort=8.*]"}, + "- containerPort: 80\n- containerPort: 8081\n"}, + {[]string{ + "spec", "template", "spec", "containers", "[name=.*]", "ports", "[containerPort=.*1]"}, + "- containerPort: 8081\n"}, + {[]string{ + "spec", "template", "spec", "containers", "[name=.*]", "ports", "[containerPort=9.*]"}, + "- containerPort: 9090\n"}, + {[]string{ + "spec", "template", "spec", "containers", "[name=s.*]", "ports", "[containerPort=8.*]"}, + "- containerPort: 8081\n"}, + {[]string{ + "spec", "template", "spec", "containers", "[name=s.*]", "ports", "[containerPort=.*2]"}, + ""}, + } + for i, u := range updates { + result, err := node.Pipe(&PathMatcher{Path: u.path}) + if !assert.NoError(t, err) { + return + } + assert.Equal(t, u.value, result.MustString(), fmt.Sprintf("%d", i)) + } +} diff --git a/kyaml/yaml/merge2/element_test.go b/kyaml/yaml/merge2/element_test.go new file mode 100644 index 000000000..bdb1100d2 --- /dev/null +++ b/kyaml/yaml/merge2/element_test.go @@ -0,0 +1,265 @@ +// Copyright 2019 The Kubernetes Authors. +// SPDX-License-Identifier: Apache-2.0 + +package merge2_test + +var elementTestCases = []testCase{ + {`merge Element -- keep field in dest`, + ` +kind: Deployment +items: +- name: foo + image: foo:v1 +`, + ` +kind: Deployment +items: +- name: foo + image: foo:v0 + command: ['run.sh'] +`, + ` +kind: Deployment +items: +- name: foo + image: foo:v1 + command: + - run.sh +`, + }, + + {`merge Element -- add field to dest`, + ` +kind: Deployment +items: +- name: foo + image: foo:v1 + command: ['run.sh'] +`, + ` +kind: Deployment +items: +- name: foo + image: foo:v0 +`, + ` +kind: Deployment +items: +- name: foo + image: foo:v1 + command: + - run.sh +`, + }, + + {`merge Element -- add list, empty in dest`, + ` +kind: Deployment +items: +- name: foo + image: foo:v1 + command: ['run.sh'] +`, + ` +kind: Deployment +items: {} +`, + ` +kind: Deployment +items: +- name: foo + image: foo:v1 + command: + - run.sh +`, + }, + + {`merge Element -- add list, missing from dest`, + ` +kind: Deployment +items: +- name: foo + image: foo:v1 + command: ['run.sh'] +`, + ` +kind: Deployment +`, + ` +kind: Deployment +items: +- name: foo + image: foo:v1 + command: + - run.sh +`, + }, + + {`merge Element -- add Element first`, + ` +kind: Deployment +items: +- name: bar + image: bar:v1 + command: ['run2.sh'] +- name: foo + image: foo:v1 +`, + ` +kind: Deployment +items: +- name: foo + image: foo:v0 +`, + ` +kind: Deployment +items: +- name: foo + image: foo:v1 +- name: bar + image: bar:v1 + command: + - run2.sh +`, + }, + + {`merge Element -- add Element second`, + ` +kind: Deployment +items: +- name: foo + image: foo:v1 +- name: bar + image: bar:v1 + command: ['run2.sh'] +`, + ` +kind: Deployment +items: +- name: foo + image: foo:v0 +`, + ` +kind: Deployment +items: +- name: foo + image: foo:v1 +- name: bar + image: bar:v1 + command: + - run2.sh +`, + }, + + // + // Test Case + // + {`keep list -- list missing from src`, + ` +kind: Deployment +`, + ` +kind: Deployment +items: +- name: foo + image: foo:v1 +- name: bar + image: bar:v1 + command: ['run2.sh'] +`, + ` +kind: Deployment +items: +- name: foo + image: foo:v1 +- name: bar + image: bar:v1 + command: + - run2.sh +`, + }, + + // + // Test Case + // + {`keep Element -- element missing in src`, + ` +kind: Deployment +items: +- name: foo + image: foo:v1 +`, + ` +kind: Deployment +items: +- name: foo + image: foo:v0 +- name: bar + image: bar:v1 + command: ['run2.sh'] +`, + ` +kind: Deployment +items: +- name: foo + image: foo:v1 +- name: bar + image: bar:v1 + command: + - run2.sh +`, + }, + + // + // Test Case + // + {`keep element -- empty list in src`, + ` +kind: Deployment +items: {} +`, + ` +kind: Deployment +items: +- name: foo + image: foo:v1 +- name: bar + image: bar:v1 + command: + - run2.sh +`, + ` +kind: Deployment +items: +- name: foo + image: foo:v1 +- name: bar + image: bar:v1 + command: + - run2.sh +`, + }, + + // + // Test Case + // + {`remove Element -- null in src`, + ` +kind: Deployment +items: null +`, + ` +kind: Deployment +items: +- name: foo + image: foo:v1 +- name: bar + image: bar:v1 + command: + - run2.sh +`, + ` +kind: Deployment +`, + }, +} diff --git a/kyaml/yaml/merge2/list_test.go b/kyaml/yaml/merge2/list_test.go new file mode 100644 index 000000000..0aa6a3dd5 --- /dev/null +++ b/kyaml/yaml/merge2/list_test.go @@ -0,0 +1,140 @@ +// Copyright 2019 The Kubernetes Authors. +// SPDX-License-Identifier: Apache-2.0 + +package merge2_test + +var listTestCases = []testCase{ + {`replace List -- different value in dest`, + ` +kind: Deployment +items: +- 1 +- 2 +- 3 +`, + ` +kind: Deployment +items: +- 0 +- 1 +`, + ` +kind: Deployment +items: +- 1 +- 2 +- 3 +`, + }, + + {`replace List -- missing from dest`, + ` +kind: Deployment +items: +- 1 +- 2 +- 3 +`, + ` +kind: Deployment +`, + ` +kind: Deployment +items: +- 1 +- 2 +- 3 +`, + }, + + // + // Test Case + // + {`keep List -- same value in src and dest`, + ` +kind: Deployment +items: +- 1 +- 2 +- 3 +`, + ` +kind: Deployment +items: +- 1 +- 2 +- 3 +`, + ` +kind: Deployment +items: +- 1 +- 2 +- 3 +`, + }, + + // + // Test Case + // + {`keep List -- unspecified in src`, + ` +kind: Deployment +`, + ` +kind: Deployment +items: +- 1 +- 2 +- 3 +`, + ` +kind: Deployment +items: +- 1 +- 2 +- 3 +`, + }, + + // + // Test Case + // + {`remove List -- null in src`, + ` +kind: Deployment +items: null +`, + ` +kind: Deployment +items: +- 1 +- 2 +- 3 +`, + ` +kind: Deployment +`, + }, + + // + // Test Case + // + {`remove list -- empty in src`, + ` +kind: Deployment +items: {} +`, + ` +kind: Deployment +items: +- 1 +- 2 +- 3 +`, + ` +kind: Deployment +items: {} +`, + }, +} diff --git a/kyaml/yaml/merge2/map_test.go b/kyaml/yaml/merge2/map_test.go new file mode 100644 index 000000000..cfdbee357 --- /dev/null +++ b/kyaml/yaml/merge2/map_test.go @@ -0,0 +1,186 @@ +// Copyright 2019 The Kubernetes Authors. +// SPDX-License-Identifier: Apache-2.0 + +package merge2_test + +var mapTestCases = []testCase{ + {`merge Map -- update field in dest`, + ` +kind: Deployment +spec: + foo: bar1 +`, + ` +kind: Deployment +spec: + foo: bar0 + baz: buz +`, + ` +kind: Deployment +spec: + foo: bar1 + baz: buz +`, + }, + + {`merge Map -- add field to dest`, + ` +kind: Deployment +spec: + foo: bar1 + baz: buz +`, + ` +kind: Deployment +spec: + foo: bar0 +`, + ` +kind: Deployment +spec: + foo: bar1 + baz: buz +`, + }, + + {`merge Map -- add list, empty in dest`, + ` +kind: Deployment +spec: + foo: bar1 + baz: buz +`, + ` +kind: Deployment +spec: {} +`, + ` +kind: Deployment +spec: + foo: bar1 + baz: buz +`, + }, + + {`merge Map -- add list, missing from dest`, + ` +kind: Deployment +spec: + foo: bar1 + baz: buz +`, + ` +kind: Deployment +`, + ` +kind: Deployment +spec: + foo: bar1 + baz: buz +`, + }, + + {`merge Map -- add Map first`, + ` +kind: Deployment +spec: + foo: bar1 + baz: buz +`, + ` +kind: Deployment +spec: + foo: bar1 +`, + ` +kind: Deployment +spec: + foo: bar1 + baz: buz +`, + }, + + {`merge Map -- add Map second`, + ` +kind: Deployment +spec: + baz: buz + foo: bar1 +`, + ` +kind: Deployment +spec: + foo: bar1 +`, + ` +kind: Deployment +spec: + foo: bar1 + baz: buz +`, + }, + + // + // Test Case + // + {`keep map -- map missing from src`, + ` +kind: Deployment +`, + ` +kind: Deployment +spec: + foo: bar1 + baz: buz +`, + ` +kind: Deployment +spec: + foo: bar1 + baz: buz +`, + }, + + // + // Test Case + // + {`keep map -- empty list in src`, + ` +kind: Deployment +items: {} +`, + ` +kind: Deployment +spec: + foo: bar1 + baz: buz +`, + ` +kind: Deployment +spec: + foo: bar1 + baz: buz +items: {} +`, + }, + + // + // Test Case + // + {`remove Map -- null in src`, + ` +kind: Deployment +spec: null +`, + ` +kind: Deployment +spec: + foo: bar1 + baz: buz +`, + ` +kind: Deployment +`, + }, +} diff --git a/kyaml/yaml/merge2/merge2.go b/kyaml/yaml/merge2/merge2.go new file mode 100644 index 000000000..c6894fab8 --- /dev/null +++ b/kyaml/yaml/merge2/merge2.go @@ -0,0 +1,201 @@ +// Copyright 2019 The Kubernetes Authors. +// SPDX-License-Identifier: Apache-2.0 + +// Package merge contains libraries for merging fields from one RNode to another +// RNode +package merge2 + +import ( + "sigs.k8s.io/kustomize/kyaml/yaml" + "sigs.k8s.io/kustomize/kyaml/yaml/walk" +) + +const Help = ` +Description: + + merge merges fields from a source to a destination, overriding the destination fields + where they differ. + + ### Merge Rules + + Fields are recursively merged using the following rules: + + - scalars + - if present only in the dest, it keeps its value + - if present in the src and is non-null, take the src value -- if ` + "`null`" + `, clear it +` + " - example src: `5`, dest: `3` => result: `5`" + ` + + - non-associative lists -- lists without a merge key + - if present only in the dest, it keeps its value + - if present in the src and is non-null, take the src value -- if ` + "`null`" + `, clear it +` + " - example src: `[1, 2, 3]`, dest: `[a, b, c]` => result: `[1, 2, 3]`" + ` + + - map keys and fields -- paired by the map-key / field-name + - if present only in the dest, it keeps its value + - if present only in the src, it is added to the dest + - if the field is present in both the src and dest, and the src value is 'null', the field is removed from the dest + - if the field is present in both the src and dest, the value is recursively merged +` + " - example src: `{'key1': 'value1', 'key2': 'value2'}`, dest: `{'key2': 'value0', 'key3': 'value3'}` => result: `{'key1': 'value1', 'key2': 'value2', 'key3': 'value3'}`" + ` + + - associative list elements -- paired by the associative key + - if present only in the dest, it keeps its value in the list + - if present only in the src, it is added to the dest list + - if the field is present in both the src and dest, the value is recursively merged + + ### Associative Keys + + Associative keys are used to identify "same" elements within 2 different lists, and merge them. + The following fields are recognized as associative keys: + +` + "[`mountPath`, `devicePath`, `ip`, `type`, `topologyKey`, `name`, `containerPort`]" + ` + + Any lists where all of the elements contain associative keys will be merged as associative lists. + + ### Example + + > Source + + apiVersion: apps/v1 + kind: Deployment + spec: + replicas: 3 # scalar + template: + spec: + containers: # associative list -- (name) + - name: nginx + image: nginx:1.7 + command: ['new_run.sh', 'arg1'] # non-associative list + - name: sidecar2 + image: sidecar2:v1 + + > Destination + + apiVersion: apps/v1 + kind: Deployment + spec: + replicas: 1 + template: + spec: + containers: + - name: nginx + image: nginx:1.6 + command: ['old_run.sh', 'arg0'] + - name: sidecar1 + image: sidecar1:v1 + + > Result + + apiVersion: apps/v1 + kind: Deployment + spec: + replicas: 3 # scalar + template: + spec: + containers: # associative list -- (name) + - name: nginx + image: nginx:1.7 + command: ['new_run.sh', 'arg1'] # non-associative list + - name: sidecar1 + image: sidecar1:v1 + - name: sidecar2 + image: sidecar2:v1 +` + +// Merge merges fields from src into dest. +func Merge(src, dest *yaml.RNode) (*yaml.RNode, error) { + return walk.Walker{Sources: []*yaml.RNode{dest, src}, Visitor: Merger{}}.Walk() +} + +// Merge parses the arguments, and merges fields from srcStr into destStr. +func MergeStrings(srcStr, destStr string) (string, error) { + src, err := yaml.Parse(srcStr) + if err != nil { + return "", err + } + dest, err := yaml.Parse(destStr) + if err != nil { + return "", err + } + + result, err := Merge(src, dest) + if err != nil { + return "", err + } + return result.String() +} + +type Merger struct { + // for forwards compatibility when new functions are added to the interface +} + +var _ walk.Visitor = Merger{} + +func (m Merger) VisitMap(nodes walk.Sources) (*yaml.RNode, error) { + if err := m.SetComments(nodes); err != nil { + return nil, err + } + if yaml.IsEmpty(nodes.Dest()) { + // Add + return nodes.Origin(), nil + } + if yaml.IsNull(nodes.Origin()) { + // clear the value + return walk.ClearNode, nil + } + // Recursively Merge dest + return nodes.Dest(), nil +} + +func (m Merger) VisitScalar(nodes walk.Sources) (*yaml.RNode, error) { + if err := m.SetComments(nodes); err != nil { + return nil, err + } + // Override value + if nodes.Origin() != nil { + return nodes.Origin(), nil + } + // Keep + return nodes.Dest(), nil +} + +func (m Merger) VisitList(nodes walk.Sources, kind walk.ListKind) (*yaml.RNode, error) { + if err := m.SetComments(nodes); err != nil { + return nil, err + } + if kind == walk.NonAssociateList { + // Override value + if nodes.Origin() != nil { + return nodes.Origin(), nil + } + // Keep + return nodes.Dest(), nil + } + + // Add + if yaml.IsEmpty(nodes.Dest()) { + return nodes.Origin(), nil + } + // Clear + if yaml.IsNull(nodes.Origin()) { + return walk.ClearNode, nil + } + // Recursively Merge dest + return nodes.Dest(), nil +} + +// SetComments copies the dest comments to the source comments if they are present +// on the source. +func (m Merger) SetComments(sources walk.Sources) error { + source := sources.Origin() + dest := sources.Dest() + if source != nil && source.YNode().FootComment != "" { + dest.YNode().FootComment = source.YNode().FootComment + } + if source != nil && source.YNode().HeadComment != "" { + dest.YNode().HeadComment = source.YNode().HeadComment + } + if source != nil && source.YNode().LineComment != "" { + dest.YNode().LineComment = source.YNode().LineComment + } + return nil +} diff --git a/kyaml/yaml/merge2/merge2_old_test.go b/kyaml/yaml/merge2/merge2_old_test.go new file mode 100644 index 000000000..95fce2b41 --- /dev/null +++ b/kyaml/yaml/merge2/merge2_old_test.go @@ -0,0 +1,510 @@ +// Copyright 2019 The Kubernetes Authors. +// SPDX-License-Identifier: Apache-2.0 + +package merge2_test + +import ( + "bytes" + "testing" + + "github.com/stretchr/testify/assert" + "sigs.k8s.io/kustomize/kyaml/kio/filters" + "sigs.k8s.io/kustomize/kyaml/yaml" + . "sigs.k8s.io/kustomize/kyaml/yaml/merge2" +) + +const dest = ` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: app + labels: + app: java + annotations: + a.b.c: d.e.f + g: h1 + i: j + m: n2 +spec: + template: + spec: + containers: + - name: nginx + image: nginx:1.7.9 + args: ['c', 'a', 'b'] + env: + - name: DEMO_GREETING + value: "Hello from the environment" + - name: DEMO_FAREWELL + value: "Such a sweet sorrow" +` + +func TestMerge_map(t *testing.T) { + dest := yaml.MustParse(dest) + src := yaml.MustParse(` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: app + labels: + app: java + annotations: + a.b.c: d.e.f + g: h2 + k: l + m: n1 +`) + + result, err := Merge(src, dest) + if !assert.NoError(t, err) { + return + } + actual, err := result.String() + if !assert.NoError(t, err) { + return + } + + expected := ` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: app + labels: + app: java + annotations: + a.b.c: d.e.f + g: h2 + i: j + k: l + m: n1 +spec: + template: + spec: + containers: + - name: nginx + image: nginx:1.7.9 + args: + - c + - a + - b + env: + - name: DEMO_GREETING + value: "Hello from the environment" + - name: DEMO_FAREWELL + value: "Such a sweet sorrow" +` + b, err := filters.FormatInput(bytes.NewBufferString(expected)) + if !assert.NoError(t, err) { + return + } + expected = b.String() + + b, err = filters.FormatInput(bytes.NewBufferString(actual)) + if !assert.NoError(t, err) { + return + } + actual = b.String() + + assert.Equal(t, expected, actual) +} + +func TestMerge_clear(t *testing.T) { + dest := yaml.MustParse(dest) + src := yaml.MustParse(` +apiVersion: apps/v1 +kind: Deployment +metadata: + annotations: null +`) + + result, err := Merge(src, dest) + if !assert.NoError(t, err) { + return + } + actual, err := result.String() + if !assert.NoError(t, err) { + return + } + + expected := ` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: app + labels: + app: java +spec: + template: + spec: + containers: + - name: nginx + image: nginx:1.7.9 + args: + - c + - a + - b + env: + - name: DEMO_GREETING + value: "Hello from the environment" + - name: DEMO_FAREWELL + value: "Such a sweet sorrow" +` + b, err := filters.FormatInput(bytes.NewBufferString(expected)) + if !assert.NoError(t, err) { + return + } + expected = b.String() + + b, err = filters.FormatInput(bytes.NewBufferString(actual)) + if !assert.NoError(t, err) { + return + } + actual = b.String() + + assert.Equal(t, expected, actual) +} + +func TestMerge_mapInverse(t *testing.T) { + dest := yaml.MustParse(dest) + src := yaml.MustParse(` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: app + labels: + app: java + annotations: + a.b.c: d.e.f + g: h2 + k: l + m: n1 +`) + + result, err := Merge(dest, src) + if !assert.NoError(t, err) { + return + } + actual, err := result.String() + if !assert.NoError(t, err) { + return + } + + expected := ` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: app + labels: + app: java + annotations: + a.b.c: d.e.f + g: h1 + i: j + k: l + m: n2 +spec: + template: + spec: + containers: + - name: nginx + image: nginx:1.7.9 + args: + - c + - a + - b + env: + - name: DEMO_GREETING + value: "Hello from the environment" + - name: DEMO_FAREWELL + value: "Such a sweet sorrow" +` + b, err := filters.FormatInput(bytes.NewBufferString(expected)) + if !assert.NoError(t, err) { + return + } + expected = b.String() + + b, err = filters.FormatInput(bytes.NewBufferString(actual)) + if !assert.NoError(t, err) { + return + } + actual = b.String() + + assert.Equal(t, expected, actual) +} + +func TestMerge_listElem(t *testing.T) { + dest := yaml.MustParse(dest) + src := yaml.MustParse(` +apiVersion: apps/v1 +kind: Deployment +spec: + template: + spec: + containers: + - name: nginx + env: + - name: DEMO_GREETING + value: "New Demo Greeting" + - name: NEW_DEMO_VALUE + value: "Another Env Not In The Dest" +`) + + result, err := Merge(src, dest) + if !assert.NoError(t, err) { + return + } + actual, err := result.String() + if !assert.NoError(t, err) { + return + } + + expected := `apiVersion: apps/v1 +kind: Deployment +metadata: + name: app + labels: + app: java + annotations: + a.b.c: d.e.f + g: h1 + i: j + m: n2 +spec: + template: + spec: + containers: + - name: nginx + image: nginx:1.7.9 + args: + - c + - a + - b + env: + - name: DEMO_GREETING + value: "New Demo Greeting" + - name: DEMO_FAREWELL + value: "Such a sweet sorrow" + - name: NEW_DEMO_VALUE + value: "Another Env Not In The Dest" +` + + b, err := filters.FormatInput(bytes.NewBufferString(expected)) + if !assert.NoError(t, err) { + return + } + expected = b.String() + + b, err = filters.FormatInput(bytes.NewBufferString(actual)) + if !assert.NoError(t, err) { + return + } + actual = b.String() + + assert.Equal(t, expected, actual) +} + +func TestMerge_list(t *testing.T) { + dest := yaml.MustParse(dest) + src := yaml.MustParse(` +apiVersion: apps/v1 +kind: Deployment +spec: + template: + spec: + containers: + - name: nginx + args: ['e', 'd', 'f'] +`) + + result, err := Merge(src, dest) + if !assert.NoError(t, err) { + return + } + actual, err := result.String() + if !assert.NoError(t, err) { + return + } + + expected := `apiVersion: apps/v1 +kind: Deployment +metadata: + name: app + labels: + app: java + annotations: + a.b.c: d.e.f + g: h1 + i: j + m: n2 +spec: + template: + spec: + containers: + - name: nginx + image: nginx:1.7.9 + args: + - e + - d + - f + env: + - name: DEMO_GREETING + value: "Hello from the environment" + - name: DEMO_FAREWELL + value: "Such a sweet sorrow" +` + + b, err := filters.FormatInput(bytes.NewBufferString(expected)) + if !assert.NoError(t, err) { + return + } + expected = b.String() + + b, err = filters.FormatInput(bytes.NewBufferString(actual)) + if !assert.NoError(t, err) { + return + } + actual = b.String() + + assert.Equal(t, expected, actual) +} + +func TestMerge_commentsKept(t *testing.T) { + actual, err := MergeStrings(` +a: + b: + c: e +`, + ` +a: + b: + # header comment + c: d +`) + if !assert.NoError(t, err) { + return + } + assert.Equal(t, `a: + b: + # header comment + c: e +`, actual) + + actual, err = MergeStrings(` +a: + b: + c: e +`, + ` +a: + b: + c: d + # footer comment +`) + if !assert.NoError(t, err) { + return + } + assert.Equal(t, `a: + b: + c: e + # footer comment +`, actual) + + actual, err = MergeStrings(` +a: + b: + c: e +`, + ` +a: + b: + c: d # line comment +`) + if !assert.NoError(t, err) { + return + } + assert.Equal(t, `a: + b: + c: e +`, actual) +} + +func TestMerge_commentsOverride(t *testing.T) { + actual, err := MergeStrings(` +a: + b: + # header comment + c: e +`, + ` +a: + b: + # replace comment + c: d +`) + if !assert.NoError(t, err) { + return + } + assert.Equal(t, `a: + b: + # replace comment + c: e +`, actual) + + actual, err = MergeStrings(` +a: + b: + c: e + # footer comment +`, + ` +a: + b: + c: d + # replace comment +`) + if !assert.NoError(t, err) { + return + } + assert.Equal(t, `a: + b: + c: e + # replace comment +`, actual) + + actual, err = MergeStrings(` +a: + b: + c: e # line comment +`, + ` +a: + b: + c: d # replace comment +`) + if !assert.NoError(t, err) { + return + } + assert.Equal(t, `a: + b: + c: e # line comment +`, actual) + + actual, err = MergeStrings(` +a: + b: + c: d # line comment +`, + ` +a: + b: + c: d # replace comment +`) + if !assert.NoError(t, err) { + return + } + assert.Equal(t, `a: + b: + c: d # line comment +`, actual) +} diff --git a/kyaml/yaml/merge2/merge2_test.go b/kyaml/yaml/merge2/merge2_test.go new file mode 100644 index 000000000..54ad164ae --- /dev/null +++ b/kyaml/yaml/merge2/merge2_test.go @@ -0,0 +1,47 @@ +// Copyright 2019 The Kubernetes Authors. +// SPDX-License-Identifier: Apache-2.0 + +package merge2_test + +import ( + "bytes" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "sigs.k8s.io/kustomize/kyaml/kio/filters" + . "sigs.k8s.io/kustomize/kyaml/yaml/merge2" +) + +var testCases = [][]testCase{scalarTestCases, listTestCases, elementTestCases, mapTestCases} + +func TestMerge(t *testing.T) { + for i := range testCases { + for _, tc := range testCases[i] { + actual, err := MergeStrings(tc.source, tc.dest) + if !assert.NoError(t, err, tc.description) { + t.FailNow() + } + e, err := filters.FormatInput(bytes.NewBufferString(tc.expected)) + if !assert.NoError(t, err) { + t.FailNow() + } + estr := strings.TrimSpace(e.String()) + a, err := filters.FormatInput(bytes.NewBufferString(actual)) + if !assert.NoError(t, err) { + t.FailNow() + } + astr := strings.TrimSpace(a.String()) + if !assert.Equal(t, estr, astr, tc.description) { + t.FailNow() + } + } + } +} + +type testCase struct { + description string + source string + dest string + expected string +} diff --git a/kyaml/yaml/merge2/scalar_test.go b/kyaml/yaml/merge2/scalar_test.go new file mode 100644 index 000000000..2cc7947e8 --- /dev/null +++ b/kyaml/yaml/merge2/scalar_test.go @@ -0,0 +1,138 @@ +// Copyright 2019 The Kubernetes Authors. +// SPDX-License-Identifier: Apache-2.0 + +package merge2_test + +var scalarTestCases = []testCase{ + {`replace scalar -- different value in dest`, + ` +kind: Deployment +field: value1 +`, + ` +kind: Deployment +field: value0 +`, + ` +kind: Deployment +field: value1 +`, + }, + + {`replace scalar -- missing from dest`, + ` +kind: Deployment +field: value1 +`, + ` +kind: Deployment +`, + ` +kind: Deployment +field: value1 +`, + }, + + // + // Test Case + // + {`keep scalar -- same value in src and dest`, + ` +kind: Deployment +field: value1 +`, + ` +kind: Deployment +field: value1 +`, + ` +kind: Deployment +field: value1 +`, + }, + + // + // Test Case + // + {`keep scalar -- unspecified in src`, + ` +kind: Deployment +`, + ` +kind: Deployment +field: value1 +`, + ` +kind: Deployment +field: value1 +`, + }, + + // + // Test Case + // + {`remove scalar -- null in src`, + ` +kind: Deployment +field: null +`, + ` +kind: Deployment +field: value1 +`, + ` +kind: Deployment +`, + }, + + // + // Test Case + // + {`remove scalar -- empty in src`, + ` +kind: Deployment +field: {} +`, + ` +kind: Deployment +field: value1 +`, + ` +kind: Deployment +field: {} +`, + }, + + // + // Test Case + // + {`remove scalar -- null in src, missing in dest`, + ` +kind: Deployment +field: null +`, + ` +kind: Deployment +`, + ` +kind: Deployment +`, + }, + + // + // Test Case + // + {`merge an empty value`, + ` +kind: Deployment +field: {} +`, + ` +kind: Deployment +`, + ` +kind: Deployment +field: {} +`, + }, +} diff --git a/kyaml/yaml/merge3/element_test.go b/kyaml/yaml/merge3/element_test.go new file mode 100644 index 000000000..d3a28a549 --- /dev/null +++ b/kyaml/yaml/merge3/element_test.go @@ -0,0 +1,422 @@ +// Copyright 2019 The Kubernetes Authors. +// SPDX-License-Identifier: Apache-2.0 + +package merge3_test + +var elementTestCases = []testCase{ + // + // Test Case + // + {`Add an element to an existing list`, + ` +kind: Deployment +containers: +- name: foo + image: foo:1 +`, + ` +kind: Deployment +containers: +- name: foo + image: foo:1 +- name: baz + image: baz:2 +`, + ` +kind: Deployment +containers: +- name: foo + image: foo:1 +`, + ` +kind: Deployment +containers: +- name: foo + image: foo:1 +- image: baz:2 + name: baz + +`, nil}, + + // + // Test Case + // + {`Add an element to a non-existing list`, + ` +kind: Deployment`, + ` +kind: Deployment +containers: +- name: foo + image: foo:bar +`, + ` +kind: Deployment +`, + ` +kind: Deployment +containers: +- image: foo:bar + name: foo +`, nil}, + + {`Add an element to a non-existing list, existing in dest`, + ` +kind: Deployment`, + ` +kind: Deployment +containers: +- name: foo + image: foo:bar +`, + ` +kind: Deployment +containers: +- name: baz + image: baz:bar +`, + ` +kind: Deployment +containers: +- name: baz + image: baz:bar +- image: foo:bar + name: foo +`, nil}, + + // + // Test Case + // TODO(pwittrock): Figure out if there is something better we can do here + // This element is missing from the destination -- only the new fields are added + {`Add a field to the element, element missing from dest`, + ` +kind: Deployment +containers: +- name: foo + image: foo:bar`, + ` +kind: Deployment +containers: +- name: foo + image: foo:bar + command: + - run.sh +`, + ` +kind: Deployment +`, + ` +kind: Deployment +containers: +- command: + - run.sh + name: foo +`, nil}, + + // + // Test Case + // + {`Update a field on the elem, element missing from the dest`, + ` +kind: Deployment +containers: +- name: foo + image: foo:bar + command: + - run.sh +`, + ` +kind: Deployment +containers: +- name: foo + image: foo:bar + command: + - run2.sh +`, + ` +kind: Deployment +`, + ` +kind: Deployment +containers: +- command: + - run2.sh + name: foo +`, nil}, + + // + // Test Case + // + {`Update a field on the elem, element present in the dest`, + ` +kind: Deployment +containers: +- name: foo + image: foo:bar + command: ['run.sh'] +`, + ` +kind: Deployment +containers: +- name: foo + image: foo:bar + command: ['run2.sh'] +`, + ` +kind: Deployment +containers: +- name: foo + image: foo:bar + command: ['run.sh'] +`, + ` +kind: Deployment +containers: +- name: foo + image: foo:bar + command: + - run2.sh +`, nil}, + + // + // Test Case + // + {`Add a field on the elem, element present in the dest`, + ` +kind: Deployment +containers: +- name: foo + image: foo:bar +`, + ` +kind: Deployment +containers: +- name: foo + image: foo:bar + command: ['run2.sh'] +`, + ` +kind: Deployment +containers: +- name: foo + image: foo:bar +`, + ` +kind: Deployment +containers: +- name: foo + image: foo:bar + command: + - run2.sh +`, nil}, + + // + // Test Case + // + {`Add a field on the elem, element and field present in the dest`, + ` +kind: Deployment +containers: +- name: foo + image: foo:bar +`, + ` +kind: Deployment +containers: +- name: foo + image: foo:bar + command: ['run2.sh'] +`, + ` +kind: Deployment +containers: +- name: foo + image: foo:bar + command: ['run.sh'] +`, + ` +kind: Deployment +containers: +- name: foo + image: foo:bar + command: + - run2.sh +`, nil}, + + // + // Test Case + // + {`Ignore an element`, + ` +kind: Deployment +containers: {} +`, + ` +kind: Deployment +containers: {} +`, + ` +kind: Deployment +containers: +- name: foo + image: foo:bar +`, + ` +kind: Deployment +containers: +- name: foo + image: foo:bar +`, nil}, + + // + // Test Case + // + {`Leave deleted`, + ` +kind: Deployment +containers: +- name: foo + image: foo:bar +`, + ` +kind: Deployment +`, + ` +kind: Deployment +`, + ` +kind: Deployment +`, nil}, + + // + // Test Case + // + {`Remove an element -- matching`, + ` +kind: Deployment +containers: +- name: foo + image: foo:bar +`, + ` +kind: Deployment +`, + ` +kind: Deployment +containers: +- name: foo + image: foo:bar +`, + ` +kind: Deployment +`, nil}, + + // + // Test Case + // + {`Remove an element -- field missing from update`, + ` +kind: Deployment +containers: +- name: foo + image: foo:bar +`, + ` +kind: Deployment +`, + ` +kind: Deployment +containers: +- name: foo + image: foo:bar + command: ['run.sh'] +`, + ` +kind: Deployment +`, nil}, + + // + // Test Case + // + {`Remove an element -- element missing`, + ` +kind: Deployment +containers: +- name: foo + image: foo:bar +- name: baz + image: baz:bar +`, + ` +kind: Deployment +containers: +- name: foo + image: foo:bar +`, + ` +kind: Deployment +containers: +- name: foo + image: foo:bar + command: ['run.sh'] +- name: baz + image: baz:bar +`, + ` +kind: Deployment +containers: +- name: foo + image: foo:bar + command: + - run.sh +`, nil}, + + // + // Test Case + // + {`Remove an element -- empty containers`, + ` +kind: Deployment +containers: +- name: foo + image: foo:bar +`, + ` +kind: Deployment +containers: {} +`, + ` +kind: Deployment +containers: +- name: foo + image: foo:bar + command: ['run.sh'] +`, + ` +kind: Deployment +`, nil}, + + // + // Test Case + // + {`Remove an element -- missing list field`, + ` +kind: Deployment +containers: +- name: foo + image: foo:bar +`, + ` +kind: Deployment +`, + ` +kind: Deployment +containers: +- name: foo + image: foo:bar + command: ['run.sh'] +`, + ` +kind: Deployment +`, nil}, +} diff --git a/kyaml/yaml/merge3/list_test.go b/kyaml/yaml/merge3/list_test.go new file mode 100644 index 000000000..0574a5687 --- /dev/null +++ b/kyaml/yaml/merge3/list_test.go @@ -0,0 +1,232 @@ +// Copyright 2019 The Kubernetes Authors. +// SPDX-License-Identifier: Apache-2.0 + +package merge3_test + +var listTestCases = []testCase{ + // List Field Test Cases + + // + // Test Case + // + {`Replace list`, + ` +list: +- 1 +- 2 +- 3`, + ` +list: +- 2 +- 3 +- 4`, + ` +list: +- 1 +- 2 +- 3`, + ` +list: +- 2 +- 3 +- 4`, nil}, + + // + // Test Case + // + {`Add an updated list`, + ` +apiVersion: apps/v1 +list: # old value +- 1 +- 2 +- 3 +`, + ` +apiVersion: apps/v1 +list: # new value +- 2 +- 3 +- 4 +`, + ` +apiVersion: apps/v1`, + ` +apiVersion: apps/v1 +list: +- 2 +- 3 +- 4 +`, nil}, + + // + // Test Case + // + {`Add keep an omitted field`, + ` +apiVersion: apps/v1 +kind: Deployment`, + ` +apiVersion: apps/v1 +kind: StatefulSet`, + ` +apiVersion: apps/v1 +list: # not present in sources +- 2 +- 3 +- 4 +`, + ` +apiVersion: apps/v1 +list: # not present in sources +- 2 +- 3 +- 4 +kind: StatefulSet +`, nil}, + + // + // Test Case + // + // TODO(#36): consider making this an error + {`Change an updated field`, + ` +apiVersion: apps/v1 +list: # old value +- 1 +- 2 +- 3`, + ` +apiVersion: apps/v1 +list: # new value +- 2 +- 3 +- 4`, + ` +apiVersion: apps/v1 +list: # conflicting value +- a +- b +- c`, + ` +apiVersion: apps/v1 +list: # conflicting value +- 2 +- 3 +- 4 +`, nil}, + + // + // Test Case + // + {`Ignore a field -- set`, + ` +apiVersion: apps/v1 +list: # ignore value +- 1 +- 2 +- 3 +`, + ` +apiVersion: apps/v1 +list: # ignore value +- 1 +- 2 +- 3`, ` +apiVersion: apps/v1 +list: +- 2 +- 3 +- 4 +`, ` +apiVersion: apps/v1 +list: +- 2 +- 3 +- 4 +`, nil}, + + // + // Test Case + // + {`Ignore a field -- empty`, + ` +apiVersion: apps/v1 +list: # ignore value +- 1 +- 2 +- 3`, + ` +apiVersion: apps/v1 +list: # ignore value +- 1 +- 2 +- 3`, + ` +apiVersion: apps/v1 +`, + ` +apiVersion: apps/v1 +`, nil}, + + // + // Test Case + // + {`Explicitly clear a field`, + ` +apiVersion: apps/v1`, + ` +apiVersion: apps/v1 +list: null # clear`, + ` +apiVersion: apps/v1 +list: # value to clear +- 1 +- 2 +- 3`, + ` +apiVersion: apps/v1`, nil}, + + // + // Test Case + // + {`Implicitly clear a field`, + ` +apiVersion: apps/v1 +list: # clear value +- 1 +- 2 +- 3`, + ` +apiVersion: apps/v1`, + ` +apiVersion: apps/v1 +list: # old value +- 1 +- 2 +- 3`, + ` +apiVersion: apps/v1`, nil}, + + // + // Test Case + // + // TODO(#36): consider making this an error + {`Implicitly clear a changed field`, + ` +apiVersion: apps/v1 +list: # old value +- 1 +- 2 +- 3`, + ` +apiVersion: apps/v1`, + ` +apiVersion: apps/v1 +list: # old value +- a +- b +- c`, + ` +apiVersion: apps/v1`, nil}, +} diff --git a/kyaml/yaml/merge3/map_test.go b/kyaml/yaml/merge3/map_test.go new file mode 100644 index 000000000..e1ffd5413 --- /dev/null +++ b/kyaml/yaml/merge3/map_test.go @@ -0,0 +1,298 @@ +// Copyright 2019 The Kubernetes Authors. +// SPDX-License-Identifier: Apache-2.0 + +package merge3_test + +var mapTestCases = []testCase{ + // + // Test Case + // + {`Add the annotations map field`, + ` +kind: Deployment`, + ` +kind: Deployment +metadata: + annotations: + d: e # add these annotations +`, + ` +kind: Deployment`, + ` +kind: Deployment +metadata: + annotations: + d: e # add these annotations`, nil}, + + // + // Test Case + // + {`Add an annotation to the field`, + ` +kind: Deployment +metadata: + annotations: + a: b`, + ` +kind: Deployment +metadata: + annotations: + a: b + d: e # add these annotations`, + ` +kind: Deployment +metadata: + annotations: + g: h # keep these annotations`, + ` +kind: Deployment +metadata: + annotations: + g: h # keep these annotations + d: e # add these annotations`, nil}, + + // + // Test Case + // + {`Add an annotation to the field, field missing from dest`, + ` +kind: Deployment +metadata: + annotations: + a: b # ignored because unchanged`, + ` +kind: Deployment +metadata: + annotations: + a: b # ignore because unchanged + d: e`, + ` +kind: Deployment`, + ` +kind: Deployment +metadata: + annotations: + d: e`, nil}, + + // + // Test Case + // + {`Update an annotation on the field, field messing rom the dest`, + ` +kind: Deployment +metadata: + annotations: + a: b + d: c`, + ` +kind: Deployment +metadata: + annotations: + a: b + d: e # set these annotations`, + ` +kind: Deployment +metadata: + annotations: + g: h # keep these annotations`, + ` +kind: Deployment +metadata: + annotations: + g: h # keep these annotations + d: e # set these annotations`, nil}, + + // + // Test Case + // + {`Add an annotation to the field, field missing from dest`, + ` +kind: Deployment +metadata: + annotations: + a: b # ignored because unchanged`, + ` +kind: Deployment +metadata: + annotations: + a: b # ignore because unchanged + d: e`, + ` +kind: Deployment`, + ` +kind: Deployment +metadata: + annotations: + d: e`, nil}, + + // + // Test Case + // + {`Remove an annotation`, + ` +apiVersion: apps/v1 +kind: Deployment +metadata: + annotations: + a: b`, + ` +apiVersion: apps/v1 +kind: Deployment +metadata: + annotations: {}`, + ` +apiVersion: apps/v1 +kind: Deployment +metadata: + annotations: + c: d + a: b`, + ` +apiVersion: apps/v1 +kind: Deployment +metadata: + annotations: + c: d`, nil}, + + // + // Test Case + // + // TODO(#36) support ~annotations~: {} deletion + {`Specify a field as empty that isn't present in the source`, + ` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: foo`, + ` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: foo + annotations: null`, + ` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: foo + annotations: + a: b`, + ` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: foo`, nil}, + + // + // Test Case + // + {`Remove an annotation`, + ` +apiVersion: apps/v1 +kind: Deployment +metadata: + annotations: + a: b`, + ` +apiVersion: apps/v1 +kind: Deployment`, + ` +apiVersion: apps/v1 +kind: Deployment +metadata: + annotations: + c: d + a: b`, + ` +apiVersion: apps/v1 +kind: Deployment +metadata: + annotations: + c: d`, nil}, + + // + // Test Case + // + {`Remove annotations field`, + ` +apiVersion: apps/v1 +kind: Deployment +metadata: + annotations: + a: b`, + ` +apiVersion: apps/v1 +kind: Deployment`, + ` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: foo`, + ` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: foo +`, nil}, + + // + // Test Case + // + {`Remove annotations field, but keep in dest`, + ` +apiVersion: apps/v1 +kind: Deployment +metadata: + annotations: + a: b`, + ` +apiVersion: apps/v1 +kind: Deployment`, + ` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: foo + annotations: + foo: bar # keep this annotation even though the parent field was removed`, + ` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: foo + annotations: + foo: bar # keep this annotation even though the parent field was removed`, nil}, + + // + // Test Case + // + {`Remove annotations, but they are already empty`, + ` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: foo + annotations: + a: b +`, + ` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: foo +`, + ` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: foo + annotations: {} +`, + ` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: foo + annotations: {} +`, nil}, +} diff --git a/kyaml/yaml/merge3/merge3.go b/kyaml/yaml/merge3/merge3.go new file mode 100644 index 000000000..ea7db0879 --- /dev/null +++ b/kyaml/yaml/merge3/merge3.go @@ -0,0 +1,88 @@ +// Copyright 2019 The Kubernetes Authors. +// SPDX-License-Identifier: Apache-2.0 + +// Package merge contains libraries for merging fields from one RNode to another +// RNode +package merge3 + +import ( + "sigs.k8s.io/kustomize/kyaml/yaml" + "sigs.k8s.io/kustomize/kyaml/yaml/walk" +) + +const Help = ` +Description: + + merge3 identifies changes between an original source + updated source and merges the result + into a destination, overriding the destination fields where they have changed between + original and updated. + + ### Resource MergeRules + + - Resources present in the original and deleted from the update are deleted. + - Resources missing from the original and added in the update are added. + - Resources present only in the dest are kept without changes. + - Resources present in both the update and the dest are merged *original + update + dest => dest*. + + ### Field Merge Rules + + Fields are recursively merged using the following rules: + + - scalars + - if present in either dest or updated and 'null', clear the value + - if unchanged between original and updated, keep dest value + - if changed between original and updated (added, deleted, changed), take the updated value + + - non-associative lists -- lists without a merge key + - if present in either dest or updated and 'null', clear the value + - if unchanged between original and updated, keep dest value + - if changed between original and updated (added, deleted, changed), take the updated value + + - map keys and fields -- paired by the map-key / field-name + - if present in either dest or updated and 'null', clear the value + - if present only in the dest, it keeps its value + - if not-present in the dest, add the delta between original-updated as a field + - otherwise recursively merge the value between original, updated, dest + + - associative list elements -- paired by the associative key + - if present only in the dest, it keeps its value + - if not-present in the dest, add the delta between original-updated as a field + - otherwise recursively merge the value between original, updated, dest + + ### Associative Keys + + Associative keys are used to identify "same" elements within 2 different lists, and merge them. + The following fields are recognized as associative keys: + +` + "[`mountPath`, `devicePath`, `ip`, `type`, `topologyKey`, `name`, `containerPort`]" + ` + + Any lists where all of the elements contain associative keys will be merged as associative lists. +` + +func Merge(dest, original, update *yaml.RNode) (*yaml.RNode, error) { + // if update == nil && original != nil => declarative deletion + + return walk.Walker{Visitor: Visitor{}, + Sources: []*yaml.RNode{dest, original, update}}.Walk() +} + +func MergeStrings(dest, original, update string) (string, error) { + srcOriginal, err := yaml.Parse(original) + if err != nil { + return "", err + } + srcUpdated, err := yaml.Parse(update) + if err != nil { + return "", err + } + d, err := yaml.Parse(dest) + if err != nil { + return "", err + } + + result, err := Merge(d, srcOriginal, srcUpdated) + if err != nil { + return "", err + } + return result.String() +} diff --git a/kyaml/yaml/merge3/merge3_test.go b/kyaml/yaml/merge3/merge3_test.go new file mode 100644 index 000000000..905e06cf2 --- /dev/null +++ b/kyaml/yaml/merge3/merge3_test.go @@ -0,0 +1,47 @@ +// Copyright 2019 The Kubernetes Authors. +// SPDX-License-Identifier: Apache-2.0 + +package merge3_test + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" + . "sigs.k8s.io/kustomize/kyaml/yaml/merge3" +) + +var testCases = [][]testCase{scalarTestCases, listTestCases, mapTestCases, elementTestCases} + +func TestMerge(t *testing.T) { + for i := range testCases { + for _, tc := range testCases[i] { + actual, err := MergeStrings(tc.local, tc.origin, tc.update) + if tc.err == nil { + if !assert.NoError(t, err, tc.description) { + t.FailNow() + } + if !assert.Equal(t, + strings.TrimSpace(tc.expected), strings.TrimSpace(actual), tc.description) { + t.FailNow() + } + } else { + if !assert.Errorf(t, err, tc.description) { + t.FailNow() + } + if !assert.Contains(t, tc.err.Error(), err.Error()) { + t.FailNow() + } + } + } + } +} + +type testCase struct { + description string + origin string + update string + local string + expected string + err error +} diff --git a/kyaml/yaml/merge3/scalar_test.go b/kyaml/yaml/merge3/scalar_test.go new file mode 100644 index 000000000..b74d4891d --- /dev/null +++ b/kyaml/yaml/merge3/scalar_test.go @@ -0,0 +1,135 @@ +// Copyright 2019 The Kubernetes Authors. +// SPDX-License-Identifier: Apache-2.0 + +package merge3_test + +var scalarTestCases = []testCase{ + // Scalar Field Test Cases + // + // Test Case + // + {`Set and updated a field`, + `kind: Deployment`, + `kind: StatefulSet`, + `kind: Deployment`, + `kind: StatefulSet`, nil}, + + {`Add an updated field`, + ` +apiVersion: apps/v1 +kind: Deployment # old value`, + ` +apiVersion: apps/v1 +kind: StatefulSet # new value`, + ` +apiVersion: apps/v1`, + ` +apiVersion: apps/v1 +kind: StatefulSet # new value`, nil}, + + {`Add keep an omitted field`, + ` +apiVersion: apps/v1 +kind: Deployment`, + ` +apiVersion: apps/v1 +kind: StatefulSet`, + ` +apiVersion: apps/v1 +spec: foo # field not present in source +`, + ` +apiVersion: apps/v1 +spec: foo # field not present in source +kind: StatefulSet +`, nil}, + + // + // Test Case + // + // TODO(#36): consider making this an error + {`Change an updated field`, + ` +apiVersion: apps/v1 +kind: Deployment # old value`, + ` +apiVersion: apps/v1 +kind: StatefulSet # new value`, + ` +apiVersion: apps/v1 +kind: Service # conflicting value`, + ` +apiVersion: apps/v1 +kind: StatefulSet # new value`, nil}, + + {`Ignore a field`, + ` +apiVersion: apps/v1 +kind: Deployment # ignore this field`, + ` +apiVersion: apps/v1 +kind: Deployment # ignore this field`, + ` +apiVersion: apps/v1`, + ` +apiVersion: apps/v1`, nil}, + + {`Explicitly clear a field`, + ` +apiVersion: apps/v1`, + ` +apiVersion: apps/v1 +kind: null # clear this value`, + ` +apiVersion: apps/v1 +kind: Deployment # value to be cleared`, + ` +apiVersion: apps/v1`, nil}, + + {`Implicitly clear a field`, + ` +apiVersion: apps/v1 +kind: Deployment # clear this field`, + ` +apiVersion: apps/v1`, + ` +apiVersion: apps/v1 +kind: Deployment # clear this field`, + ` +apiVersion: apps/v1`, nil}, + + // + // Test Case + // + // TODO(#36): consider making this an error + {`Implicitly clear a changed field`, + ` +apiVersion: apps/v1 +kind: Deployment`, + ` +apiVersion: apps/v1`, + ` +apiVersion: apps/v1 +kind: StatefulSet`, + ` +apiVersion: apps/v1`, nil}, + + // + // Test Case + // + {`Merge an empty scalar value`, + ` +apiVersion: apps/v1 +`, + ` +apiVersion: apps/v1 +kind: {} +`, + ` +apiVersion: apps/v1 +`, + ` +apiVersion: apps/v1 +kind: {} +`, nil}, +} diff --git a/kyaml/yaml/merge3/visitor.go b/kyaml/yaml/merge3/visitor.go new file mode 100644 index 000000000..ab39cdc8a --- /dev/null +++ b/kyaml/yaml/merge3/visitor.go @@ -0,0 +1,161 @@ +// Copyright 2019 The Kubernetes Authors. +// SPDX-License-Identifier: Apache-2.0 + +package merge3 + +import ( + "sigs.k8s.io/kustomize/kyaml/yaml" + "sigs.k8s.io/kustomize/kyaml/yaml/walk" +) + +type ConflictStrategy uint + +const ( + // TODO: Support more strategies + TakeUpdate ConflictStrategy = 1 + iota +) + +type Visitor struct{} + +func (m Visitor) VisitMap(nodes walk.Sources) (*yaml.RNode, error) { + if yaml.IsNull(nodes.Updated()) || yaml.IsNull(nodes.Dest()) { + // explicitly cleared from either dest or update + return walk.ClearNode, nil + } + if nodes.Dest() == nil && nodes.Updated() == nil { + // implicitly cleared missing from both dest and update + return walk.ClearNode, nil + } + + if nodes.Dest() == nil { + // not cleared, but missing from the dest + // initialize a new value that can be recursively merged + return yaml.NewRNode(&yaml.Node{Kind: yaml.MappingNode}), nil + } + // recursively merge the dest with the original and updated + return nodes.Dest(), nil +} + +func (m Visitor) visitAList(nodes walk.Sources) (*yaml.RNode, error) { + if yaml.IsEmpty(nodes.Updated()) && !yaml.IsEmpty(nodes.Origin()) { + // implicitly cleared from update -- element was deleted + return walk.ClearNode, nil + } + if yaml.IsEmpty(nodes.Dest()) { + // not cleared, but missing from the dest + // initialize a new value that can be recursively merged + return yaml.NewRNode(&yaml.Node{Kind: yaml.SequenceNode}), nil + } + + // recursively merge the dest with the original and updated + return nodes.Dest(), nil +} + +func (m Visitor) VisitScalar(nodes walk.Sources) (*yaml.RNode, error) { + if yaml.IsNull(nodes.Updated()) || yaml.IsNull(nodes.Dest()) { + // explicitly cleared from either dest or update + return nil, nil + } + if yaml.IsEmpty(nodes.Updated()) != yaml.IsEmpty(nodes.Origin()) { + // value added or removed in update + return nodes.Updated(), nil + } + if yaml.IsEmpty(nodes.Updated()) && yaml.IsEmpty(nodes.Origin()) { + // value added or removed in update + return nodes.Dest(), nil + } + + if nodes.Updated().YNode().Value != nodes.Origin().YNode().Value { + // value changed in update + return nodes.Updated(), nil + } + + // unchanged between origin and update, keep the dest + return nodes.Dest(), nil +} + +func (m Visitor) visitNAList(nodes walk.Sources) (*yaml.RNode, error) { + if yaml.IsNull(nodes.Updated()) || yaml.IsNull(nodes.Dest()) { + // explicitly cleared from either dest or update + return walk.ClearNode, nil + } + + if yaml.IsEmpty(nodes.Updated()) != yaml.IsEmpty(nodes.Origin()) { + // value added or removed in update + return nodes.Updated(), nil + } + if yaml.IsEmpty(nodes.Updated()) && yaml.IsEmpty(nodes.Origin()) { + // value not present in source or dest + return nodes.Dest(), nil + } + + // compare origin and update values to see if they have changed + values, err := m.getStrValues(nodes) + if err != nil { + return nil, err + } + if values.Update != values.Origin { + // value changed in update + return nodes.Updated(), nil + } + + // unchanged between origin and update, keep the dest + return nodes.Dest(), nil +} + +func (m Visitor) VisitList(nodes walk.Sources, kind walk.ListKind) (*yaml.RNode, error) { + if kind == walk.AssociativeList { + return m.visitAList(nodes) + } + // non-associative list + return m.visitNAList(nodes) +} + +func (m Visitor) getStrValues(nodes walk.Sources) (strValues, error) { + var uStr, oStr, dStr string + var err error + if nodes.Updated() != nil && nodes.Updated().YNode() != nil { + s := nodes.Updated().YNode().Style + defer func() { + nodes.Updated().YNode().Style = s + }() + nodes.Updated().YNode().Style = yaml.FlowStyle | yaml.SingleQuotedStyle + uStr, err = nodes.Updated().String() + if err != nil { + return strValues{}, err + } + } + if nodes.Origin() != nil && nodes.Origin().YNode() != nil { + s := nodes.Origin().YNode().Style + defer func() { + nodes.Origin().YNode().Style = s + }() + nodes.Origin().YNode().Style = yaml.FlowStyle | yaml.SingleQuotedStyle + oStr, err = nodes.Origin().String() + if err != nil { + return strValues{}, err + } + + } + if nodes.Dest() != nil && nodes.Dest().YNode() != nil { + s := nodes.Dest().YNode().Style + defer func() { + nodes.Dest().YNode().Style = s + }() + nodes.Dest().YNode().Style = yaml.FlowStyle | yaml.SingleQuotedStyle + dStr, err = nodes.Dest().String() + if err != nil { + return strValues{}, err + } + } + + return strValues{Origin: oStr, Update: uStr, Dest: dStr}, nil +} + +type strValues struct { + Origin string + Update string + Dest string +} + +var _ walk.Visitor = Visitor{} diff --git a/kyaml/yaml/order.go b/kyaml/yaml/order.go new file mode 100644 index 000000000..157defc82 --- /dev/null +++ b/kyaml/yaml/order.go @@ -0,0 +1,108 @@ +// Copyright 2019 The Kubernetes Authors. +// SPDX-License-Identifier: Apache-2.0 + +package yaml + +// fieldSortOrder contains the relative ordering of fields when formatting an +// object. +var fieldSortOrder = []string{ + // top-level metadata + "name", "generateName", "namespace", "clusterName", + "apiVersion", "kind", "metadata", "type", + "labels", "annotations", + "spec", "status", + + // secret and configmap + "stringData", "data", "binaryData", + + //cronjobspec, daemonsetspec, deploymentspec, statefulsetspec, + // jobspec fields + "parallelism", "completions", "activeDeadlineSeconds", "backoffLimit", + "replicas", "selector", "manualSelector", "template", + "ttlSecondsAfterFinished", "volumeClaimTemplates", "service", "serviceName", + "podManagementPolicy", "updateStrategy", "strategy", "minReadySeconds", + "revision", "revisionHistoryLimit", "paused", "progressDeadlineSeconds", + + // podspec + // podspec scalars + "restartPolicy", "terminationGracePeriodSeconds", + "activeDeadlineSeconds", "dnsPolicy", "serviceAccountName", + "serviceAccount", "automountServiceAccountToken", "nodeName", + "hostNetwork", "hostPID", "hostIPC", "shareProcessNamespace", "hostname", + "subdomain", "schedulerName", "priorityClassName", "priority", + "runtimeClassName", "enableServiceLinks", + + // podspec lists and maps + "nodeSelector", "hostAliases", + + // podspec objects + "initContainers", "containers", "volumes", "securityContext", + "imagePullSecrets", "affinity", "tolerations", "dnsConfig", + "readinessGates", + + // containers + "image", "command", "args", "workingDir", "ports", "envFrom", "env", + "resources", "volumeMounts", "volumeDevices", "livenessProbe", + "readinessProbe", "lifecycle", "terminationMessagePath", + "terminationMessagePolicy", "imagePullPolicy", "securityContext", + "stdin", "stdinOnce", "tty", + + // service + "clusterIP", "externalIPs", "loadBalancerIP", "loadBalancerSourceRanges", + "externalName", "externalTrafficPolicy", "sessionAffinity", + + // ports + "protocol", "port", "targetPort", "hostPort", "containerPort", "hostIP", + + // volumemount + "readOnly", "mountPath", "subPath", "subPathExpr", "mountPropagation", + + // envvar + envvarsource + "value", "valueFrom", "fieldRef", "resourceFieldRef", "configMapKeyRef", + "secretKeyRef", "prefix", "configMapRef", "secretRef", +} + +type set map[string]interface{} + +func newSet(values ...string) set { + m := map[string]interface{}{} + for _, value := range values { + m[value] = nil + } + return m +} + +func (s set) Has(key string) bool { + _, found := s[key] + return found +} + +// whitelistedListSortKinds contains the set of kinds that are whitelisted +// for sorting list field elements +var WhitelistedListSortKinds = newSet( + "CronJob", "DaemonSet", "Deployment", "Job", "ReplicaSet", "StatefulSet", + "ValidatingWebhookConfiguration") + +// whitelistedListSortApis contains the set of apis that are whitelisted for +// sorting list field elements +var WhitelistedListSortApis = newSet( + "apps/v1", "apps/v1beta1", "apps/v1beta2", "batch/v1", "batch/v1beta1", + "extensions/v1beta1", "v1", "admissionregistration.k8s.io/v1beta1") + +// whitelistedListSortFields contains json paths to list fields that should +// be sorted, and the field they should be sorted by +var WhitelistedListSortFields = map[string]string{ + ".spec.template.spec.containers": "name", + ".webhooks.rules.operations": "", +} + +// FieldOrder indexes fields and maps them to relative precedence +var FieldOrder map[string]int + +func init() { + // create an index of field orderings + FieldOrder = map[string]int{} + for i, f := range fieldSortOrder { + FieldOrder[f] = i + 1 + } +} diff --git a/kyaml/yaml/types.go b/kyaml/yaml/types.go new file mode 100644 index 000000000..06f956fc4 --- /dev/null +++ b/kyaml/yaml/types.go @@ -0,0 +1,534 @@ +// Copyright 2019 The Kubernetes Authors. +// SPDX-License-Identifier: Apache-2.0 + +package yaml + +import ( + "bytes" + "errors" + "reflect" + "strings" + + "gopkg.in/yaml.v3" + "sigs.k8s.io/kustomize/kyaml/sets" +) + +const ( + // NullNodeTag is the tag set for a yaml.Document that contains no data -- e.g. it isn't a + // Map, Slice, Document, etc + NullNodeTag = "!!null" +) + +func NullNode() *RNode { + return NewRNode(&Node{Tag: NullNodeTag}) +} + +func IsMissingOrNull(node *RNode) bool { + if node == nil || node.YNode() == nil || node.YNode().Tag == NullNodeTag { + return true + } + return false +} + +func IsEmpty(node *RNode) bool { + if node == nil || node.YNode() == nil || node.YNode().Tag == NullNodeTag { + return true + } + + if node.YNode().Kind == yaml.MappingNode && len(node.YNode().Content) == 0 { + return true + } + if node.YNode().Kind == yaml.SequenceNode && len(node.YNode().Content) == 0 { + return true + } + + return false +} + +func IsNull(node *RNode) bool { + return node != nil && node.YNode() != nil && node.YNode().Tag == NullNodeTag +} + +func IsFieldEmpty(node *MapNode) bool { + if node == nil || node.Value == nil || node.Value.YNode() == nil || + node.Value.YNode().Tag == NullNodeTag { + return true + } + + if node.Value.YNode().Kind == yaml.MappingNode && len(node.Value.YNode().Content) == 0 { + return true + } + if node.Value.YNode().Kind == yaml.SequenceNode && len(node.Value.YNode().Content) == 0 { + return true + } + + return false +} + +func IsFieldNull(node *MapNode) bool { + return node != nil && node.Value != nil && node.Value.YNode() != nil && + node.Value.YNode().Tag == NullNodeTag +} + +// Parser parses values into configuration. +type Parser struct { + Kind string `yaml:"kind,omitempty"` + Value string `yaml:"value,omitempty"` +} + +func (p Parser) Filter(_ *RNode) (*RNode, error) { + d := yaml.NewDecoder(bytes.NewBuffer([]byte(p.Value))) + o := &RNode{value: &yaml.Node{}} + return o, d.Decode(o.value) +} + +// Parse parses a yaml string into an *RNode +func Parse(value string) (*RNode, error) { + return Parser{Value: value}.Filter(nil) +} + +// TODO(pwittrock): test this +func GetStyle(styles ...string) Style { + var style Style + for _, s := range styles { + switch s { + case "TaggedStyle": + style |= TaggedStyle + case "DoubleQuotedStyle": + style |= DoubleQuotedStyle + case "SingleQuotedStyle": + style |= SingleQuotedStyle + case "LiteralStyle": + style |= LiteralStyle + case "FoldedStyle": + style |= FoldedStyle + case "FlowStyle": + style |= FlowStyle + } + } + return style +} + +// MustParse parses a yaml string into an *RNode and panics if there is an error +func MustParse(value string) *RNode { + v, err := Parser{Value: value}.Filter(nil) + if err != nil { + panic(err) + } + return v +} + +// NewScalarRNode returns a new Scalar *RNode containing the provided value. +func NewScalarRNode(value string) *RNode { + return &RNode{ + value: &yaml.Node{ + Kind: yaml.ScalarNode, + Value: value, + }} +} + +// NewListRNode returns a new List *RNode containing the provided value. +func NewListRNode(values ...string) *RNode { + seq := &RNode{value: &yaml.Node{Kind: yaml.SequenceNode}} + for _, v := range values { + seq.value.Content = append(seq.value.Content, &yaml.Node{ + Kind: yaml.ScalarNode, + Value: v, + }) + } + return seq +} + +// NewRNode returns a new *RNode containing the provided value. +func NewRNode(value *yaml.Node) *RNode { + if value != nil { + value.Style = 0 + } + return &RNode{value: value} +} + +// GrepFilter may modify or walk the RNode. +// When possible, Filters should be serializable to yaml so that they can be described +// declaratively as data. +// +// Analogous to http://www.linfo.org/filters.html +type Filter interface { + Filter(object *RNode) (*RNode, error) +} + +type FilterFunc func(object *RNode) (*RNode, error) + +func (f FilterFunc) Filter(object *RNode) (*RNode, error) { + return f(object) +} + +// RNode provides functions for manipulating Kubernetes Resources +// Objects unmarshalled into *yaml.Nodes +type RNode struct { + // fieldPath contains the path from the root of the KubernetesObject to + // this field. + // Only field names are captured in the path. + // e.g. a image field in a Deployment would be + // 'spec.template.spec.containers.image' + fieldPath []string + + // FieldValue contains the value. + // FieldValue is always set: + // field: field value + // list entry: list entry value + // object root: object root + value *yaml.Node + + Match []string +} + +// MapNode wraps a field key and value. +type MapNode struct { + Key *RNode + Value *RNode +} + +type MapNodeSlice []*MapNode + +func (m MapNodeSlice) Keys() []*RNode { + var keys []*RNode + for i := range m { + if m[i] != nil { + keys = append(keys, m[i].Key) + } + } + return keys +} + +func (m MapNodeSlice) Values() []*RNode { + var values []*RNode + for i := range m { + if m[i] != nil { + values = append(values, m[i].Value) + } else { + values = append(values, nil) + } + } + return values +} + +// ResourceMeta contains the metadata for a Resource. +type ResourceMeta struct { + ApiVersion string `yaml:"apiVersion,omitempty"` + Kind string `yaml:"kind,omitempty"` + ObjectMeta `yaml:"metadata,omitempty"` +} + +func NewResourceMeta(name string, typeMeta ResourceMeta) ResourceMeta { + return ResourceMeta{ + Kind: typeMeta.Kind, + ApiVersion: typeMeta.ApiVersion, + ObjectMeta: ObjectMeta{Name: name}, + } +} + +type ObjectMeta struct { + Name string `yaml:"name,omitempty"` + Namespace string `yaml:"namespace,omitempty"` + Labels map[string]string `yaml:"labels,omitempty"` + Annotations map[string]string `yaml:"annotations,omitempty"` +} + +var MissingMetaError = errors.New("missing Resource metadata") + +func (rn *RNode) GetMeta() (ResourceMeta, error) { + m := ResourceMeta{} + b := &bytes.Buffer{} + e := NewEncoder(b) + if err := e.Encode(rn.YNode()); err != nil { + return m, err + } + if err := e.Close(); err != nil { + return m, err + } + d := yaml.NewDecoder(b) + d.KnownFields(false) // only want to parse the metadata + if err := d.Decode(&m); err != nil { + return m, err + } + if reflect.DeepEqual(m, ResourceMeta{}) { + return m, MissingMetaError + } + return m, nil +} + +// Pipe sequentially invokes each GrepFilter, and passes the result to the next +// GrepFilter. +// +// Analogous to http://www.linfo.org/pipes.html +// +// * rn is provided as input to the first GrepFilter. +// * if any GrepFilter returns an error, immediately return the error +// * if any GrepFilter returns a nil RNode, immediately return nil, nil +// * if all Filters succeed with non-empty results, return the final result +func (rn *RNode) Pipe(functions ...Filter) (*RNode, error) { + // check if rn is nil to make chaining Pipe calls easier + if rn == nil { + return nil, nil + } + + var err error + var v *RNode + if rn.value != nil && rn.value.Kind == yaml.DocumentNode { + // the first node may be a DocumentNode containing a single MappingNode + v = &RNode{value: rn.value.Content[0]} + } else { + v = rn + } + + // return each fn in sequence until encountering an error or missing value + for _, c := range functions { + v, err = c.Filter(v) + if err != nil || v == nil { + return v, err + } + } + return v, err +} + +// Document returns the Node RNode for the value. Does not unwrap the node if it is a +// DocumentNodes +func (rn *RNode) Document() *yaml.Node { + return rn.value +} + +// YNode returns the yaml.Node value. If the yaml.Node value is a DocumentNode, +// YNode will return the DocumentNode Content entry instead of the DocumentNode. +func (rn *RNode) YNode() *yaml.Node { + if rn == nil || rn.value == nil { + return nil + } + if rn.value.Kind == yaml.DocumentNode { + return rn.value.Content[0] + } + return rn.value +} + +// SetYNode sets the yaml.Node value. +func (rn *RNode) SetYNode(node *yaml.Node) { + if rn.value == nil || node == nil { + rn.value = node + return + } + *rn.value = *node +} + +// SetYNode sets the value on a Document. +func (rn *RNode) AppendToFieldPath(parts ...string) { + rn.fieldPath = append(rn.fieldPath, parts...) +} + +// FieldPath returns the field path from the object root to rn. Does not include list indexes. +func (rn *RNode) FieldPath() []string { + return rn.fieldPath +} + +const ( + Trim = "Trim" + Flow = "Flow" +) + +func String(node *yaml.Node, opts ...string) (string, error) { + if node == nil { + return "", nil + } + optsSet := sets.String{} + optsSet.Insert(opts...) + if optsSet.Has(Flow) { + oldStyle := node.Style + defer func() { + node.Style = oldStyle + }() + node.Style = yaml.FlowStyle + } + + b := &bytes.Buffer{} + e := NewEncoder(b) + err := e.Encode(node) + e.Close() + val := b.String() + if optsSet.Has(Trim) { + val = strings.TrimSpace(val) + } + return val, err +} + +// NewScalarRNode returns the yaml NewScalarRNode representation of the RNode value. +func (rn *RNode) String() (string, error) { + if rn == nil { + return "", nil + } + return String(rn.value) +} + +func (rn *RNode) MustString() string { + s, err := rn.String() + if err != nil { + panic(err) + } + return s +} + +// Content returns the value node's Content field. +func (rn *RNode) Content() []*yaml.Node { + if rn == nil { + return nil + } + return rn.YNode().Content +} + +// Fields returns the list of fields for a ResourceNode containing a MappingNode +// value. +func (rn *RNode) Fields() ([]string, error) { + if err := ErrorIfInvalid(rn, yaml.MappingNode); err != nil { + return nil, err + } + var fields []string + for i := 0; i < len(rn.Content()); i += 2 { + fields = append(fields, rn.Content()[i].Value) + } + return fields, nil +} + +// Field returns the fieldName, fieldValue pair for MappingNodes. Returns nil for non-MappingNodes. +func (rn *RNode) Field(field string) *MapNode { + if rn.YNode().Kind != yaml.MappingNode { + return nil + } + for i := 0; i < len(rn.Content()); IncrementFieldIndex(&i) { + isMatchingField := rn.Content()[i].Value == field + if isMatchingField { + return &MapNode{Key: NewRNode(rn.Content()[i]), Value: NewRNode(rn.Content()[i+1])} + } + } + return nil +} + +// VisitFields calls fn for each field in rn. +func (rn *RNode) VisitFields(fn func(node *MapNode) error) error { + // get the list of srcFieldNames + srcFieldNames, err := rn.Fields() + if err != nil { + return err + } + + // visit each field + for _, fieldName := range srcFieldNames { + if err := fn(rn.Field(fieldName)); err != nil { + return err + } + } + return nil +} + +// Elements returns a list of elements for a ResourceNode containing a +// SequenceNode value. +func (rn *RNode) Elements() ([]*RNode, error) { + if err := ErrorIfInvalid(rn, yaml.SequenceNode); err != nil { + return nil, err + } + var elements []*RNode + for i := 0; i < len(rn.Content()); i += 1 { + elements = append(elements, NewRNode(rn.Content()[i])) + } + return elements, nil +} + +func (rn *RNode) ElementValues(key string) ([]string, error) { + if err := ErrorIfInvalid(rn, yaml.SequenceNode); err != nil { + return nil, err + } + var elements []string + for i := 0; i < len(rn.Content()); i += 1 { + field := NewRNode(rn.Content()[i]).Field(key) + if !IsFieldEmpty(field) { + elements = append(elements, field.Value.YNode().Value) + } + } + return elements, nil +} + +// Element returns the element in the list which contains the field matching the value. +// Returns nil for non-SequenceNodes +func (rn *RNode) Element(key, value string) *RNode { + if rn.YNode().Kind != yaml.SequenceNode { + return nil + } + elem, err := rn.Pipe(MatchElement(key, value)) + if err != nil { + return nil + } + return elem +} + +// VisitElements calls fn for each element in the list. +func (rn *RNode) VisitElements(fn func(node *RNode) error) error { + elements, err := rn.Elements() + if err != nil { + return err + } + + for i := range elements { + if err := fn(elements[i]); err != nil { + return err + } + } + return nil +} + +// AssociativeSequencePaths is a map of paths to sequences that have associative keys. +// The order sets the precedence of the merge keys -- if multiple keys are present +// in the list, then the FIRST key which ALL elements have is used as the +// associative key. +var AssociativeSequenceKeys = []string{ + "mountPath", "devicePath", "ip", "type", "topologyKey", "name", "containerPort", +} + +func IsAssociative(nodes []*RNode) bool { + for i := range nodes { + node := nodes[i] + if IsEmpty(node) { + continue + } + if node.IsAssociative() { + return true + } + } + return false +} + +// IsAssociative returns true if the RNode is for an associative list. +func (rn *RNode) IsAssociative() bool { + return rn.GetAssociativeKey() != "" +} + +// GetAssociativeKey returns the associative key used to merge the list, or "" if the +// list is not associative. +func (rn *RNode) GetAssociativeKey() string { + // look for any associative keys in the first element + for _, key := range AssociativeSequenceKeys { + if checkKey(key, rn.Content()) { + return key + } + } + + // element doesn't have an associative keys + return "" +} + +// checkKey returns true if all elems have the key +func checkKey(key string, elems []*Node) bool { + count := 0 + for i := range elems { + elem := NewRNode(elems[i]) + if elem.Field(key) != nil { + count++ + } + } + return count == len(elems) +} diff --git a/kyaml/yaml/walk/associative_sequence.go b/kyaml/yaml/walk/associative_sequence.go new file mode 100644 index 000000000..479d8a22d --- /dev/null +++ b/kyaml/yaml/walk/associative_sequence.go @@ -0,0 +1,128 @@ +// Copyright 2019 The Kubernetes Authors. +// SPDX-License-Identifier: Apache-2.0 + +package walk + +import ( + "strings" + + "github.com/go-errors/errors" + "sigs.k8s.io/kustomize/kyaml/sets" + "sigs.k8s.io/kustomize/kyaml/yaml" +) + +func (l *Walker) walkAssociativeSequence() (*yaml.RNode, error) { + + // may require initializing the dest node + dest, err := l.Sources.setDestNode(l.VisitList(l.Sources, AssociativeList)) + if dest == nil || err != nil { + return nil, err + } + + // find the list of elements we need to recursively walk + key, err := l.elementKey() + if err != nil { + return nil, err + } + values, err := l.elementValues(key) + if err != nil { + return nil, err + } + + // recursively set the elements in the list + for _, value := range values { + val, err := Walker{Visitor: l, + Sources: l.elementValue(key, value)}.Walk() + if err != nil { + return nil, err + } + if yaml.IsEmpty(val) { + _, err = dest.Pipe(yaml.ElementSetter{Key: key, Value: value}) + if err != nil { + return nil, err + } + continue + } + + if val.Field(key) == nil { + // make sure the key is set on the field + _, err = val.Pipe(yaml.SetField(key, yaml.NewScalarRNode(value))) + if err != nil { + return nil, err + } + } + + // this handles empty and non-empty values + _, err = dest.Pipe(yaml.ElementSetter{Element: val.YNode(), Key: key, Value: value}) + if err != nil { + return nil, err + } + } + // field is empty + if yaml.IsEmpty(dest) { + return nil, nil + } + return dest, nil +} + +// elementKey returns the merge key to use for the associative list +func (l Walker) elementKey() (string, error) { + var key string + for i := range l.Sources { + if l.Sources[i] != nil && len(l.Sources[i].Content()) > 0 { + newKey := l.Sources[i].GetAssociativeKey() + if key != "" && key != newKey { + return "", errors.Errorf( + "conflicting merge keys [%s,%s] for field %s", + key, newKey, strings.Join(l.Path, ".")) + } + key = newKey + } + } + if key == "" { + return "", errors.Errorf("no merge key found for field %s", + strings.Join(l.Path, ".")) + } + return key, nil +} + +// elementValues returns a slice containing all values for the field across all elements +// from all sources. +// Return value slice is ordered using the original ordering from the elements, where +// elements missing from earlier sources appear later. +func (l Walker) elementValues(key string) ([]string, error) { + // use slice to to keep elements in the original order + // dest node must be first + var returnValues []string + seen := sets.String{} + for i := range l.Sources { + if l.Sources[i] == nil { + continue + } + + // add the value of the field for each element + // don't check error, we know this is a list node + values, _ := l.Sources[i].ElementValues(key) + for _, s := range values { + if seen.Has(s) { + continue + } + returnValues = append(returnValues, s) + seen.Insert(s) + } + } + return returnValues, nil +} + +// fieldValue returns a slice containing each source's value for fieldName +func (l Walker) elementValue(key, value string) []*yaml.RNode { + var fields []*yaml.RNode + for i := range l.Sources { + if l.Sources[i] == nil { + fields = append(fields, nil) + continue + } + fields = append(fields, l.Sources[i].Element(key, value)) + } + return fields +} diff --git a/kyaml/yaml/walk/map.go b/kyaml/yaml/walk/map.go new file mode 100644 index 000000000..375c807c2 --- /dev/null +++ b/kyaml/yaml/walk/map.go @@ -0,0 +1,81 @@ +// Copyright 2019 The Kubernetes Authors. +// SPDX-License-Identifier: Apache-2.0 + +package walk + +import ( + "sort" + + "sigs.k8s.io/kustomize/kyaml/sets" + "sigs.k8s.io/kustomize/kyaml/yaml" +) + +// walkMap returns the value of VisitMap +// +// - call VisitMap +// - set the return value on l.Dest +// - walk each source field +// - set each source field value on l.Dest +func (l Walker) walkMap() (*yaml.RNode, error) { + // get the new map value + dest, err := l.Sources.setDestNode(l.VisitMap(l.Sources)) + if dest == nil || err != nil { + return nil, err + } + + // recursively set the field values on the map + for _, key := range l.fieldNames() { + val, err := Walker{Visitor: l, + Sources: l.fieldValue(key), Path: append(l.Path, key)}.Walk() + if err != nil { + return nil, err + } + + // this handles empty and non-empty values + _, err = dest.Pipe(yaml.FieldSetter{Name: key, Value: val}) + if err != nil { + return nil, err + } + } + + return dest, nil +} + +// valueIfPresent returns node.Value if node is non-nil, otherwise returns nil +func (l Walker) valueIfPresent(node *yaml.MapNode) *yaml.RNode { + if node == nil { + return nil + } + return node.Value +} + +// fieldNames returns a sorted slice containing the names of all fields that appear in any of +// the sources +func (l Walker) fieldNames() []string { + fields := sets.String{} + for _, s := range l.Sources { + if s == nil { + continue + } + // don't check error, we know this is a mapping node + sFields, _ := s.Fields() + fields.Insert(sFields...) + } + result := fields.List() + sort.Strings(result) + return result +} + +// fieldValue returns a slice containing each source's value for fieldName +func (l Walker) fieldValue(fieldName string) []*yaml.RNode { + var fields []*yaml.RNode + for i := range l.Sources { + if l.Sources[i] == nil { + fields = append(fields, nil) + continue + } + field := l.Sources[i].Field(fieldName) + fields = append(fields, l.valueIfPresent(field)) + } + return fields +} diff --git a/kyaml/yaml/walk/nonassociative_sequence.go b/kyaml/yaml/walk/nonassociative_sequence.go new file mode 100644 index 000000000..7a34da916 --- /dev/null +++ b/kyaml/yaml/walk/nonassociative_sequence.go @@ -0,0 +1,13 @@ +// Copyright 2019 The Kubernetes Authors. +// SPDX-License-Identifier: Apache-2.0 + +package walk + +import ( + "sigs.k8s.io/kustomize/kyaml/yaml" +) + +// walkNonAssociativeSequence returns the value of VisitList +func (l Walker) walkNonAssociativeSequence() (*yaml.RNode, error) { + return l.VisitList(l.Sources, NonAssociateList) +} diff --git a/kyaml/yaml/walk/scalar.go b/kyaml/yaml/walk/scalar.go new file mode 100644 index 000000000..18d63be46 --- /dev/null +++ b/kyaml/yaml/walk/scalar.go @@ -0,0 +1,11 @@ +// Copyright 2019 The Kubernetes Authors. +// SPDX-License-Identifier: Apache-2.0 + +package walk + +import "sigs.k8s.io/kustomize/kyaml/yaml" + +// walkScalar returns the value of VisitScalar +func (l Walker) walkScalar() (*yaml.RNode, error) { + return l.VisitScalar(l.Sources) +} diff --git a/kyaml/yaml/walk/visitor.go b/kyaml/yaml/walk/visitor.go new file mode 100644 index 000000000..50982e553 --- /dev/null +++ b/kyaml/yaml/walk/visitor.go @@ -0,0 +1,38 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package walk + +import ( + "sigs.k8s.io/kustomize/kyaml/yaml" +) + +type ListKind int32 + +const ( + AssociativeList ListKind = 1 + iota + NonAssociateList +) + +// Visitor is invoked by walk with source and destination node pairs +type Visitor interface { + VisitMap(nodes Sources) (*yaml.RNode, error) + + VisitScalar(nodes Sources) (*yaml.RNode, error) + + VisitList(nodes Sources, kind ListKind) (*yaml.RNode, error) +} + +// NoOp is returned if GrepFilter should do nothing after calling Set +var ClearNode *yaml.RNode = nil diff --git a/kyaml/yaml/walk/walk.go b/kyaml/yaml/walk/walk.go new file mode 100644 index 000000000..3441ae21a --- /dev/null +++ b/kyaml/yaml/walk/walk.go @@ -0,0 +1,144 @@ +// Copyright 2019 The Kubernetes Authors. +// SPDX-License-Identifier: Apache-2.0 + +package walk + +import ( + "fmt" + "os" + "strings" + + "sigs.k8s.io/kustomize/kyaml/yaml" +) + +// Filter walks the Source RNode and modifies the RNode provided to GrepFilter. +type Walker struct { + // Visitor is invoked by GrepFilter + Visitor + + // Source is the RNode to walk. All Source fields and associative list elements + // will be visited. + Sources Sources + + // Path is the field path to the current Source Node. + Path []string +} + +func (l Walker) Kind() yaml.Kind { + for _, s := range l.Sources { + if !yaml.IsEmpty(s) { + return s.YNode().Kind + } + } + return 0 +} + +// GrepFilter implements yaml.GrepFilter +func (l Walker) Walk() (*yaml.RNode, error) { + // invoke the handler for the corresponding node type + switch l.Kind() { + case yaml.MappingNode: + if err := yaml.ErrorIfAnyInvalidAndNonNull(yaml.MappingNode, l.Sources...); err != nil { + return nil, err + } + return l.walkMap() + case yaml.SequenceNode: + if err := yaml.ErrorIfAnyInvalidAndNonNull(yaml.SequenceNode, l.Sources...); err != nil { + return nil, err + } + if yaml.IsAssociative(l.Sources) { + return l.walkAssociativeSequence() + } else { + return l.walkNonAssociativeSequence() + } + case yaml.ScalarNode: + if err := yaml.ErrorIfAnyInvalidAndNonNull(yaml.ScalarNode, l.Sources...); err != nil { + return nil, err + } + return l.walkScalar() + case 0: + // walk empty nodes as maps + return l.walkMap() + default: + return nil, nil + } +} + +const ( + DestIndex = iota + OriginIndex + UpdatedIndex +) + +type Sources []*yaml.RNode + +// Dest returns the destination node +func (s Sources) Dest() *yaml.RNode { + if len(s) <= DestIndex { + return nil + } + return s[DestIndex] +} + +// Origin returns the origin node +func (s Sources) Origin() *yaml.RNode { + if len(s) <= OriginIndex { + return nil + } + return s[OriginIndex] +} + +// Updated returns the updated node +func (s Sources) Updated() *yaml.RNode { + if len(s) <= UpdatedIndex { + return nil + } + return s[UpdatedIndex] +} + +func (s Sources) String() string { + var values []string + for i := range s { + str, err := s[i].String() + if err != nil { + fmt.Fprintf(os.Stderr, "%v\n", err) + } + values = append(values, str) + } + return strings.Join(values, "\n") +} + +// setDestNode sets the destination source node +func (s Sources) setDestNode(node *yaml.RNode, err error) (*yaml.RNode, error) { + if err != nil { + return nil, err + } + s[0] = node + return node, nil +} + +type FieldSources []*yaml.MapNode + +// Dest returns the destination node +func (s FieldSources) Dest() *yaml.MapNode { + if len(s) <= DestIndex { + return nil + } + return s[DestIndex] +} + +// Origin returns the origin node +func (s FieldSources) Origin() *yaml.MapNode { + if len(s) <= OriginIndex { + return nil + } + return s[OriginIndex] +} + +// Updated returns the updated node +func (s FieldSources) Updated() *yaml.MapNode { + if len(s) <= UpdatedIndex { + return nil + } + return s[UpdatedIndex] +} diff --git a/travis/kyaml-pre-commit.sh b/travis/kyaml-pre-commit.sh new file mode 100755 index 000000000..68c43d9cf --- /dev/null +++ b/travis/kyaml-pre-commit.sh @@ -0,0 +1,5 @@ +#!/bin/bash +set -e + +cd kyaml +make all \ No newline at end of file