kyaml: initial support for yaml and resource manipulation

This commit is contained in:
Phillip Wittrock
2019-11-04 11:27:47 -08:00
parent 588297f1f9
commit efd7c8e3f7
92 changed files with 13733 additions and 0 deletions

View File

@@ -0,0 +1,265 @@
// Copyright 2019 The Kubernetes Authors.
// SPDX-License-Identifier: Apache-2.0
package merge2_test
var elementTestCases = []testCase{
{`merge Element -- keep field in dest`,
`
kind: Deployment
items:
- name: foo
image: foo:v1
`,
`
kind: Deployment
items:
- name: foo
image: foo:v0
command: ['run.sh']
`,
`
kind: Deployment
items:
- name: foo
image: foo:v1
command:
- run.sh
`,
},
{`merge Element -- add field to dest`,
`
kind: Deployment
items:
- name: foo
image: foo:v1
command: ['run.sh']
`,
`
kind: Deployment
items:
- name: foo
image: foo:v0
`,
`
kind: Deployment
items:
- name: foo
image: foo:v1
command:
- run.sh
`,
},
{`merge Element -- add list, empty in dest`,
`
kind: Deployment
items:
- name: foo
image: foo:v1
command: ['run.sh']
`,
`
kind: Deployment
items: {}
`,
`
kind: Deployment
items:
- name: foo
image: foo:v1
command:
- run.sh
`,
},
{`merge Element -- add list, missing from dest`,
`
kind: Deployment
items:
- name: foo
image: foo:v1
command: ['run.sh']
`,
`
kind: Deployment
`,
`
kind: Deployment
items:
- name: foo
image: foo:v1
command:
- run.sh
`,
},
{`merge Element -- add Element first`,
`
kind: Deployment
items:
- name: bar
image: bar:v1
command: ['run2.sh']
- name: foo
image: foo:v1
`,
`
kind: Deployment
items:
- name: foo
image: foo:v0
`,
`
kind: Deployment
items:
- name: foo
image: foo:v1
- name: bar
image: bar:v1
command:
- run2.sh
`,
},
{`merge Element -- add Element second`,
`
kind: Deployment
items:
- name: foo
image: foo:v1
- name: bar
image: bar:v1
command: ['run2.sh']
`,
`
kind: Deployment
items:
- name: foo
image: foo:v0
`,
`
kind: Deployment
items:
- name: foo
image: foo:v1
- name: bar
image: bar:v1
command:
- run2.sh
`,
},
//
// Test Case
//
{`keep list -- list missing from src`,
`
kind: Deployment
`,
`
kind: Deployment
items:
- name: foo
image: foo:v1
- name: bar
image: bar:v1
command: ['run2.sh']
`,
`
kind: Deployment
items:
- name: foo
image: foo:v1
- name: bar
image: bar:v1
command:
- run2.sh
`,
},
//
// Test Case
//
{`keep Element -- element missing in src`,
`
kind: Deployment
items:
- name: foo
image: foo:v1
`,
`
kind: Deployment
items:
- name: foo
image: foo:v0
- name: bar
image: bar:v1
command: ['run2.sh']
`,
`
kind: Deployment
items:
- name: foo
image: foo:v1
- name: bar
image: bar:v1
command:
- run2.sh
`,
},
//
// Test Case
//
{`keep element -- empty list in src`,
`
kind: Deployment
items: {}
`,
`
kind: Deployment
items:
- name: foo
image: foo:v1
- name: bar
image: bar:v1
command:
- run2.sh
`,
`
kind: Deployment
items:
- name: foo
image: foo:v1
- name: bar
image: bar:v1
command:
- run2.sh
`,
},
//
// Test Case
//
{`remove Element -- null in src`,
`
kind: Deployment
items: null
`,
`
kind: Deployment
items:
- name: foo
image: foo:v1
- name: bar
image: bar:v1
command:
- run2.sh
`,
`
kind: Deployment
`,
},
}

View File

@@ -0,0 +1,140 @@
// Copyright 2019 The Kubernetes Authors.
// SPDX-License-Identifier: Apache-2.0
package merge2_test
var listTestCases = []testCase{
{`replace List -- different value in dest`,
`
kind: Deployment
items:
- 1
- 2
- 3
`,
`
kind: Deployment
items:
- 0
- 1
`,
`
kind: Deployment
items:
- 1
- 2
- 3
`,
},
{`replace List -- missing from dest`,
`
kind: Deployment
items:
- 1
- 2
- 3
`,
`
kind: Deployment
`,
`
kind: Deployment
items:
- 1
- 2
- 3
`,
},
//
// Test Case
//
{`keep List -- same value in src and dest`,
`
kind: Deployment
items:
- 1
- 2
- 3
`,
`
kind: Deployment
items:
- 1
- 2
- 3
`,
`
kind: Deployment
items:
- 1
- 2
- 3
`,
},
//
// Test Case
//
{`keep List -- unspecified in src`,
`
kind: Deployment
`,
`
kind: Deployment
items:
- 1
- 2
- 3
`,
`
kind: Deployment
items:
- 1
- 2
- 3
`,
},
//
// Test Case
//
{`remove List -- null in src`,
`
kind: Deployment
items: null
`,
`
kind: Deployment
items:
- 1
- 2
- 3
`,
`
kind: Deployment
`,
},
//
// Test Case
//
{`remove list -- empty in src`,
`
kind: Deployment
items: {}
`,
`
kind: Deployment
items:
- 1
- 2
- 3
`,
`
kind: Deployment
items: {}
`,
},
}

View File

@@ -0,0 +1,186 @@
// Copyright 2019 The Kubernetes Authors.
// SPDX-License-Identifier: Apache-2.0
package merge2_test
var mapTestCases = []testCase{
{`merge Map -- update field in dest`,
`
kind: Deployment
spec:
foo: bar1
`,
`
kind: Deployment
spec:
foo: bar0
baz: buz
`,
`
kind: Deployment
spec:
foo: bar1
baz: buz
`,
},
{`merge Map -- add field to dest`,
`
kind: Deployment
spec:
foo: bar1
baz: buz
`,
`
kind: Deployment
spec:
foo: bar0
`,
`
kind: Deployment
spec:
foo: bar1
baz: buz
`,
},
{`merge Map -- add list, empty in dest`,
`
kind: Deployment
spec:
foo: bar1
baz: buz
`,
`
kind: Deployment
spec: {}
`,
`
kind: Deployment
spec:
foo: bar1
baz: buz
`,
},
{`merge Map -- add list, missing from dest`,
`
kind: Deployment
spec:
foo: bar1
baz: buz
`,
`
kind: Deployment
`,
`
kind: Deployment
spec:
foo: bar1
baz: buz
`,
},
{`merge Map -- add Map first`,
`
kind: Deployment
spec:
foo: bar1
baz: buz
`,
`
kind: Deployment
spec:
foo: bar1
`,
`
kind: Deployment
spec:
foo: bar1
baz: buz
`,
},
{`merge Map -- add Map second`,
`
kind: Deployment
spec:
baz: buz
foo: bar1
`,
`
kind: Deployment
spec:
foo: bar1
`,
`
kind: Deployment
spec:
foo: bar1
baz: buz
`,
},
//
// Test Case
//
{`keep map -- map missing from src`,
`
kind: Deployment
`,
`
kind: Deployment
spec:
foo: bar1
baz: buz
`,
`
kind: Deployment
spec:
foo: bar1
baz: buz
`,
},
//
// Test Case
//
{`keep map -- empty list in src`,
`
kind: Deployment
items: {}
`,
`
kind: Deployment
spec:
foo: bar1
baz: buz
`,
`
kind: Deployment
spec:
foo: bar1
baz: buz
items: {}
`,
},
//
// Test Case
//
{`remove Map -- null in src`,
`
kind: Deployment
spec: null
`,
`
kind: Deployment
spec:
foo: bar1
baz: buz
`,
`
kind: Deployment
`,
},
}

201
kyaml/yaml/merge2/merge2.go Normal file
View File

@@ -0,0 +1,201 @@
// Copyright 2019 The Kubernetes Authors.
// SPDX-License-Identifier: Apache-2.0
// Package merge contains libraries for merging fields from one RNode to another
// RNode
package merge2
import (
"sigs.k8s.io/kustomize/kyaml/yaml"
"sigs.k8s.io/kustomize/kyaml/yaml/walk"
)
const Help = `
Description:
merge merges fields from a source to a destination, overriding the destination fields
where they differ.
### Merge Rules
Fields are recursively merged using the following rules:
- scalars
- if present only in the dest, it keeps its value
- if present in the src and is non-null, take the src value -- if ` + "`null`" + `, clear it
` + " - example src: `5`, dest: `3` => result: `5`" + `
- non-associative lists -- lists without a merge key
- if present only in the dest, it keeps its value
- if present in the src and is non-null, take the src value -- if ` + "`null`" + `, clear it
` + " - example src: `[1, 2, 3]`, dest: `[a, b, c]` => result: `[1, 2, 3]`" + `
- map keys and fields -- paired by the map-key / field-name
- if present only in the dest, it keeps its value
- if present only in the src, it is added to the dest
- if the field is present in both the src and dest, and the src value is 'null', the field is removed from the dest
- if the field is present in both the src and dest, the value is recursively merged
` + " - example src: `{'key1': 'value1', 'key2': 'value2'}`, dest: `{'key2': 'value0', 'key3': 'value3'}` => result: `{'key1': 'value1', 'key2': 'value2', 'key3': 'value3'}`" + `
- associative list elements -- paired by the associative key
- if present only in the dest, it keeps its value in the list
- if present only in the src, it is added to the dest list
- if the field is present in both the src and dest, the value is recursively merged
### Associative Keys
Associative keys are used to identify "same" elements within 2 different lists, and merge them.
The following fields are recognized as associative keys:
` + "[`mountPath`, `devicePath`, `ip`, `type`, `topologyKey`, `name`, `containerPort`]" + `
Any lists where all of the elements contain associative keys will be merged as associative lists.
### Example
> Source
apiVersion: apps/v1
kind: Deployment
spec:
replicas: 3 # scalar
template:
spec:
containers: # associative list -- (name)
- name: nginx
image: nginx:1.7
command: ['new_run.sh', 'arg1'] # non-associative list
- name: sidecar2
image: sidecar2:v1
> Destination
apiVersion: apps/v1
kind: Deployment
spec:
replicas: 1
template:
spec:
containers:
- name: nginx
image: nginx:1.6
command: ['old_run.sh', 'arg0']
- name: sidecar1
image: sidecar1:v1
> Result
apiVersion: apps/v1
kind: Deployment
spec:
replicas: 3 # scalar
template:
spec:
containers: # associative list -- (name)
- name: nginx
image: nginx:1.7
command: ['new_run.sh', 'arg1'] # non-associative list
- name: sidecar1
image: sidecar1:v1
- name: sidecar2
image: sidecar2:v1
`
// Merge merges fields from src into dest.
func Merge(src, dest *yaml.RNode) (*yaml.RNode, error) {
return walk.Walker{Sources: []*yaml.RNode{dest, src}, Visitor: Merger{}}.Walk()
}
// Merge parses the arguments, and merges fields from srcStr into destStr.
func MergeStrings(srcStr, destStr string) (string, error) {
src, err := yaml.Parse(srcStr)
if err != nil {
return "", err
}
dest, err := yaml.Parse(destStr)
if err != nil {
return "", err
}
result, err := Merge(src, dest)
if err != nil {
return "", err
}
return result.String()
}
type Merger struct {
// for forwards compatibility when new functions are added to the interface
}
var _ walk.Visitor = Merger{}
func (m Merger) VisitMap(nodes walk.Sources) (*yaml.RNode, error) {
if err := m.SetComments(nodes); err != nil {
return nil, err
}
if yaml.IsEmpty(nodes.Dest()) {
// Add
return nodes.Origin(), nil
}
if yaml.IsNull(nodes.Origin()) {
// clear the value
return walk.ClearNode, nil
}
// Recursively Merge dest
return nodes.Dest(), nil
}
func (m Merger) VisitScalar(nodes walk.Sources) (*yaml.RNode, error) {
if err := m.SetComments(nodes); err != nil {
return nil, err
}
// Override value
if nodes.Origin() != nil {
return nodes.Origin(), nil
}
// Keep
return nodes.Dest(), nil
}
func (m Merger) VisitList(nodes walk.Sources, kind walk.ListKind) (*yaml.RNode, error) {
if err := m.SetComments(nodes); err != nil {
return nil, err
}
if kind == walk.NonAssociateList {
// Override value
if nodes.Origin() != nil {
return nodes.Origin(), nil
}
// Keep
return nodes.Dest(), nil
}
// Add
if yaml.IsEmpty(nodes.Dest()) {
return nodes.Origin(), nil
}
// Clear
if yaml.IsNull(nodes.Origin()) {
return walk.ClearNode, nil
}
// Recursively Merge dest
return nodes.Dest(), nil
}
// SetComments copies the dest comments to the source comments if they are present
// on the source.
func (m Merger) SetComments(sources walk.Sources) error {
source := sources.Origin()
dest := sources.Dest()
if source != nil && source.YNode().FootComment != "" {
dest.YNode().FootComment = source.YNode().FootComment
}
if source != nil && source.YNode().HeadComment != "" {
dest.YNode().HeadComment = source.YNode().HeadComment
}
if source != nil && source.YNode().LineComment != "" {
dest.YNode().LineComment = source.YNode().LineComment
}
return nil
}

View File

@@ -0,0 +1,510 @@
// Copyright 2019 The Kubernetes Authors.
// SPDX-License-Identifier: Apache-2.0
package merge2_test
import (
"bytes"
"testing"
"github.com/stretchr/testify/assert"
"sigs.k8s.io/kustomize/kyaml/kio/filters"
"sigs.k8s.io/kustomize/kyaml/yaml"
. "sigs.k8s.io/kustomize/kyaml/yaml/merge2"
)
const dest = `
apiVersion: apps/v1
kind: Deployment
metadata:
name: app
labels:
app: java
annotations:
a.b.c: d.e.f
g: h1
i: j
m: n2
spec:
template:
spec:
containers:
- name: nginx
image: nginx:1.7.9
args: ['c', 'a', 'b']
env:
- name: DEMO_GREETING
value: "Hello from the environment"
- name: DEMO_FAREWELL
value: "Such a sweet sorrow"
`
func TestMerge_map(t *testing.T) {
dest := yaml.MustParse(dest)
src := yaml.MustParse(`
apiVersion: apps/v1
kind: Deployment
metadata:
name: app
labels:
app: java
annotations:
a.b.c: d.e.f
g: h2
k: l
m: n1
`)
result, err := Merge(src, dest)
if !assert.NoError(t, err) {
return
}
actual, err := result.String()
if !assert.NoError(t, err) {
return
}
expected := `
apiVersion: apps/v1
kind: Deployment
metadata:
name: app
labels:
app: java
annotations:
a.b.c: d.e.f
g: h2
i: j
k: l
m: n1
spec:
template:
spec:
containers:
- name: nginx
image: nginx:1.7.9
args:
- c
- a
- b
env:
- name: DEMO_GREETING
value: "Hello from the environment"
- name: DEMO_FAREWELL
value: "Such a sweet sorrow"
`
b, err := filters.FormatInput(bytes.NewBufferString(expected))
if !assert.NoError(t, err) {
return
}
expected = b.String()
b, err = filters.FormatInput(bytes.NewBufferString(actual))
if !assert.NoError(t, err) {
return
}
actual = b.String()
assert.Equal(t, expected, actual)
}
func TestMerge_clear(t *testing.T) {
dest := yaml.MustParse(dest)
src := yaml.MustParse(`
apiVersion: apps/v1
kind: Deployment
metadata:
annotations: null
`)
result, err := Merge(src, dest)
if !assert.NoError(t, err) {
return
}
actual, err := result.String()
if !assert.NoError(t, err) {
return
}
expected := `
apiVersion: apps/v1
kind: Deployment
metadata:
name: app
labels:
app: java
spec:
template:
spec:
containers:
- name: nginx
image: nginx:1.7.9
args:
- c
- a
- b
env:
- name: DEMO_GREETING
value: "Hello from the environment"
- name: DEMO_FAREWELL
value: "Such a sweet sorrow"
`
b, err := filters.FormatInput(bytes.NewBufferString(expected))
if !assert.NoError(t, err) {
return
}
expected = b.String()
b, err = filters.FormatInput(bytes.NewBufferString(actual))
if !assert.NoError(t, err) {
return
}
actual = b.String()
assert.Equal(t, expected, actual)
}
func TestMerge_mapInverse(t *testing.T) {
dest := yaml.MustParse(dest)
src := yaml.MustParse(`
apiVersion: apps/v1
kind: Deployment
metadata:
name: app
labels:
app: java
annotations:
a.b.c: d.e.f
g: h2
k: l
m: n1
`)
result, err := Merge(dest, src)
if !assert.NoError(t, err) {
return
}
actual, err := result.String()
if !assert.NoError(t, err) {
return
}
expected := `
apiVersion: apps/v1
kind: Deployment
metadata:
name: app
labels:
app: java
annotations:
a.b.c: d.e.f
g: h1
i: j
k: l
m: n2
spec:
template:
spec:
containers:
- name: nginx
image: nginx:1.7.9
args:
- c
- a
- b
env:
- name: DEMO_GREETING
value: "Hello from the environment"
- name: DEMO_FAREWELL
value: "Such a sweet sorrow"
`
b, err := filters.FormatInput(bytes.NewBufferString(expected))
if !assert.NoError(t, err) {
return
}
expected = b.String()
b, err = filters.FormatInput(bytes.NewBufferString(actual))
if !assert.NoError(t, err) {
return
}
actual = b.String()
assert.Equal(t, expected, actual)
}
func TestMerge_listElem(t *testing.T) {
dest := yaml.MustParse(dest)
src := yaml.MustParse(`
apiVersion: apps/v1
kind: Deployment
spec:
template:
spec:
containers:
- name: nginx
env:
- name: DEMO_GREETING
value: "New Demo Greeting"
- name: NEW_DEMO_VALUE
value: "Another Env Not In The Dest"
`)
result, err := Merge(src, dest)
if !assert.NoError(t, err) {
return
}
actual, err := result.String()
if !assert.NoError(t, err) {
return
}
expected := `apiVersion: apps/v1
kind: Deployment
metadata:
name: app
labels:
app: java
annotations:
a.b.c: d.e.f
g: h1
i: j
m: n2
spec:
template:
spec:
containers:
- name: nginx
image: nginx:1.7.9
args:
- c
- a
- b
env:
- name: DEMO_GREETING
value: "New Demo Greeting"
- name: DEMO_FAREWELL
value: "Such a sweet sorrow"
- name: NEW_DEMO_VALUE
value: "Another Env Not In The Dest"
`
b, err := filters.FormatInput(bytes.NewBufferString(expected))
if !assert.NoError(t, err) {
return
}
expected = b.String()
b, err = filters.FormatInput(bytes.NewBufferString(actual))
if !assert.NoError(t, err) {
return
}
actual = b.String()
assert.Equal(t, expected, actual)
}
func TestMerge_list(t *testing.T) {
dest := yaml.MustParse(dest)
src := yaml.MustParse(`
apiVersion: apps/v1
kind: Deployment
spec:
template:
spec:
containers:
- name: nginx
args: ['e', 'd', 'f']
`)
result, err := Merge(src, dest)
if !assert.NoError(t, err) {
return
}
actual, err := result.String()
if !assert.NoError(t, err) {
return
}
expected := `apiVersion: apps/v1
kind: Deployment
metadata:
name: app
labels:
app: java
annotations:
a.b.c: d.e.f
g: h1
i: j
m: n2
spec:
template:
spec:
containers:
- name: nginx
image: nginx:1.7.9
args:
- e
- d
- f
env:
- name: DEMO_GREETING
value: "Hello from the environment"
- name: DEMO_FAREWELL
value: "Such a sweet sorrow"
`
b, err := filters.FormatInput(bytes.NewBufferString(expected))
if !assert.NoError(t, err) {
return
}
expected = b.String()
b, err = filters.FormatInput(bytes.NewBufferString(actual))
if !assert.NoError(t, err) {
return
}
actual = b.String()
assert.Equal(t, expected, actual)
}
func TestMerge_commentsKept(t *testing.T) {
actual, err := MergeStrings(`
a:
b:
c: e
`,
`
a:
b:
# header comment
c: d
`)
if !assert.NoError(t, err) {
return
}
assert.Equal(t, `a:
b:
# header comment
c: e
`, actual)
actual, err = MergeStrings(`
a:
b:
c: e
`,
`
a:
b:
c: d
# footer comment
`)
if !assert.NoError(t, err) {
return
}
assert.Equal(t, `a:
b:
c: e
# footer comment
`, actual)
actual, err = MergeStrings(`
a:
b:
c: e
`,
`
a:
b:
c: d # line comment
`)
if !assert.NoError(t, err) {
return
}
assert.Equal(t, `a:
b:
c: e
`, actual)
}
func TestMerge_commentsOverride(t *testing.T) {
actual, err := MergeStrings(`
a:
b:
# header comment
c: e
`,
`
a:
b:
# replace comment
c: d
`)
if !assert.NoError(t, err) {
return
}
assert.Equal(t, `a:
b:
# replace comment
c: e
`, actual)
actual, err = MergeStrings(`
a:
b:
c: e
# footer comment
`,
`
a:
b:
c: d
# replace comment
`)
if !assert.NoError(t, err) {
return
}
assert.Equal(t, `a:
b:
c: e
# replace comment
`, actual)
actual, err = MergeStrings(`
a:
b:
c: e # line comment
`,
`
a:
b:
c: d # replace comment
`)
if !assert.NoError(t, err) {
return
}
assert.Equal(t, `a:
b:
c: e # line comment
`, actual)
actual, err = MergeStrings(`
a:
b:
c: d # line comment
`,
`
a:
b:
c: d # replace comment
`)
if !assert.NoError(t, err) {
return
}
assert.Equal(t, `a:
b:
c: d # line comment
`, actual)
}

View File

@@ -0,0 +1,47 @@
// Copyright 2019 The Kubernetes Authors.
// SPDX-License-Identifier: Apache-2.0
package merge2_test
import (
"bytes"
"strings"
"testing"
"github.com/stretchr/testify/assert"
"sigs.k8s.io/kustomize/kyaml/kio/filters"
. "sigs.k8s.io/kustomize/kyaml/yaml/merge2"
)
var testCases = [][]testCase{scalarTestCases, listTestCases, elementTestCases, mapTestCases}
func TestMerge(t *testing.T) {
for i := range testCases {
for _, tc := range testCases[i] {
actual, err := MergeStrings(tc.source, tc.dest)
if !assert.NoError(t, err, tc.description) {
t.FailNow()
}
e, err := filters.FormatInput(bytes.NewBufferString(tc.expected))
if !assert.NoError(t, err) {
t.FailNow()
}
estr := strings.TrimSpace(e.String())
a, err := filters.FormatInput(bytes.NewBufferString(actual))
if !assert.NoError(t, err) {
t.FailNow()
}
astr := strings.TrimSpace(a.String())
if !assert.Equal(t, estr, astr, tc.description) {
t.FailNow()
}
}
}
}
type testCase struct {
description string
source string
dest string
expected string
}

View File

@@ -0,0 +1,138 @@
// Copyright 2019 The Kubernetes Authors.
// SPDX-License-Identifier: Apache-2.0
package merge2_test
var scalarTestCases = []testCase{
{`replace scalar -- different value in dest`,
`
kind: Deployment
field: value1
`,
`
kind: Deployment
field: value0
`,
`
kind: Deployment
field: value1
`,
},
{`replace scalar -- missing from dest`,
`
kind: Deployment
field: value1
`,
`
kind: Deployment
`,
`
kind: Deployment
field: value1
`,
},
//
// Test Case
//
{`keep scalar -- same value in src and dest`,
`
kind: Deployment
field: value1
`,
`
kind: Deployment
field: value1
`,
`
kind: Deployment
field: value1
`,
},
//
// Test Case
//
{`keep scalar -- unspecified in src`,
`
kind: Deployment
`,
`
kind: Deployment
field: value1
`,
`
kind: Deployment
field: value1
`,
},
//
// Test Case
//
{`remove scalar -- null in src`,
`
kind: Deployment
field: null
`,
`
kind: Deployment
field: value1
`,
`
kind: Deployment
`,
},
//
// Test Case
//
{`remove scalar -- empty in src`,
`
kind: Deployment
field: {}
`,
`
kind: Deployment
field: value1
`,
`
kind: Deployment
field: {}
`,
},
//
// Test Case
//
{`remove scalar -- null in src, missing in dest`,
`
kind: Deployment
field: null
`,
`
kind: Deployment
`,
`
kind: Deployment
`,
},
//
// Test Case
//
{`merge an empty value`,
`
kind: Deployment
field: {}
`,
`
kind: Deployment
`,
`
kind: Deployment
field: {}
`,
},
}