Preserve order when merging.

This commit is contained in:
Jeffrey Regan
2019-06-03 11:22:53 -07:00
committed by jregan
parent c094780aae
commit c63ebbdfc4
9 changed files with 254 additions and 271 deletions

View File

@@ -42,18 +42,14 @@ func (ra *ResAccumulator) Vars() []types.Var {
return ra.varSet.Set()
}
func (ra *ResAccumulator) MergeResourcesWithErrorOnIdCollision(
resources resmap.ResMap) (err error) {
ra.resMap, err = resmap.MergeWithErrorOnIdCollision(
resources, ra.resMap)
return err
func (ra *ResAccumulator) AppendAll(
resources resmap.ResMap) error {
return ra.resMap.AppendAll(resources)
}
func (ra *ResAccumulator) MergeResourcesWithOverride(
resources resmap.ResMap) (err error) {
ra.resMap, err = resmap.MergeWithOverride(
ra.resMap, resources)
return err
func (ra *ResAccumulator) AbsorbAll(
resources resmap.ResMap) error {
return ra.resMap.AbsorbAll(resources)
}
func (ra *ResAccumulator) MergeConfig(
@@ -71,7 +67,7 @@ func (ra *ResAccumulator) MergeVars(incoming []types.Var) error {
}
func (ra *ResAccumulator) MergeAccumulator(other *ResAccumulator) (err error) {
err = ra.MergeResourcesWithErrorOnIdCollision(other.resMap)
err = ra.AppendAll(other.resMap)
if err != nil {
return err
}

View File

@@ -41,7 +41,7 @@ func makeResAccumulator() (*ResAccumulator, *resource.Factory, error) {
}
rf := resource.NewFactory(
kunstruct.NewKunstructuredFactoryImpl())
err = ra.MergeResourcesWithErrorOnIdCollision(
err = ra.AppendAll(
resmap.FromMap(map[resid.ResId]*resource.Resource{
resid.NewResId(
gvk.Gvk{Group: "apps", Version: "v1", Kind: "Deployment"},
@@ -188,7 +188,7 @@ func TestResolveVarsVarNeedsDisambiguation(t *testing.T) {
}),
})
err = ra.MergeResourcesWithErrorOnIdCollision(rm0)
err = ra.AppendAll(rm0)
if err != nil {
t.Fatalf("unexpected err: %v", err)
}

View File

@@ -153,7 +153,7 @@ func (o *Options) emitResources(
if o.outputPath != "" && fSys.IsDir(o.outputPath) {
return writeIndividualFiles(fSys, o.outputPath, m)
}
res, err := m.EncodeAsYaml()
res, err := m.AsYaml(resmap.LegacySort)
if err != nil {
return err
}

View File

@@ -193,7 +193,7 @@ func (th *KustTestHarness) AssertActualEqualsExpected(
if len(expected) > 0 && expected[0] == 10 {
expected = expected[1:]
}
actual, err := m.EncodeAsYaml()
actual, err := m.AsYaml(resmap.LegacySort)
if err != nil {
th.t.Fatalf("Unexpected err: %v", err)
}

View File

@@ -140,7 +140,7 @@ func (p *ExecPlugin) Transform(rm resmap.ResMap) error {
}
// encode the ResMap so it can be fed to the plugin
resources, err := inputRM.EncodeAsYaml()
resources, err := inputRM.AsYaml(resmap.Identity)
if err != nil {
return err
}

View File

@@ -84,17 +84,34 @@ type ResMap interface {
// Error on Id collision.
AppendWithId(resid.ResId, *resource.Resource) error
// AsMap returns ResId, *Resource pairs in
// arbitrary order via a map.
// AppendAll appends another ResMap to self,
// failing on any Id collision.
AppendAll(ResMap) error
// AbsorbAll appends, replaces or merges the contents
// of another ResMap into self,
// allowing and sometimes demanding ID collisions.
// A collision would be demanded, say, when a generated
// ConfigMap has the "replace" option in its generation
// instructions, meaning it _must_ replace
// something in the known set of resources.
// If a resource id for resource X is found to already
// be in self, then the behavior field for X must
// be BehaviorMerge or BehaviorReplace. If X is not in
// self, then its behavior _cannot_ be merge or replace.
AbsorbAll(ResMap) error
// AsMap returns (ResId, *Resource) pairs in
// arbitrary order via a generated map.
// The map is discardable, and edits to map structure
// have no impact on the ResMap.
// The Ids are copies, but the resources are pointers,
// so the resources themselves can be modified.
AsMap() map[resid.ResId]*resource.Resource
// EncodeAsYaml emits the resources as YAML in a byte slice.
// Resources are separated by `---`.
EncodeAsYaml() ([]byte, error)
// AsYaml returns the yaml form of resources, after twiddling.
// Certainly nobody will abuse the twiddler.
AsYaml(f ResTwiddler) ([]byte, error)
// Gets the resource with the given Id, else nil.
GetById(resid.ResId) *resource.Resource
@@ -350,33 +367,42 @@ func (m *resWrangler) GetMatchingIds(matches IdMatcher) []resid.ResId {
return result
}
// EncodeAsYaml implements ResMap.
func (m *resWrangler) EncodeAsYaml() ([]byte, error) {
// TODO: should be able to suppress this sort
// and rely on ordering as specified in the ResMap
// internal rList.
// Identity returns Resources as is.
func Identity(m ResMap) []*resource.Resource {
return m.Resources()
}
// LegacySort return Resources in a particular order.
func LegacySort(m ResMap) []*resource.Resource {
resources := make([]*resource.Resource, m.Size())
ids := m.AllIds()
sort.Sort(IdSlice(ids))
for i, id := range ids {
resources[i] = m.GetById(id)
}
return resources
}
type ResTwiddler func(ResMap) []*resource.Resource
// AsYaml implements ResMap.
func (m *resWrangler) AsYaml(twiddle ResTwiddler) ([]byte, error) {
firstObj := true
var b []byte
buf := bytes.NewBuffer(b)
for _, id := range ids {
obj := m.GetById(id)
out, err := yaml.Marshal(obj.Map())
for _, res := range twiddle(m) {
out, err := yaml.Marshal(res.Map())
if err != nil {
return nil, err
}
if firstObj {
firstObj = false
} else {
_, err = buf.WriteString("---\n")
if err != nil {
if _, err = buf.WriteString("---\n"); err != nil {
return nil, err
}
}
_, err = buf.Write(out)
if err != nil {
if _, err = buf.Write(out); err != nil {
return nil, err
}
}
@@ -387,7 +413,7 @@ func (m *resWrangler) EncodeAsYaml() ([]byte, error) {
func (m *resWrangler) ErrorIfNotEqual(other ResMap) error {
m2, ok := other.(*resWrangler)
if !ok {
panic(fmt.Errorf("bad cast to resmapImpl"))
panic(fmt.Errorf("bad cast"))
}
if m.Size() != m2.Size() {
return fmt.Errorf(
@@ -462,86 +488,93 @@ func (m *resWrangler) ResourcesThatCouldReference(inputId resid.ResId) ResMap {
return result
}
// MergeWithErrorOnIdCollision combines multiple ResMap instances, failing on
// key collision and skipping nil maps.
// If all of the maps are nil, an empty ResMap is returned.
func MergeWithErrorOnIdCollision(maps ...ResMap) (ResMap, error) {
result := New()
for _, m := range maps {
if m == nil {
continue
// AppendAll implements ResMap.
func (m *resWrangler) AppendAll(other ResMap) error {
if other == nil {
return nil
}
w2, ok := other.(*resWrangler)
if !ok {
panic(fmt.Errorf("bad cast"))
}
for i, res := range w2.Resources() {
id, err := w2.idMappingToIndex(i)
if err != nil {
panic("map is unrecoverably corrupted; " + err.Error())
}
for id, res := range m.AsMap() {
err := result.AppendWithId(id, res)
if err != nil {
return nil, err
}
err = m.AppendWithId(id, res)
if err != nil {
return err
}
}
return result, nil
return nil
}
// MergeWithOverride combines multiple ResMap instances, allowing and sometimes
// demanding certain collisions and skipping nil maps.
// A collision would be demanded, say, when a generated ConfigMap has the
// "replace" option in its generation instructions, meaning it is supposed
// to replace something from the raw resources list.
// If all of the maps are nil, an empty ResMap is returned.
// When looping over the instances to combine them, if a resource id for
// resource X is found to be already in the combined map, then the behavior
// field for X must be BehaviorMerge or BehaviorReplace. If X is not in the
// map, then it's behavior cannot be merge or replace.
// nolint: gocyclo
func MergeWithOverride(maps ...ResMap) (ResMap, error) {
if len(maps) == 0 {
return New(), nil
// AbsorbAll implements ResMap.
func (m *resWrangler) AbsorbAll(other ResMap) error {
if other == nil {
return nil
}
result := maps[0]
if result == nil {
result = New()
w2, ok := other.(*resWrangler)
if !ok {
panic(fmt.Errorf("bad cast"))
}
for _, m := range maps[1:] {
if m == nil {
continue
for i, r := range w2.Resources() {
id, err := w2.idMappingToIndex(i)
if err != nil {
panic("map is unrecoverably corrupted; " + err.Error())
}
for id, r := range m.AsMap() {
matchedId := result.GetMatchingIds(id.GvknEquals)
if len(matchedId) == 1 {
id = matchedId[0]
old := result.GetById(id)
if old == nil {
return nil, fmt.Errorf("id lookup failure")
}
switch r.Behavior() {
case types.BehaviorReplace:
r.Replace(old)
err := result.ReplaceResource(id, r)
if err != nil {
return nil, err
}
case types.BehaviorMerge:
r.Merge(old)
err := result.ReplaceResource(id, r)
if err != nil {
return nil, err
}
default:
return nil, fmt.Errorf("id %#v exists; must merge or replace", id)
}
} else if len(matchedId) == 0 {
switch r.Behavior() {
case types.BehaviorMerge, types.BehaviorReplace:
return nil, fmt.Errorf("id %#v does not exist; cannot merge or replace", id)
default:
err := result.AppendWithId(id, r)
if err != nil {
return nil, err
}
}
} else {
return nil, fmt.Errorf("merge conflict, found multiple objects %v the Resmap %v can merge into", matchedId, id)
err = m.appendReplaceOrMerge(id, r)
if err != nil {
return err
}
}
return nil
}
func (m *resWrangler) appendReplaceOrMerge(
idForRes resid.ResId, res *resource.Resource) error {
matchedId := m.GetMatchingIds(idForRes.GvknEquals)
switch len(matchedId) {
case 0:
switch res.Behavior() {
case types.BehaviorMerge, types.BehaviorReplace:
return fmt.Errorf(
"id %#v does not exist; cannot merge or replace", idForRes)
default:
// presumably types.BehaviorCreate
err := m.AppendWithId(idForRes, res)
if err != nil {
return err
}
}
case 1:
mId := matchedId[0]
old := m.GetById(mId)
if old == nil {
return fmt.Errorf("id lookup failure")
}
switch res.Behavior() {
case types.BehaviorReplace:
res.Replace(old)
err := m.ReplaceResource(mId, res)
if err != nil {
return err
}
case types.BehaviorMerge:
res.Merge(old)
err := m.ReplaceResource(mId, res)
if err != nil {
return err
}
default:
return fmt.Errorf(
"id %#v exists; must merge or replace", idForRes)
}
default:
return fmt.Errorf(
"found multiple objects %v that could accept merge of %v",
matchedId, idForRes)
}
return result, nil
return nil
}

View File

@@ -17,7 +17,6 @@ import (
)
var deploy = gvk.Gvk{Group: "apps", Version: "v1", Kind: "Deployment"}
var statefulset = gvk.Gvk{Group: "apps", Version: "v1", Kind: "StatefulSet"}
var rf = resource.NewFactory(
kunstruct.NewKunstructuredFactoryImpl())
var rmF = NewFactory(rf)
@@ -161,25 +160,24 @@ kind: ConfigMap
metadata:
name: cm2
`)
input := FromMap(map[resid.ResId]*resource.Resource{
resid.NewResId(cmap, "cm1"): rf.FromMap(
map[string]interface{}{
"apiVersion": "v1",
"kind": "ConfigMap",
"metadata": map[string]interface{}{
"name": "cm1",
},
}),
resid.NewResId(cmap, "cm2"): rf.FromMap(
map[string]interface{}{
"apiVersion": "v1",
"kind": "ConfigMap",
"metadata": map[string]interface{}{
"name": "cm2",
},
}),
})
out, err := input.EncodeAsYaml()
input := New()
input.Append(rf.FromMap(
map[string]interface{}{
"apiVersion": "v1",
"kind": "ConfigMap",
"metadata": map[string]interface{}{
"name": "cm1",
},
}))
input.Append(rf.FromMap(
map[string]interface{}{
"apiVersion": "v1",
"kind": "ConfigMap",
"metadata": map[string]interface{}{
"name": "cm2",
},
}))
out, err := input.AsYaml(Identity)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
@@ -581,172 +579,127 @@ func TestErrorIfNotEqual(t *testing.T) {
if err == nil {
t.Fatalf("%v should not equal %v %v", rm1, rm2, err)
}
}
func TestMergeWithoutOverride(t *testing.T) {
input1 := FromMap(map[resid.ResId]*resource.Resource{
resid.NewResId(deploy, "deploy1"): rf.FromMap(
map[string]interface{}{
"apiVersion": "apps/v1",
"kind": "Deployment",
"metadata": map[string]interface{}{
"name": "foo-deploy1",
},
}),
})
input2 := FromMap(map[resid.ResId]*resource.Resource{
resid.NewResId(statefulset, "stateful1"): rf.FromMap(
map[string]interface{}{
"apiVersion": "apps/v1",
"kind": "StatefulSet",
"metadata": map[string]interface{}{
"name": "bar-stateful",
},
}),
})
input := []ResMap{input1, input2}
expected := FromMap(map[resid.ResId]*resource.Resource{
resid.NewResId(deploy, "deploy1"): rf.FromMap(
map[string]interface{}{
"apiVersion": "apps/v1",
"kind": "Deployment",
"metadata": map[string]interface{}{
"name": "foo-deploy1",
},
}),
resid.NewResId(statefulset, "stateful1"): rf.FromMap(
map[string]interface{}{
"apiVersion": "apps/v1",
"kind": "StatefulSet",
"metadata": map[string]interface{}{
"name": "bar-stateful",
},
}),
})
merged, err := MergeWithErrorOnIdCollision(input...)
if err != nil {
func TestAppendAll(t *testing.T) {
r1 := rf.FromMap(
map[string]interface{}{
"apiVersion": "apps/v1",
"kind": "Deployment",
"metadata": map[string]interface{}{
"name": "foo-deploy1",
},
})
input1 := rmF.FromResource(r1)
r2 := rf.FromMap(
map[string]interface{}{
"apiVersion": "apps/v1",
"kind": "StatefulSet",
"metadata": map[string]interface{}{
"name": "bar-stateful",
},
})
input2 := rmF.FromResource(r2)
expected := New()
expected.Append(r1)
expected.Append(r2)
if err := input1.AppendAll(input2); err != nil {
t.Fatalf("unexpected error: %v", err)
}
err = expected.ErrorIfNotEqual(merged)
if err != nil {
t.Fatalf("%#v doesn't equal expected %#v", merged, expected)
if err := expected.ErrorIfNotEqual(input1); err != nil {
input1.Debug("1")
expected.Debug("ex")
t.Fatalf("%#v doesn't equal expected %#v", input1, expected)
}
input3 := []ResMap{merged, nil}
merged1, err := MergeWithErrorOnIdCollision(input3...)
if err != nil {
if err := input1.AppendAll(nil); err != nil {
t.Fatalf("unexpected error: %v", err)
}
err = expected.ErrorIfNotEqual(merged1)
if err != nil {
t.Fatal(err)
}
input4 := []ResMap{nil, merged}
merged2, err := MergeWithErrorOnIdCollision(input4...)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
err = expected.ErrorIfNotEqual(merged2)
if err != nil {
t.Fatal(err)
if err := expected.ErrorIfNotEqual(input1); err != nil {
t.Fatalf("%#v doesn't equal expected %#v", input1, expected)
}
}
func generateMergeFixtures(b types.GenerationBehavior) []ResMap {
input1 := FromMap(map[resid.ResId]*resource.Resource{
resid.NewResId(cmap, "cmap"): rf.FromMapAndOption(
map[string]interface{}{
"apiVersion": "apps/v1",
"kind": "ConfigMap",
"metadata": map[string]interface{}{
"name": "cmap",
},
"data": map[string]interface{}{
"a": "x",
"b": "y",
},
}, &types.GeneratorArgs{
Behavior: "create",
}, nil),
})
input2 := FromMap(map[resid.ResId]*resource.Resource{
resid.NewResId(cmap, "cmap"): rf.FromMapAndOption(
map[string]interface{}{
"apiVersion": "apps/v1",
"kind": "ConfigMap",
"metadata": map[string]interface{}{
"name": "cmap",
},
"data": map[string]interface{}{
"a": "u",
"b": "v",
"c": "w",
},
}, &types.GeneratorArgs{
Behavior: b.String(),
}, nil),
})
return []ResMap{input1, input2}
func makeMap1() ResMap {
return rmF.FromResource(rf.FromMapAndOption(
map[string]interface{}{
"apiVersion": "apps/v1",
"kind": "ConfigMap",
"metadata": map[string]interface{}{
"name": "cmap",
},
"data": map[string]interface{}{
"a": "x",
"b": "y",
},
}, &types.GeneratorArgs{
Behavior: "create",
}, nil))
}
func TestMergeWithOverride(t *testing.T) {
expected := FromMap(map[resid.ResId]*resource.Resource{
resid.NewResId(cmap, "cmap"): rf.FromMapAndOption(
map[string]interface{}{
"apiVersion": "apps/v1",
"kind": "ConfigMap",
"metadata": map[string]interface{}{
"annotations": map[string]interface{}{},
"labels": map[string]interface{}{},
"name": "cmap",
},
"data": map[string]interface{}{
"a": "u",
"b": "v",
"c": "w",
},
}, &types.GeneratorArgs{
Behavior: "create",
}, nil),
})
merged, err := MergeWithOverride(generateMergeFixtures(types.BehaviorMerge)...)
if err != nil {
func makeMap2(b types.GenerationBehavior) ResMap {
return rmF.FromResource(rf.FromMapAndOption(
map[string]interface{}{
"apiVersion": "apps/v1",
"kind": "ConfigMap",
"metadata": map[string]interface{}{
"name": "cmap",
},
"data": map[string]interface{}{
"a": "u",
"b": "v",
"c": "w",
},
}, &types.GeneratorArgs{
Behavior: b.String(),
}, nil))
}
func TestAbsorbAll(t *testing.T) {
expected := rmF.FromResource(rf.FromMapAndOption(
map[string]interface{}{
"apiVersion": "apps/v1",
"kind": "ConfigMap",
"metadata": map[string]interface{}{
"annotations": map[string]interface{}{},
"labels": map[string]interface{}{},
"name": "cmap",
},
"data": map[string]interface{}{
"a": "u",
"b": "v",
"c": "w",
},
}, &types.GeneratorArgs{
Behavior: "create",
}, nil))
w := makeMap1()
if err := w.AbsorbAll(makeMap2(types.BehaviorMerge)); err != nil {
t.Fatalf("unexpected error: %v", err)
}
err = expected.ErrorIfNotEqual(merged)
if err != nil {
if err := expected.ErrorIfNotEqual(w); err != nil {
t.Fatal(err)
}
input3 := []ResMap{merged, nil}
merged1, err := MergeWithOverride(input3...)
if err != nil {
w = makeMap1()
if err := w.AbsorbAll(nil); err != nil {
t.Fatalf("unexpected error: %v", err)
}
err = expected.ErrorIfNotEqual(merged1)
if err != nil {
if err := w.ErrorIfNotEqual(makeMap1()); err != nil {
t.Fatal(err)
}
input4 := []ResMap{nil, merged}
merged2, err := MergeWithOverride(input4...)
if err != nil {
w = makeMap1()
w2 := makeMap2(types.BehaviorReplace)
if err := w.AbsorbAll(w2); err != nil {
t.Fatalf("unexpected error: %v", err)
}
err = expected.ErrorIfNotEqual(merged2)
if err != nil {
if err := w2.ErrorIfNotEqual(w); err != nil {
t.Fatal(err)
}
inputs := generateMergeFixtures(types.BehaviorReplace)
replaced, err := MergeWithOverride(inputs...)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
expectedReplaced := inputs[1]
err = expectedReplaced.ErrorIfNotEqual(replaced)
if err != nil {
t.Fatal(err)
}
_, err = MergeWithOverride(generateMergeFixtures(types.BehaviorUnspecified)...)
w = makeMap1()
w2 = makeMap2(types.BehaviorUnspecified)
err := w.AbsorbAll(w2)
if err == nil {
t.Fatal("Merging with GenerationBehavior BehaviorUnspecified should return an error but does not")
t.Fatalf("expected error with unspecified behavior")
}
}

View File

@@ -263,7 +263,7 @@ func (kt *KustTarget) runGenerators(
return err
}
// The legacy generators allow override.
err = ra.MergeResourcesWithOverride(resMap)
err = ra.AbsorbAll(resMap)
if err != nil {
return errors.Wrapf(err, "merging from generator %v", g)
}
@@ -277,7 +277,7 @@ func (kt *KustTarget) runGenerators(
if err != nil {
return err
}
err = ra.MergeResourcesWithErrorOnIdCollision(resMap)
err = ra.AppendAll(resMap)
if err != nil {
return errors.Wrapf(err, "merging from generator %v", g)
}
@@ -380,7 +380,7 @@ func (kt *KustTarget) accumulateFile(
if err != nil {
return errors.Wrapf(err, "accumulating resources from '%s'", path)
}
err = ra.MergeResourcesWithErrorOnIdCollision(resources)
err = ra.AppendAll(resources)
if err != nil {
return errors.Wrapf(err, "merging resources from '%s'", path)
}

View File

@@ -8,6 +8,7 @@ import (
"testing"
"sigs.k8s.io/kustomize/pkg/kusttest"
"sigs.k8s.io/kustomize/pkg/resmap"
"sigs.k8s.io/kustomize/plugin"
)
@@ -32,7 +33,7 @@ kind: PrintWorkDir
metadata:
name: whatever
`)
a, err := m.EncodeAsYaml()
a, err := m.AsYaml(resmap.Identity)
if err != nil {
t.Error(err)
}