From 4f21d600456ee9e577f8874cbd859560066f01d1 Mon Sep 17 00:00:00 2001 From: natasha41575 Date: Wed, 29 Dec 2021 15:13:04 -0800 Subject: [PATCH] helper methods, types, and unit tests for transformer annotations --- api/internal/target/kusttarget.go | 2 +- api/internal/utils/annotations.go | 5 +- ...adata_test.go => originannotation_test.go} | 1 - api/resmap/resmap.go | 23 +++- api/resmap/reswrangler.go | 44 ++++++ api/resmap/reswrangler_test.go | 89 ++++++++++++ api/resource/origin.go | 26 +++- api/resource/origin_test.go | 79 ++++++++++- api/resource/resource.go | 47 ++++++- api/resource/resource_test.go | 127 ++++++++++++++++++ 10 files changed, 428 insertions(+), 15 deletions(-) rename api/krusty/{buildmetadata_test.go => originannotation_test.go} (99%) diff --git a/api/internal/target/kusttarget.go b/api/internal/target/kusttarget.go index 30dbbde89..06078b3af 100644 --- a/api/internal/target/kusttarget.go +++ b/api/internal/target/kusttarget.go @@ -497,7 +497,7 @@ func (kt *KustTarget) accumulateFile( if err != nil { return errors.Wrapf(err, "cannot add path annotation for '%s'", path) } - err = resources.AnnotateAll(utils.OriginAnnotation, originAnno) + err = resources.AnnotateAll(utils.OriginAnnotationKey, originAnno) if err != nil || originAnno == "" { return errors.Wrapf(err, "cannot add path annotation for '%s'", path) } diff --git a/api/internal/utils/annotations.go b/api/internal/utils/annotations.go index 6c9cecba6..3cc5d4140 100644 --- a/api/internal/utils/annotations.go +++ b/api/internal/utils/annotations.go @@ -3,6 +3,7 @@ package utils import "sigs.k8s.io/kustomize/api/konfig" const ( + // build annotations BuildAnnotationPreviousKinds = konfig.ConfigAnnoDomain + "/previousKinds" BuildAnnotationPreviousNames = konfig.ConfigAnnoDomain + "/previousNames" BuildAnnotationPrefixes = konfig.ConfigAnnoDomain + "/prefixes" @@ -17,7 +18,9 @@ const ( BuildAnnotationAllowNameChange = konfig.ConfigAnnoDomain + "/allowNameChange" BuildAnnotationAllowKindChange = konfig.ConfigAnnoDomain + "/allowKindChange" - OriginAnnotation = "config.kubernetes.io/origin" + // for keeping track of origin and transformer data + OriginAnnotationKey = "config.kubernetes.io/origin" + TransformerAnnotationKey = "config.kubernetes.io/transformations" Enabled = "enabled" ) diff --git a/api/krusty/buildmetadata_test.go b/api/krusty/originannotation_test.go similarity index 99% rename from api/krusty/buildmetadata_test.go rename to api/krusty/originannotation_test.go index 1ad8c475f..a293ff70f 100644 --- a/api/krusty/buildmetadata_test.go +++ b/api/krusty/originannotation_test.go @@ -11,7 +11,6 @@ import ( "github.com/stretchr/testify/assert" "sigs.k8s.io/kustomize/api/internal/utils" "sigs.k8s.io/kustomize/api/krusty" - _ "sigs.k8s.io/kustomize/api/krusty" kusttest_test "sigs.k8s.io/kustomize/api/testutils/kusttest" "sigs.k8s.io/kustomize/kyaml/filesys" ) diff --git a/api/resmap/resmap.go b/api/resmap/resmap.go index e7bacc33d..ae89966af 100644 --- a/api/resmap/resmap.go +++ b/api/resmap/resmap.go @@ -21,6 +21,13 @@ type Transformer interface { Transform(m ResMap) error } +// A TransformerWithProperties contains a Transformer and stores +// some of its properties +type TransformerWithProperties struct { + Transformer + Origin *resource.Origin +} + // A Generator creates an instance of ResMap. type Generator interface { Generate() (ResMap, error) @@ -144,10 +151,24 @@ type ResMap interface { AbsorbAll(ResMap) error // AddOriginAnnotation will add the provided origin as - // an annotation to all resources in the Resmap, if + // an origin annotation to all resources in the ResMap, if // the origin is not nil. AddOriginAnnotation(origin *resource.Origin) error + // RemoveOriginAnnotation will remove the origin annotation + // from all resources in the ResMap + RemoveOriginAnnotations() error + + // AddTransformerAnnotation will add the provided origin as + // an origin annotation if the resource doesn't have one; a + // transformer annotation otherwise; to all resources in + // ResMap + AddTransformerAnnotation(origin *resource.Origin) error + + // RemoveTransformerAnnotation will remove the transformer annotation + // from all resources in the ResMap + RemoveTransformerAnnotations() error + // AnnotateAll annotates all resources in the ResMap with // the provided key value pair. AnnotateAll(key string, value string) error diff --git a/api/resmap/reswrangler.go b/api/resmap/reswrangler.go index 9f634dc37..5eb4008dc 100644 --- a/api/resmap/reswrangler.go +++ b/api/resmap/reswrangler.go @@ -503,6 +503,50 @@ func (m *resWrangler) AddOriginAnnotation(origin *resource.Origin) error { return nil } +// RemoveOriginAnnotation implements ResMap +func (m *resWrangler) RemoveOriginAnnotations() error { + for _, res := range m.rList { + if err := res.SetOrigin(nil); err != nil { + return err + } + } + return nil +} + +// AddTransformerAnnotation implements ResMap +func (m *resWrangler) AddTransformerAnnotation(origin *resource.Origin) error { + for _, res := range m.rList { + or, err := res.GetOrigin() + if err != nil { + return err + } + if or == nil { + // the resource does not have an origin annotation, so + // we assume that the transformer generated the resource + // rather than modifying it + err = res.SetOrigin(origin) + } else { + // the resource already has an origin annotation, so we + // record the provided origin as a transformation + err = res.AddTransformation(origin) + } + if err != nil { + return err + } + } + return nil +} + +// RemoveTransformerAnnotations implements ResMap +func (m *resWrangler) RemoveTransformerAnnotations() error { + for _, res := range m.rList { + if err := res.ClearTransformations(); err != nil { + return err + } + } + return nil +} + func (m *resWrangler) appendReplaceOrMerge(res *resource.Resource) error { id := res.CurId() matches := m.GetMatchingResourcesByAnyId(id.Equals) diff --git a/api/resmap/reswrangler_test.go b/api/resmap/reswrangler_test.go index a47dddab7..92416df6e 100644 --- a/api/resmap/reswrangler_test.go +++ b/api/resmap/reswrangler_test.go @@ -26,6 +26,34 @@ import ( var depProvider = provider.NewDefaultDepProvider() var rf = depProvider.GetResourceFactory() var rmF = NewFactory(rf) +var origin1 = &resource.Origin{ + Repo: "github.com/myrepo", + Ref: "master", + ConfiguredIn: "config.yaml", + ConfiguredBy: yaml.ResourceIdentifier{ + TypeMeta: yaml.TypeMeta{ + APIVersion: "builtin", + Kind: "Generator", + }, + NameMeta: yaml.NameMeta{ + Name: "my-name", + Namespace: "my-namespace", + }, + }, +} +var origin2 = &resource.Origin{ + ConfiguredIn: "../base/config.yaml", + ConfiguredBy: yaml.ResourceIdentifier{ + TypeMeta: yaml.TypeMeta{ + APIVersion: "builtin", + Kind: "Generator", + }, + NameMeta: yaml.NameMeta{ + Name: "my-name", + Namespace: "my-namespace", + }, + }, +} func doAppend(t *testing.T, w ResMap, r *resource.Resource) { err := w.Append(r) @@ -1527,6 +1555,67 @@ $patch: delete } } +func TestOriginAnnotations(t *testing.T) { + w := New() + for i := 0; i < 3; i++ { + assert.NoError(t, w.Append(makeCm(i))) + } + // this should add an origin annotation to every resource + assert.NoError(t, w.AddOriginAnnotation(origin1)) + resources := w.Resources() + for _, res := range resources { + or, err := res.GetOrigin() + assert.NoError(t, err) + assert.Equal(t, origin1, or) + } + // this should not overwrite the existing origin annotations + assert.NoError(t, w.AddOriginAnnotation(origin2)) + for _, res := range resources { + or, err := res.GetOrigin() + assert.NoError(t, err) + assert.Equal(t, origin1, or) + } + // this should remove origin annotations from all resources + assert.NoError(t, w.RemoveOriginAnnotations()) + for _, res := range resources { + or, err := res.GetOrigin() + assert.NoError(t, err) + assert.Nil(t, or) + } +} + +func TestTransformerAnnotations(t *testing.T) { + w := New() + for i := 0; i < 3; i++ { + assert.NoError(t, w.Append(makeCm(i))) + } + // this should add an origin annotation to every resource + assert.NoError(t, w.AddTransformerAnnotation(origin1)) + resources := w.Resources() + for _, res := range resources { + or, err := res.GetOrigin() + assert.NoError(t, err) + assert.Equal(t, origin1, or) + } + // this should add a transformer annotation to every resource + assert.NoError(t, w.AddTransformerAnnotation(origin2)) + for _, res := range resources { + or, err := res.GetOrigin() + assert.NoError(t, err) + assert.Equal(t, origin1, or) + tr, err := res.GetTransformations() + assert.NoError(t, err) + assert.Equal(t, resource.Transformations{origin2}, tr) + } + // remove transformer annotations from all resources + assert.NoError(t, w.RemoveTransformerAnnotations()) + for _, res := range resources { + tr, err := res.GetTransformations() + assert.NoError(t, err) + assert.Nil(t, tr) + } +} + // baseResource produces a base object which used to test // patch transformation // Also the structure is matching the Deployment syntax diff --git a/api/resource/origin.go b/api/resource/origin.go index 3a835ca39..d5ac3a1a4 100644 --- a/api/resource/origin.go +++ b/api/resource/origin.go @@ -11,30 +11,31 @@ import ( kyaml "sigs.k8s.io/kustomize/kyaml/yaml" ) -// Origin retains information about where resources in the output -// of `kustomize build` originated from +// Origin retains information about the origin of resources and transformer configs +// that contributed to the output of `kustomize build` type Origin struct { // Path is the path to the resource. If a local resource, this path is // rooted from the directory upon which `kustomize build` was invoked. If a // remote resource, this path is rooted from the root of the remote repo. Path string `json:"path,omitempty" yaml:"path,omitempty"` - // Repo is the remote repository that the resource originated from if it is + // Repo is the remote repository that the resource or transformer originated from if it is // not from a local file Repo string `json:"repo,omitempty" yaml:"repo,omitempty"` - // Ref is the ref of the remote repository that the resource originated from + // Ref is the ref of the remote repository that the resource or transformer originated from // if it is not from a local file Ref string `json:"ref,omitempty" yaml:"ref,omitempty"` // The following fields only apply to resources that have been - // generated by fields other than the `resources` field. + // generated by fields other than the `resources` field, or to transformer + // configs. - // ConfiguredIn is the file path to the generator config that created the + // ConfiguredIn is the file path to the generator or transformer config that created the // resource ConfiguredIn string `json:"configuredIn,omitempty" yaml:"configuredIn,omitempty"` - // ConfiguredBy is the ObjectReference of the generator config + // ConfiguredBy is the ObjectReference of the generator or transformer config ConfiguredBy kyaml.ResourceIdentifier `json:"configuredBy,omitempty" yaml:"configuredBy,omitempty"` } @@ -67,6 +68,17 @@ func (origin *Origin) String() (string, error) { return string(anno), err } +// Transformations is a list of Origin +type Transformations []*Origin + +// String returns a string version of Transformations +func (transformations *Transformations) String() (string, error) { + anno, err := kyaml.Marshal(transformations) + return string(anno), err +} + +// OriginFromCustomPlugin takes a custom plugin defined as a resource +// and returns an origin object to describe it func OriginFromCustomPlugin(res *Resource) (*Origin, error) { origin, err := res.GetOrigin() if err != nil { diff --git a/api/resource/origin_test.go b/api/resource/origin_test.go index 43d5223ad..46ae2b8d0 100644 --- a/api/resource/origin_test.go +++ b/api/resource/origin_test.go @@ -8,6 +8,7 @@ import ( "github.com/stretchr/testify/assert" . "sigs.k8s.io/kustomize/api/resource" + kyaml "sigs.k8s.io/kustomize/kyaml/yaml" ) func TestOriginAppend(t *testing.T) { @@ -78,6 +79,82 @@ repo: github.com/kubernetes-sigs/kustomize/examples/multibases/dev/ for _, test := range tests { actual, err := test.in.String() assert.NoError(t, err) - assert.Equal(t, actual, test.expected) + assert.Equal(t, test.expected, actual) + } +} + +func TestTransformationsString(t *testing.T) { + origin1 := &Origin{ + Repo: "github.com/myrepo", + Ref: "master", + ConfiguredIn: "config.yaml", + ConfiguredBy: kyaml.ResourceIdentifier{ + TypeMeta: kyaml.TypeMeta{ + APIVersion: "builtin", + Kind: "Generator", + }, + NameMeta: kyaml.NameMeta{ + Name: "my-name", + Namespace: "my-namespace", + }, + }, + } + origin2 := &Origin{ + ConfiguredIn: "../base/config.yaml", + ConfiguredBy: kyaml.ResourceIdentifier{ + TypeMeta: kyaml.TypeMeta{ + APIVersion: "builtin", + Kind: "Generator", + }, + NameMeta: kyaml.NameMeta{ + Name: "my-name", + Namespace: "my-namespace", + }, + }, + } + tests := []struct { + in Transformations + expected string + }{ + { + in: Transformations{origin1}, + expected: `- repo: github.com/myrepo + ref: master + configuredIn: config.yaml + configuredBy: + apiVersion: builtin + kind: Generator + name: my-name + namespace: my-namespace +`, + }, + { + in: Transformations{origin1, origin2}, + expected: `- repo: github.com/myrepo + ref: master + configuredIn: config.yaml + configuredBy: + apiVersion: builtin + kind: Generator + name: my-name + namespace: my-namespace +- configuredIn: ../base/config.yaml + configuredBy: + apiVersion: builtin + kind: Generator + name: my-name + namespace: my-namespace +`, + }, + { + in: Transformations{}, + expected: `[] +`, + }, + } + for _, test := range tests { + actual, err := test.in.String() + assert.NoError(t, err) + assert.Equal(t, test.expected, actual) } } diff --git a/api/resource/resource.go b/api/resource/resource.go index bd6bb6c81..faf882045 100644 --- a/api/resource/resource.go +++ b/api/resource/resource.go @@ -69,7 +69,7 @@ func (r *Resource) SetGvk(gvk resid.Gvk) { func (r *Resource) GetOrigin() (*Origin, error) { annotations := r.GetAnnotations() - originAnnotations, ok := annotations[utils.OriginAnnotation] + originAnnotations, ok := annotations[utils.OriginAnnotationKey] if !ok { return nil, nil } @@ -82,11 +82,52 @@ func (r *Resource) GetOrigin() (*Origin, error) { func (r *Resource) SetOrigin(origin *Origin) error { annotations := r.GetAnnotations() - originStr, err := origin.String() + if origin == nil { + delete(annotations, utils.OriginAnnotationKey) + } else { + originStr, err := origin.String() + if err != nil { + return err + } + annotations[utils.OriginAnnotationKey] = originStr + } + return r.SetAnnotations(annotations) +} + +func (r *Resource) GetTransformations() (Transformations, error) { + annotations := r.GetAnnotations() + transformerAnnotations, ok := annotations[utils.TransformerAnnotationKey] + if !ok { + return nil, nil + } + var transformations Transformations + if err := yaml.Unmarshal([]byte(transformerAnnotations), &transformations); err != nil { + return nil, err + } + return transformations, nil +} + +func (r *Resource) AddTransformation(origin *Origin) error { + annotations := r.GetAnnotations() + transformations, err := r.GetTransformations() if err != nil { return err } - annotations[utils.OriginAnnotation] = originStr + if transformations == nil { + transformations = Transformations{} + } + transformations = append(transformations, origin) + transformationStr, err := transformations.String() + if err != nil { + return err + } + annotations[utils.TransformerAnnotationKey] = transformationStr + return r.SetAnnotations(annotations) +} + +func (r *Resource) ClearTransformations() error { + annotations := r.GetAnnotations() + delete(annotations, utils.TransformerAnnotationKey) return r.SetAnnotations(annotations) } diff --git a/api/resource/resource_test.go b/api/resource/resource_test.go index 89668c203..199d7b8e7 100644 --- a/api/resource/resource_test.go +++ b/api/resource/resource_test.go @@ -14,6 +14,7 @@ import ( . "sigs.k8s.io/kustomize/api/resource" "sigs.k8s.io/kustomize/api/types" "sigs.k8s.io/kustomize/kyaml/resid" + kyaml "sigs.k8s.io/kustomize/kyaml/yaml" ) var factory = provider.NewDefaultDepProvider().GetResourceFactory() @@ -1427,3 +1428,129 @@ spec: resid.FromString("knd2.ver2.gr2/name2.ns2"), }) } + +func TestOrigin(t *testing.T) { + r, err := factory.FromBytes([]byte(` +apiVersion: v1 +kind: Deployment +metadata: + name: clown +spec: + numReplicas: 1 +`)) + assert.NoError(t, err) + origin := &Origin{ + Path: "deployment.yaml", + Repo: "github.com/myrepo", + Ref: "master", + } + assert.NoError(t, r.SetOrigin(origin)) + assert.Equal(t, `apiVersion: v1 +kind: Deployment +metadata: + name: clown + annotations: + config.kubernetes.io/origin: | + path: deployment.yaml + repo: github.com/myrepo + ref: master +spec: + numReplicas: 1 +`, r.MustString()) + or, err := r.GetOrigin() + assert.NoError(t, err) + assert.Equal(t, origin, or) +} + +func TestTransformations(t *testing.T) { + r, err := factory.FromBytes([]byte(` +apiVersion: v1 +kind: Deployment +metadata: + name: clown +spec: + numReplicas: 1 +`)) + assert.NoError(t, err) + origin1 := &Origin{ + Repo: "github.com/myrepo", + Ref: "master", + ConfiguredIn: "config.yaml", + ConfiguredBy: kyaml.ResourceIdentifier{ + TypeMeta: kyaml.TypeMeta{ + APIVersion: "builtin", + Kind: "Generator", + }, + NameMeta: kyaml.NameMeta{ + Name: "my-name", + Namespace: "my-namespace", + }, + }, + } + origin2 := &Origin{ + ConfiguredIn: "../base/config.yaml", + ConfiguredBy: kyaml.ResourceIdentifier{ + TypeMeta: kyaml.TypeMeta{ + APIVersion: "builtin", + Kind: "Generator", + }, + NameMeta: kyaml.NameMeta{ + Name: "my-name", + Namespace: "my-namespace", + }, + }, + } + assert.NoError(t, r.AddTransformation(origin1)) + assert.Equal(t, `apiVersion: v1 +kind: Deployment +metadata: + name: clown + annotations: + config.kubernetes.io/transformations: | + - repo: github.com/myrepo + ref: master + configuredIn: config.yaml + configuredBy: + apiVersion: builtin + kind: Generator + name: my-name + namespace: my-namespace +spec: + numReplicas: 1 +`, r.MustString()) + assert.NoError(t, r.AddTransformation(origin2)) + assert.Equal(t, `apiVersion: v1 +kind: Deployment +metadata: + name: clown + annotations: + config.kubernetes.io/transformations: | + - repo: github.com/myrepo + ref: master + configuredIn: config.yaml + configuredBy: + apiVersion: builtin + kind: Generator + name: my-name + namespace: my-namespace + - configuredIn: ../base/config.yaml + configuredBy: + apiVersion: builtin + kind: Generator + name: my-name + namespace: my-namespace +spec: + numReplicas: 1 +`, r.MustString()) + transformations, err := r.GetTransformations() + assert.NoError(t, err) + assert.Equal(t, Transformations{origin1, origin2}, transformations) + assert.NoError(t, r.ClearTransformations()) + assert.Equal(t, `apiVersion: v1 +kind: Deployment +metadata: + name: clown +spec: + numReplicas: 1 +`, r.MustString()) +}