From 79fbe7c4ccfcac1002a6c897594459144ceaf99c Mon Sep 17 00:00:00 2001 From: Richard Marshall Date: Thu, 22 Aug 2019 08:06:58 -0700 Subject: [PATCH 1/4] Support resource generator options in exec plugins --- pkg/plugins/execplugin.go | 39 ++++++++++++++++++++++++++++++++++++++- pkg/resource/resource.go | 5 +++++ pkg/target/kusttarget.go | 20 ++++++-------------- 3 files changed, 49 insertions(+), 15 deletions(-) diff --git a/pkg/plugins/execplugin.go b/pkg/plugins/execplugin.go index 3af9211cc..061ee8693 100644 --- a/pkg/plugins/execplugin.go +++ b/pkg/plugins/execplugin.go @@ -9,17 +9,21 @@ import ( "io/ioutil" "os" "os/exec" + "strconv" "strings" "github.com/pkg/errors" "sigs.k8s.io/kustomize/v3/pkg/ifc" "sigs.k8s.io/kustomize/v3/pkg/resid" "sigs.k8s.io/kustomize/v3/pkg/resmap" + "sigs.k8s.io/kustomize/v3/pkg/types" "sigs.k8s.io/yaml" ) const ( idAnnotation = "kustomize.config.k8s.io/id" + hashAnnotation = "kustomize.config.k8s.io/needs-hash" + behaviorAnnotation = "kustomize.config.k8s.io/behavior" tmpConfigFilePrefix = "kust-plugin-config-" ) @@ -96,7 +100,11 @@ func (p *ExecPlugin) Generate() (resmap.ResMap, error) { if err != nil { return nil, err } - return p.rf.NewResMapFromBytes(output) + rm, err := p.rf.NewResMapFromBytes(output) + if err != nil { + return nil, err + } + return p.updateResourceOptions(rm) } func (p *ExecPlugin) Transform(rm resmap.ResMap) error { @@ -223,3 +231,32 @@ func (p *ExecPlugin) updateResMapValues(output []byte, rm resmap.ResMap) error { } return nil } + +// updateResourceOptions updates the generator options for each resource in the +// given ResMap based on plugin provided annotations. +func (p *ExecPlugin) updateResourceOptions(rm resmap.ResMap) (resmap.ResMap, error) { + for _, r := range rm.Resources() { + // Disable name hashing by default and require plugin to explicitly + // request it for each resource. + annotations := r.GetAnnotations() + behavior := annotations[behaviorAnnotation] + var needsHash bool + if val, ok := annotations[hashAnnotation]; ok { + b, err := strconv.ParseBool(val) + if err != nil { + return nil, fmt.Errorf("the annotation %q contains an invalid value (%q)", hashAnnotation, val) + } + needsHash = b + } + delete(annotations, hashAnnotation) + delete(annotations, behaviorAnnotation) + if len(annotations) == 0 { + annotations = nil + } + r.SetAnnotations(annotations) + r.SetOptions(types.NewGenArgs( + &types.GeneratorArgs{Behavior: behavior}, + &types.GeneratorOptions{DisableNameSuffixHash: !needsHash})) + } + return rm, nil +} diff --git a/pkg/resource/resource.go b/pkg/resource/resource.go index fa2c87cfe..01dd4649e 100644 --- a/pkg/resource/resource.go +++ b/pkg/resource/resource.go @@ -242,6 +242,11 @@ func (r *Resource) AsYAML() ([]byte, error) { return yaml.JSONToYAML(json) } +// SetOptions updates the generator options for the resource. +func (r *Resource) SetOptions(o *types.GenArgs) { + r.options = o +} + // Behavior returns the behavior for the resource. func (r *Resource) Behavior() types.GenerationBehavior { return r.options.Behavior() diff --git a/pkg/target/kusttarget.go b/pkg/target/kusttarget.go index 3abb12972..2bbfe2b32 100644 --- a/pkg/target/kusttarget.go +++ b/pkg/target/kusttarget.go @@ -250,31 +250,23 @@ func (kt *KustTarget) AccumulateTarget() ( func (kt *KustTarget) runGenerators( ra *accumulator.ResAccumulator) error { - generators, err := kt.configureBuiltinGenerators() + var generators []resmap.Generator + gs, err := kt.configureBuiltinGenerators() if err != nil { return err } - for _, g := range generators { - resMap, err := g.Generate() - if err != nil { - return err - } - // The legacy generators allow override. - err = ra.AbsorbAll(resMap) - if err != nil { - return errors.Wrapf(err, "merging from generator %v", g) - } - } - generators, err = kt.configureExternalGenerators() + generators = append(generators, gs...) + gs, err = kt.configureExternalGenerators() if err != nil { return errors.Wrap(err, "loading generator plugins") } + generators = append(generators, gs...) for _, g := range generators { resMap, err := g.Generate() if err != nil { return err } - err = ra.AppendAll(resMap) + err = ra.AbsorbAll(resMap) if err != nil { return errors.Wrapf(err, "merging from generator %v", g) } From ac9424fa3ecf03c485c8c6653d4dd28b61618456 Mon Sep 17 00:00:00 2001 From: Richard Marshall Date: Mon, 2 Sep 2019 08:46:09 -0700 Subject: [PATCH 2/4] tests: Add unit tests for update resource options --- pkg/plugins/execplugin.go | 6 +- pkg/plugins/execplugin_test.go | 98 +++++++++++++++++++ .../v1/bashedconfigmap/BashedConfigMap | 6 +- .../bashedconfigmap/BashedConfigMap_test.go | 5 +- 4 files changed, 108 insertions(+), 7 deletions(-) diff --git a/pkg/plugins/execplugin.go b/pkg/plugins/execplugin.go index 061ee8693..03b324b7a 100644 --- a/pkg/plugins/execplugin.go +++ b/pkg/plugins/execplugin.go @@ -192,10 +192,8 @@ func (p *ExecPlugin) getResMapWithIdAnnotation(rm resmap.ResMap) (resmap.ResMap, return inputRM, nil } -/* -updateResMapValues updates the Resource value in the given ResMap -with the emitted Resource values in output. -*/ +// updateResMapValues updates the Resource value in the given ResMap +// with the emitted Resource values in output. func (p *ExecPlugin) updateResMapValues(output []byte, rm resmap.ResMap) error { outputRM, err := p.rf.NewResMapFromBytes(output) if err != nil { diff --git a/pkg/plugins/execplugin_test.go b/pkg/plugins/execplugin_test.go index 7dc7708d2..2375f16e8 100644 --- a/pkg/plugins/execplugin_test.go +++ b/pkg/plugins/execplugin_test.go @@ -17,6 +17,7 @@ limitations under the License. package plugins import ( + "fmt" "strings" "testing" @@ -24,6 +25,7 @@ import ( "sigs.k8s.io/kustomize/v3/k8sdeps/kunstruct" "sigs.k8s.io/kustomize/v3/pkg/resmap" "sigs.k8s.io/kustomize/v3/pkg/resource" + "sigs.k8s.io/kustomize/v3/pkg/types" ) func TestExecPluginConfig(t *testing.T) { @@ -87,3 +89,99 @@ metadata: t.Fatalf("unexpected arg array: %v", p.args) } } + +func makeConfigMap(rf *resource.Factory, name, behavior string, hashValue *string) *resource.Resource { + r := rf.FromMap(map[string]interface{}{ + "apiVersion": "v1", + "kind": "ConfigMap", + "metadata": map[string]interface{}{"name": name}, + }) + annotations := map[string]string{} + if behavior != "" { + annotations[behaviorAnnotation] = behavior + } + if hashValue != nil { + annotations[hashAnnotation] = *hashValue + } + if len(annotations) > 0 { + r.SetAnnotations(annotations) + } + return r +} + +func makeConfigMapOptions(rf *resource.Factory, name, behavior string, disableHash bool) *resource.Resource { + return rf.FromMapAndOption(map[string]interface{}{ + "apiVersion": "v1", + "kind": "ConfigMap", + "metadata": map[string]interface{}{"name": name}, + }, &types.GeneratorArgs{Behavior: behavior}, &types.GeneratorOptions{DisableNameSuffixHash: disableHash}) +} + +func strptr(s string) *string { + return &s +} + +func TestUpdateResourceOptions(t *testing.T) { + p := NewExecPlugin("") + rf := resource.NewFactory(kunstruct.NewKunstructuredFactoryImpl()) + in := resmap.New() + expected := resmap.New() + cases := []struct { + behavior string + needsHash bool + hashValue *string + }{ + {hashValue: strptr("false")}, + {hashValue: strptr("true"), needsHash: true}, + {behavior: "replace"}, + {behavior: "merge"}, + {behavior: "create"}, + {behavior: "nonsense"}, + {behavior: "merge", hashValue: strptr("false")}, + {behavior: "merge", hashValue: strptr("true"), needsHash: true}, + } + for i, c := range cases { + name := fmt.Sprintf("test%d", i) + in.Append(makeConfigMap(rf, name, c.behavior, c.hashValue)) + expected.Append(makeConfigMapOptions(rf, name, c.behavior, !c.needsHash)) + } + actual, err := p.updateResourceOptions(in) + if err != nil { + t.Fatalf("unexpected error: %v", err.Error()) + } + for i, a := range expected.Resources() { + b := actual.GetByIndex(i) + if b == nil { + t.Fatalf("resource %d missing from processed map", i) + } + if !a.Equals(b) { + t.Errorf("expected %v got %v", a, b) + } + if a.NeedHashSuffix() != b.NeedHashSuffix() { + t.Errorf("") + } + if a.Behavior() != b.Behavior() { + t.Errorf("expected %v got %v", a.Behavior(), b.Behavior()) + } + } +} + +func TestUpdateResourceOptionsWithInvalidHashAnnotationValues(t *testing.T) { + p := NewExecPlugin("") + rf := resource.NewFactory(kunstruct.NewKunstructuredFactoryImpl()) + cases := []string{ + "", + "FaLsE", + "TrUe", + "potato", + } + for i, c := range cases { + name := fmt.Sprintf("test%d", i) + in := resmap.New() + in.Append(makeConfigMap(rf, name, "", &c)) + _, err := p.updateResourceOptions(in) + if err == nil { + t.Errorf("expected error from value %q", c) + } + } +} diff --git a/plugin/someteam.example.com/v1/bashedconfigmap/BashedConfigMap b/plugin/someteam.example.com/v1/bashedconfigmap/BashedConfigMap index 890423807..5aca2a0ff 100755 --- a/plugin/someteam.example.com/v1/bashedconfigmap/BashedConfigMap +++ b/plugin/someteam.example.com/v1/bashedconfigmap/BashedConfigMap @@ -3,12 +3,14 @@ # Skip the config file name argument. shift -echo " +cat < Date: Tue, 24 Sep 2019 07:42:27 -0700 Subject: [PATCH 3/4] docs: Exec plugin generator options --- docs/authoriing.md | 37 +++++++++++++++++++++++++++++++++++++ docs/plugins/README.md | 40 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 77 insertions(+) create mode 100644 docs/authoriing.md diff --git a/docs/authoriing.md b/docs/authoriing.md new file mode 100644 index 000000000..40c3d6f11 --- /dev/null +++ b/docs/authoriing.md @@ -0,0 +1,37 @@ +# kustomization authoring + +kustomize provides sub-commands for managing the contents of a kustomization file from the command line. + +## kustomize create + +The `kustomize create` command will create a new kustomization in the current directory. + +When run without any flags the command will create an empty `kustomization.yaml` file that can then be updated manually or with the `kustomize edit` sub-commands. + +``` +kustomize create --namespace=myapp --resources=deployment.yaml,service.yaml --label=app=myapp +``` + +### Detecting resources + +> NOTE: Resource detection will not follow symlinks. + +Flags: + --annotation string Add one or more common annotations. + --autodetect Search for kubernetes resources in the current directory to be added to the kustomization file. + -h, --help help for create + --label string Add one or more common labels. + --nameprefix string Sets the value of the namePrefix field in the kustomization file. + --namespace string Set the value of the namespace field in the customization file. + --namesuffix string Sets the value of the nameSuffix field in the kustomization file. + --recursive Enable recursive directory searching for resource auto-detection. + --resources string Name of a file containing a file to add to the kustomization file. + +## kustomize edit + +With an existing kustomization file the `kustomize edit` command + +* add +* set +* remove +* fix \ No newline at end of file diff --git a/docs/plugins/README.md b/docs/plugins/README.md index f250da22b..90b972f03 100644 --- a/docs/plugins/README.md +++ b/docs/plugins/README.md @@ -244,6 +244,46 @@ kustomize uses an exec plugin adapter to provide marshalled resources on `stdin` and capture `stdout` for further processing. +#### Generator Options + +A generator exec plugin can adjust the generator options for the resources it emits by setting one of the following internal annotations. + +> NOTE: These annotations are local to kustomize and will not be included in the final output. + +**`kustomize.config.k8s.io/needs-hash`** + +Resources can be marked as needing to be processed by the internal hash transformer by including the `needs-hash` annotation. When set valid values for the annotation are `"true"` and `"false"` which respectively enable or disable hash suffixing for the resource. Omitting the annotation is equivalent to setting the value `"false"`. + +If this annotation is set on a resource not supported by the hash transformer the build will fail. + +Example: +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: cm-test + annotations: + kustomize.config.k8s.io/needs-hash: "true" +data: + foo: bar +``` + +**`kustomize.config.k8s.io/behavior`** + +The `behavior` annotation will influence how conflicts are handled for resources emitted by the plugin. Valid values include "create", "merge", and "replace" with "create" being the default. + +Example: +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: cm-test + annotations: + kustomize.config.k8s.io/behavior: "merge" +data: + foo: bar +``` + ### Go plugins Be sure to read [Go plugin caveats](goPluginCaveats.md). From 9288dec02a262acb08c029ecb2528dddc05b8457 Mon Sep 17 00:00:00 2001 From: Jeff Regan Date: Thu, 26 Sep 2019 09:56:04 -0700 Subject: [PATCH 4/4] Fix failing BashedConfigMapTest --- plugin/someteam.example.com/v1/bashedconfigmap/BashedConfigMap | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugin/someteam.example.com/v1/bashedconfigmap/BashedConfigMap b/plugin/someteam.example.com/v1/bashedconfigmap/BashedConfigMap index 5aca2a0ff..825a86e78 100755 --- a/plugin/someteam.example.com/v1/bashedconfigmap/BashedConfigMap +++ b/plugin/someteam.example.com/v1/bashedconfigmap/BashedConfigMap @@ -9,7 +9,7 @@ apiVersion: v1 metadata: name: example-configmap-test annotations: - kustomize.config.k8s.io/needs-hash: "" + kustomize.config.k8s.io/needs-hash: "true" data: username: $1 password: $2