From 025200cc12cae9bc22ab1d7c8da9600f367b4920 Mon Sep 17 00:00:00 2001 From: Phillip Wittrock Date: Thu, 13 Feb 2020 16:23:01 -0800 Subject: [PATCH] support for adding setter substitution - refactor add setter to include file updates - support add substitution file updates --- kyaml/setters2/add.go | 126 ++++++++++ kyaml/setters2/add_test.go | 152 +++++++++++++ kyaml/setters2/set.go | 43 +++- kyaml/setters2/set_test.go | 215 ++++++++++++++++++ kyaml/setters2/setterdefinition.go | 56 ----- kyaml/setters2/setterdefinition_test.go | 86 ------- kyaml/setters2/substitutiondefinition.go | 60 ----- kyaml/setters2/substitutiondefinition_test.go | 86 ------- 8 files changed, 533 insertions(+), 291 deletions(-) delete mode 100644 kyaml/setters2/setterdefinition.go delete mode 100644 kyaml/setters2/setterdefinition_test.go delete mode 100644 kyaml/setters2/substitutiondefinition.go delete mode 100644 kyaml/setters2/substitutiondefinition_test.go diff --git a/kyaml/setters2/add.go b/kyaml/setters2/add.go index 9dac90cfb..8bf7cc49d 100644 --- a/kyaml/setters2/add.go +++ b/kyaml/setters2/add.go @@ -9,6 +9,7 @@ import ( "github.com/go-openapi/spec" "sigs.k8s.io/kustomize/kyaml/errors" "sigs.k8s.io/kustomize/kyaml/fieldmeta" + "sigs.k8s.io/kustomize/kyaml/openapi" "sigs.k8s.io/kustomize/kyaml/yaml" ) @@ -72,3 +73,128 @@ func (a *Add) visitScalar(object *yaml.RNode, p string) error { } return nil } + +const ( + // CLIDefinitionsPrefix is the prefix for cli definition keys. + CLIDefinitionsPrefix = "io.k8s.cli." + + // SetterDefinitionPrefix is the prefix for setter definition keys. + SetterDefinitionPrefix = CLIDefinitionsPrefix + "setters." + + // SubstitutionDefinitionPrefix is the prefix for substitution definition keys. + SubstitutionDefinitionPrefix = CLIDefinitionsPrefix + "substitutions." + + // DefinitionsPrefix is the prefix used to reference definitions in the OpenAPI + DefinitionsPrefix = "#/definitions/" +) + +// SetterDefinition may be used to update a files OpenAPI definitions with a new setter. +type SetterDefinition struct { + // Name is the name of the setter to create or update. + Name string `yaml:"name"` + + // Value is the value of the setter. + Value string `yaml:"value"` + + // SetBy is the person or role that last set the value. + SetBy string `yaml:"setBy,omitempty"` + + // Description is a description of the value. + Description string `yaml:"description,omitempty"` + + // Count is the number of fields set by this setter. + Count int `yaml:"count,omitempty"` +} + +func (sd SetterDefinition) AddToFile(path string) error { + return yaml.UpdateFile(sd, path) +} + +func (sd SetterDefinition) Filter(object *yaml.RNode) (*yaml.RNode, error) { + key := SetterDefinitionPrefix + sd.Name + + def, err := object.Pipe(yaml.LookupCreate( + yaml.MappingNode, openapi.SupplementaryOpenAPIFieldName, "definitions", key)) + if err != nil { + return nil, err + } + if sd.Description != "" { + err = def.PipeE(yaml.FieldSetter{Name: "description", StringValue: sd.Description}) + if err != nil { + return nil, err + } + // don't write the description to the extension + sd.Description = "" + } + + ext, err := def.Pipe(yaml.LookupCreate(yaml.MappingNode, K8sCliExtensionKey)) + if err != nil { + return nil, err + } + + b, err := yaml.Marshal(sd) + if err != nil { + return nil, err + } + y, err := yaml.Parse(string(b)) + if err != nil { + return nil, err + } + + if err := ext.PipeE(yaml.SetField("setter", y)); err != nil { + return nil, err + } + + return object, nil +} + +// SetterDefinition may be used to update a files OpenAPI definitions with a new substitution. +type SubstitutionDefinition struct { + // Name is the name of the substitution to create or update + Name string `yaml:"name"` + + // Pattern is the substitution pattern into which setter values are substituted + Pattern string `yaml:"pattern"` + + // Values are setters which are substituted into pattern to produce a field value + Values []Value `yaml:"values"` +} + +type Value struct { + // Marker is the string marker in pattern that is replace by the referenced setter. + Marker string `yaml:"marker"` + + // Ref is a reference to a setter to pull the replacement value from. + Ref string `yaml:"ref"` +} + +func (sd SubstitutionDefinition) AddToFile(path string) error { + return yaml.UpdateFile(sd, path) +} + +func (sd SubstitutionDefinition) Filter(object *yaml.RNode) (*yaml.RNode, error) { + // create the substitution extension value by marshalling the SubstitutionDefinition itself + b, err := yaml.Marshal(sd) + if err != nil { + return nil, err + } + sub, err := yaml.Parse(string(b)) + if err != nil { + return nil, err + } + + // lookup or create the definition for the substitution + defKey := SubstitutionDefinitionPrefix + sd.Name + def, err := object.Pipe(yaml.LookupCreate( + yaml.MappingNode, openapi.SupplementaryOpenAPIFieldName, "definitions", defKey, "x-k8s-cli")) + if err != nil { + return nil, err + } + + // set the substitution on the definition + if err := def.PipeE(yaml.SetField("substitution", sub)); err != nil { + return nil, err + } + + return object, nil +} diff --git a/kyaml/setters2/add_test.go b/kyaml/setters2/add_test.go index 8a867c63e..ab5524287 100644 --- a/kyaml/setters2/add_test.go +++ b/kyaml/setters2/add_test.go @@ -4,6 +4,9 @@ package setters2 import ( + "io/ioutil" + "os" + "path/filepath" "strings" "testing" @@ -204,3 +207,152 @@ spec: }) } } + +var resourcefile = `apiVersion: resource.dev/v1alpha1 +kind: resourcefile +metadata: + name: hello-world-set +upstream: + type: git + git: + commit: 5c1c019b59299a4f6c7edd1ff5ff54d720621bbe + directory: /package-examples/helloworld-set + ref: v0.1.0 +packageMetadata: + shortDescription: example package using setters` + +func TestAdd_Filter2(t *testing.T) { + path := filepath.Join(os.TempDir(), "resourcefile") + + //write initial resourcefile to temp path + err := ioutil.WriteFile(path, []byte(resourcefile), 0666) + if !assert.NoError(t, err) { + t.FailNow() + } + + //add a setter definition + sd := SetterDefinition{ + Name: "image", + Value: "1", + } + + err = sd.AddToFile(path) + + if !assert.NoError(t, err) { + t.FailNow() + } + + // update setter definition + sd2 := SetterDefinition{ + Name: "image", + Value: "2", + } + + err = sd2.AddToFile(path) + + if !assert.NoError(t, err) { + t.FailNow() + } + + b, err := ioutil.ReadFile(path) + if err != nil { + t.FailNow() + } + + expected := `apiVersion: resource.dev/v1alpha1 +kind: resourcefile +metadata: + name: hello-world-set +upstream: + type: git + git: + commit: 5c1c019b59299a4f6c7edd1ff5ff54d720621bbe + directory: /package-examples/helloworld-set + ref: v0.1.0 +packageMetadata: + shortDescription: example package using setters +openAPI: + definitions: + io.k8s.cli.setters.image: + x-k8s-cli: + setter: + name: image + value: "2" +` + assert.Equal(t, expected, string(b)) +} + +func TestAddUpdateSubstitution(t *testing.T) { + path := filepath.Join(os.TempDir(), "resourcefile") + + //write initial resourcefile to temp path + err := ioutil.WriteFile(path, []byte(resourcefile), 0666) + if !assert.NoError(t, err) { + t.FailNow() + } + + value1 := Value{ + Marker: "IMAGE_NAME", + Ref: "#/definitions/io.k8s.cli.setters.image-name", + } + + value2 := Value{ + Marker: "IMAGE_TAG", + Ref: "#/definitions/io.k8s.cli.setters.image-tag", + } + + values := []Value{value1, value2} + + //add a setter definition + subd := SubstitutionDefinition{ + Name: "image", + Pattern: "IMAGE_NAME:IMAGE_TAG", + Values: values, + } + + err = subd.AddToFile(path) + + if !assert.NoError(t, err) { + t.FailNow() + } + + // update setter definition + subd2 := SubstitutionDefinition{ + Name: "image", + Pattern: "IMAGE_NAME:IMAGE_TAG2", + } + + err = subd2.AddToFile(path) + + if !assert.NoError(t, err) { + t.FailNow() + } + + b, err := ioutil.ReadFile(path) + if err != nil { + t.FailNow() + } + + expected := `apiVersion: resource.dev/v1alpha1 +kind: resourcefile +metadata: + name: hello-world-set +upstream: + type: git + git: + commit: 5c1c019b59299a4f6c7edd1ff5ff54d720621bbe + directory: /package-examples/helloworld-set + ref: v0.1.0 +packageMetadata: + shortDescription: example package using setters +openAPI: + definitions: + io.k8s.cli.substitutions.image: + x-k8s-cli: + substitution: + name: image + pattern: IMAGE_NAME:IMAGE_TAG2 + values: [] +` + assert.Equal(t, expected, string(b)) +} diff --git a/kyaml/setters2/set.go b/kyaml/setters2/set.go index dc81e014b..f671c4026 100644 --- a/kyaml/setters2/set.go +++ b/kyaml/setters2/set.go @@ -17,6 +17,9 @@ type Set struct { // Name is the name of the setter to set on the object. i.e. matches the x-k8s-cli.setter.name // of the setter that should have its value applied to fields which reference it. Name string + + // Count is the number of fields that were updated by calling Filter + Count int } // Filter implements Set as a yaml.Filter @@ -25,7 +28,7 @@ func (s *Set) Filter(object *yaml.RNode) (*yaml.RNode, error) { } // visitScalar -func (s *Set) visitScalar(object *yaml.RNode, _ string) error { +func (s *Set) visitScalar(object *yaml.RNode, p string) error { // get the openAPI for this field describing how to apply the setter ext, err := getExtFromComment(object) if err != nil { @@ -37,14 +40,18 @@ func (s *Set) visitScalar(object *yaml.RNode, _ string) error { // perform a direct set of the field if it matches if s.set(object, ext) { + s.Count++ return nil } // perform a substitution of the field if it matches - if sub, err := s.substitute(object, ext); sub || err != nil { + sub, err := s.substitute(object, ext) + if err != nil { return err } - + if sub { + s.Count++ + } return nil } @@ -111,3 +118,33 @@ func (s *Set) set(field *yaml.RNode, ext *cliExtension) bool { field.YNode().Value = ext.Setter.Value return true } + +// SetOpenAPI updates a setter value +type SetOpenAPI struct { + // Name is the name of the setter to add + Name string `yaml:"name"` + // Value is the current value of the setter + Value string `yaml:"value"` +} + +// UpdateFile updates the OpenAPI definitions in a file with the given setter value. +func (s SetOpenAPI) UpdateFile(path string) error { + return yaml.UpdateFile(s, path) +} + +func (s SetOpenAPI) Filter(object *yaml.RNode) (*yaml.RNode, error) { + key := SetterDefinitionPrefix + s.Name + def, err := object.Pipe(yaml.Lookup( + "openAPI", "definitions", key, "x-k8s-cli", "setter")) + if err != nil { + return nil, err + } + if def == nil { + return nil, errors.Errorf("no setter %s found", s.Name) + } + if err := def.PipeE(&yaml.FieldSetter{Name: "value", StringValue: s.Value}); err != nil { + return nil, err + } + + return object, nil +} diff --git a/kyaml/setters2/set_test.go b/kyaml/setters2/set_test.go index 07e984a2b..1550a6244 100644 --- a/kyaml/setters2/set_test.go +++ b/kyaml/setters2/set_test.go @@ -438,3 +438,218 @@ func initSchema(t *testing.T, s string) { t.FailNow() } } + +func TestSetOpenAPI_Filter(t *testing.T) { + var tests = []struct { + name string + setter string + value string + input string + expected string + description string + setBy string + err string + }{ + { + name: "set-replicas", + setter: "replicas", + value: "3", + input: ` +openAPI: + definitions: + io.k8s.cli.setters.no-match-1': + x-k8s-cli: + setter: + name: no-match-1 + value: "1" + io.k8s.cli.setters.replicas: + x-k8s-cli: + setter: + name: replicas + value: "4" + io.k8s.cli.setters.no-match-2': + x-k8s-cli: + setter: + name: no-match-2 + value: "2" + `, + expected: ` +openAPI: + definitions: + io.k8s.cli.setters.no-match-1': + x-k8s-cli: + setter: + name: no-match-1 + value: "1" + io.k8s.cli.setters.replicas: + x-k8s-cli: + setter: + name: replicas + value: "3" + io.k8s.cli.setters.no-match-2': + x-k8s-cli: + setter: + name: no-match-2 + value: "2" +`, + }, + { + name: "set-replicas-description", + setter: "replicas", + value: "3", + description: "hello world", + input: ` +openAPI: + definitions: + io.k8s.cli.setters.no-match-1': + x-k8s-cli: + setter: + name: no-match-1 + value: "1" + io.k8s.cli.setters.replicas: + x-k8s-cli: + setter: + name: replicas + value: "4" + io.k8s.cli.setters.no-match-2': + x-k8s-cli: + setter: + name: no-match-2 + value: "2" + `, + expected: ` +openAPI: + definitions: + io.k8s.cli.setters.no-match-1': + x-k8s-cli: + setter: + name: no-match-1 + value: "1" + io.k8s.cli.setters.replicas: + x-k8s-cli: + setter: + name: replicas + value: "3" + description: hello world + io.k8s.cli.setters.no-match-2': + x-k8s-cli: + setter: + name: no-match-2 + value: "2" +`, + }, + { + name: "set-replicas-set-by", + setter: "replicas", + value: "3", + setBy: "carl", + input: ` +openAPI: + definitions: + io.k8s.cli.setters.no-match-1': + x-k8s-cli: + setter: + name: no-match-1 + value: "1" + io.k8s.cli.setters.replicas: + x-k8s-cli: + setter: + name: replicas + value: "4" + io.k8s.cli.setters.no-match-2': + x-k8s-cli: + setter: + name: no-match-2 + value: "2" + `, + expected: ` +openAPI: + definitions: + io.k8s.cli.setters.no-match-1': + x-k8s-cli: + setter: + name: no-match-1 + value: "1" + io.k8s.cli.setters.replicas: + x-k8s-cli: + setter: + name: replicas + value: "3" + setBy: carl + io.k8s.cli.setters.no-match-2': + x-k8s-cli: + setter: + name: no-match-2 + value: "2" +`, + }, + { + name: "error", + setter: "replicas", + err: "no setter replicas found", + input: ` +openAPI: + definitions: + io.k8s.cli.setters.no-match-1': + x-k8s-cli: + setter: + name: no-match-1 + value: "1" + io.k8s.cli.setters.no-match-2': + x-k8s-cli: + setter: + name: no-match-2 + value: "2" + `, + expected: ` +openAPI: + definitions: + io.k8s.cli.setters.no-match-1': + x-k8s-cli: + setter: + name: no-match-1 + value: "1" + io.k8s.cli.setters.no-match-2': + x-k8s-cli: + setter: + name: no-match-2 + value: "2" + `, + }, + } + for i := range tests { + test := tests[i] + t.Run(test.name, func(t *testing.T) { + in, err := yaml.Parse(test.input) + if !assert.NoError(t, err) { + t.FailNow() + } + + // invoke the setter + instance := &SetOpenAPI{ + Name: test.setter, Value: test.value, + SetBy: test.setBy, Description: test.description} + result, err := instance.Filter(in) + if test.err != "" { + if !assert.EqualError(t, err, test.err) { + t.FailNow() + } + return + } + if !assert.NoError(t, err) { + t.FailNow() + } + + // compare the actual and expected output + actual, err := result.String() + if !assert.NoError(t, err) { + t.FailNow() + } + actual = strings.TrimSpace(actual) + expected := strings.TrimSpace(test.expected) + if !assert.Equal(t, expected, actual) { + t.FailNow() + } + }) + } +} diff --git a/kyaml/setters2/setterdefinition.go b/kyaml/setters2/setterdefinition.go deleted file mode 100644 index be8604dcf..000000000 --- a/kyaml/setters2/setterdefinition.go +++ /dev/null @@ -1,56 +0,0 @@ -// Copyright 2019 The Kubernetes Authors. -// SPDX-License-Identifier: Apache-2.0 - -package setters2 - -import ( - "io/ioutil" - - "sigs.k8s.io/kustomize/kyaml/yaml" -) - -const DefinitionPrefix = "io.k8s.cli.setters." - -type SetterDefinition struct { - Name string - Value string -} - -func (sd SetterDefinition) AddSetterToFile(path string) error { - b, err := ioutil.ReadFile(path) - if err != nil { - return err - } - y, err := yaml.Parse(string(b)) - if err != nil { - return err - } - if err := y.PipeE(sd); err != nil { - return err - } - out, err := y.String() - if err != nil { - return err - } - if err := ioutil.WriteFile(path, []byte(out), 0600); err != nil { - return err - } - return nil -} - -func (sd SetterDefinition) Filter(object *yaml.RNode) (*yaml.RNode, error) { - key := DefinitionPrefix + sd.Name - - def, err := object.Pipe(yaml.LookupCreate( - yaml.MappingNode, "openAPI", "definitions", key, "x-k8s-cli", "setter")) - if err != nil { - return nil, err - } - if err := def.PipeE(yaml.FieldSetter{Name: "name", StringValue: sd.Name}); err != nil { - return nil, err - } - if err := def.PipeE(yaml.FieldSetter{Name: "value", StringValue: sd.Value}); err != nil { - return nil, err - } - return object, nil -} diff --git a/kyaml/setters2/setterdefinition_test.go b/kyaml/setters2/setterdefinition_test.go deleted file mode 100644 index f233271e2..000000000 --- a/kyaml/setters2/setterdefinition_test.go +++ /dev/null @@ -1,86 +0,0 @@ -// Copyright 2019 The Kubernetes Authors. -// SPDX-License-Identifier: Apache-2.0 - -package setters2 - -import ( - "io/ioutil" - "os" - "testing" - - "github.com/stretchr/testify/assert" -) - -var resourcefile = `apiVersion: resource.dev/v1alpha1 -kind: resourcefile -metadata: - name: hello-world-set -upstream: - type: git - git: - commit: 5c1c019b59299a4f6c7edd1ff5ff54d720621bbe - directory: /package-examples/helloworld-set - ref: v0.1.0 -packageMetadata: - shortDescription: example package using setters` - -func TestAddUpdateSetter(t *testing.T) { - path := os.TempDir() + "/resourcefile" - - //write initial resourcefile to temp path - err := ioutil.WriteFile(path, []byte(resourcefile), 0666) - if !assert.NoError(t, err) { - t.FailNow() - } - - //add a setter definition - sd := SetterDefinition{ - Name: "image", - Value: "1", - } - - err = sd.AddSetterToFile(path) - - if !assert.NoError(t, err) { - t.FailNow() - } - - // update setter definition - sd2 := SetterDefinition{ - Name: "image", - Value: "2", - } - - err = sd2.AddSetterToFile(path) - - if !assert.NoError(t, err) { - t.FailNow() - } - - b, err := ioutil.ReadFile(path) - if err != nil { - t.FailNow() - } - - expected := `apiVersion: resource.dev/v1alpha1 -kind: resourcefile -metadata: - name: hello-world-set -upstream: - type: git - git: - commit: 5c1c019b59299a4f6c7edd1ff5ff54d720621bbe - directory: /package-examples/helloworld-set - ref: v0.1.0 -packageMetadata: - shortDescription: example package using setters -openAPI: - definitions: - io.k8s.cli.setters.image: - x-k8s-cli: - setter: - name: image - value: 2 -` - assert.Equal(t, expected, string(b)) -} diff --git a/kyaml/setters2/substitutiondefinition.go b/kyaml/setters2/substitutiondefinition.go deleted file mode 100644 index 17d4ff34e..000000000 --- a/kyaml/setters2/substitutiondefinition.go +++ /dev/null @@ -1,60 +0,0 @@ -// Copyright 2019 The Kubernetes Authors. -// SPDX-License-Identifier: Apache-2.0 - -package setters2 - -import ( - "io/ioutil" - - "sigs.k8s.io/kustomize/kyaml/yaml" -) - -type SubstitutionDefinition struct { - Name string `yaml:"name"` - Pattern string `yaml:"pattern"` - Values []Value `yaml:"value"` -} - -type Value struct { - Marker string `yaml:"marker"` - Ref string `yaml:"ref"` -} - -func (subd SubstitutionDefinition) AddSubstitutionToFile(path string) error { - b, err := ioutil.ReadFile(path) - if err != nil { - return err - } - y, err := yaml.Parse(string(b)) - if err != nil { - return err - } - if err := y.PipeE(subd); err != nil { - return err - } - out, err := y.String() - if err != nil { - return err - } - if err := ioutil.WriteFile(path, []byte(out), 0666); err != nil { - return err - } - return nil -} - -func (subd SubstitutionDefinition) Filter(object *yaml.RNode) (*yaml.RNode, error) { - key := DefinitionPrefix + subd.Name - - def, err := object.Pipe(yaml.LookupCreate( - yaml.MappingNode, "openAPI", "definitions", key, "x-k8s-cli", "substitution")) - if err != nil { - return nil, err - } - if err := def.PipeE(yaml.FieldSetter{Name: "name", StringValue: subd.Name}); err != nil { - return nil, err - } - if err := def.PipeE(yaml.FieldSetter{Name: "pattern", StringValue: subd.Pattern}); err != nil { - return nil, err - } - return object, nil -} diff --git a/kyaml/setters2/substitutiondefinition_test.go b/kyaml/setters2/substitutiondefinition_test.go deleted file mode 100644 index 9b239bd89..000000000 --- a/kyaml/setters2/substitutiondefinition_test.go +++ /dev/null @@ -1,86 +0,0 @@ -// Copyright 2019 The Kubernetes Authors. -// SPDX-License-Identifier: Apache-2.0 - -package setters2 - -import ( - "io/ioutil" - "os" - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestAddUpdateSubstitution(t *testing.T) { - path := os.TempDir() + "/resourcefile" - - //write initial resourcefile to temp path - err := ioutil.WriteFile(path, []byte(resourcefile), 0666) - if !assert.NoError(t, err) { - t.FailNow() - } - - value1 := Value{ - Marker: "IMAGE_NAME", - Ref: "#/definitions/io.k8s.cli.setters.image-name", - } - - value2 := Value{ - Marker: "IMAGE_TAG", - Ref: "#/definitions/io.k8s.cli.setters.image-tag", - } - - values := []Value{value1, value2} - - //add a setter definition - subd := SubstitutionDefinition{ - Name: "image", - Pattern: "IMAGE_NAME:IMAGE_TAG", - Values: values, - } - - err = subd.AddSubstitutionToFile(path) - - if !assert.NoError(t, err) { - t.FailNow() - } - - // update setter definition - subd2 := SubstitutionDefinition{ - Name: "image", - Pattern: "IMAGE_NAME:IMAGE_TAG2", - } - - err = subd2.AddSubstitutionToFile(path) - - if !assert.NoError(t, err) { - t.FailNow() - } - - b, err := ioutil.ReadFile(path) - if err != nil { - t.FailNow() - } - - expected := `apiVersion: resource.dev/v1alpha1 -kind: resourcefile -metadata: - name: hello-world-set -upstream: - type: git - git: - commit: 5c1c019b59299a4f6c7edd1ff5ff54d720621bbe - directory: /package-examples/helloworld-set - ref: v0.1.0 -packageMetadata: - shortDescription: example package using setters -openAPI: - definitions: - io.k8s.cli.setters.image: - x-k8s-cli: - substitution: - name: image - pattern: IMAGE_NAME:IMAGE_TAG2 -` - assert.Equal(t, expected, string(b)) -}