Fix V1 Setters and migrate to latest

This commit is contained in:
Phani Teja Marupaka
2020-07-27 00:50:25 -07:00
parent 9ba04e3f7d
commit 6faff2d031
11 changed files with 893 additions and 45 deletions

View File

@@ -0,0 +1,74 @@
// Copyright 2019 The Kubernetes Authors.
// SPDX-License-Identifier: Apache-2.0
package fixsetters
import (
"encoding/json"
"fmt"
"strings"
"github.com/go-openapi/spec"
"sigs.k8s.io/kustomize/kyaml/errors"
"sigs.k8s.io/kustomize/kyaml/fieldmeta"
"sigs.k8s.io/kustomize/kyaml/yaml"
)
// FieldMeta contains metadata that may be attached to fields as comments
type FieldMetaV1 struct {
Schema spec.Schema
Extensions XKustomize
}
type XKustomize struct {
SetBy string `yaml:"setBy,omitempty" json:"setBy,omitempty"`
PartialFieldSetters []PartialFieldSetter `yaml:"partialSetters,omitempty" json:"partialSetters,omitempty"`
FieldSetter *PartialFieldSetter `yaml:"setter,omitempty" json:"setter,omitempty"`
}
// PartialFieldSetter defines how to set part of a field rather than the full field
// value. e.g. the tag part of an image field
type PartialFieldSetter struct {
// Name is the name of this setter.
Name string `yaml:"name" json:"name"`
// Value is the current value that has been set.
Value string `yaml:"value" json:"value"`
}
// UpgradeV1SetterComment reads the FieldMeta from a node and upgrade the
// setters comment to latest
func (fm *FieldMetaV1) UpgradeV1SetterComment(n *yaml.RNode) error {
// check for metadata on head and line comments
comments := []string{n.YNode().LineComment, n.YNode().HeadComment}
for _, c := range comments {
if c == "" {
continue
}
c := strings.TrimLeft(c, "#")
if err := fm.Schema.UnmarshalJSON([]byte(c)); err != nil {
// note: don't return an error if the comment isn't a fieldmeta struct
return nil
}
fe := fm.Schema.VendorExtensible.Extensions["x-kustomize"]
if fe == nil {
return nil
}
b, err := json.Marshal(fe)
if err != nil {
return errors.Wrap(err)
}
// delete line comment after parsing info into fieldmeta
n.YNode().HeadComment = ""
n.YNode().LineComment = ""
err = json.Unmarshal(b, &fm.Extensions)
if fm.Extensions.FieldSetter != nil {
n.YNode().LineComment = fmt.Sprintf(`{"%s":"%s"}`, fieldmeta.ShortHandRef(), fm.Extensions.FieldSetter.Name)
}
return err
}
return nil
}

View File

@@ -0,0 +1,141 @@
// Copyright 2019 The Kubernetes Authors.
// SPDX-License-Identifier: Apache-2.0
package fixsetters
import (
"os"
"sigs.k8s.io/kustomize/kyaml/kio"
"sigs.k8s.io/kustomize/kyaml/setters2"
"sigs.k8s.io/kustomize/kyaml/setters2/settersutil"
)
// SetterFixer fixes setters in the input package
type SetterFixer struct {
// PkgPath is path to the resource package
PkgPath string
// OpenAPIPath is path to the openAPI file in the package
OpenAPIPath string
// DryRun only displays the actions without performing
DryRun bool
}
// SetterFixerV1Result holds the results of V1 setters fix
type SetterFixerV1Result struct {
// NeedFix indicates if the resource in pkgPath are on V1 version of setters
// and need to be fixed
NeedFix bool
// CreatedSetters are setters created as part of this fix
CreatedSetters []string
// CreatedSubst are substitutions created as part of this fix
CreatedSubst []string
// FailedSetters are setters failed to create from current V1 setters
FailedSetters map[string]error
// FailedSubst are substitutions failed to be created from V1 partial setters
FailedSubst map[string]error
}
// FixSettersV1 reads the package and upgrades v1 version of setters
// to latest
func (f *SetterFixer) FixV1Setters() (SetterFixerV1Result, error) {
sfr := SetterFixerV1Result{
FailedSetters: make(map[string]error),
FailedSubst: make(map[string]error),
}
// KrmFile need not exist for dryRun
if !f.DryRun {
_, err := os.Stat(f.OpenAPIPath)
if err != nil {
return sfr, err
}
}
var err error
// lookup for all setters and partial setters in v1 format
// delete the v1 format comments after lookup
l := UpgradeV1Setters{}
if f.DryRun {
err = applyReadFilter(&l, f.PkgPath)
} else {
err = applyWriteFilter(&l, f.PkgPath)
}
if err != nil {
return sfr, err
}
if len(l.SetterCounts) > 0 {
sfr.NeedFix = true
} else {
return sfr, nil
}
// for each v1 setter create the equivalent in v2,
for _, setter := range l.SetterCounts {
sd := setters2.SetterDefinition{
Name: setter.Name,
Value: setter.Value,
Description: setter.Description,
SetBy: setter.SetBy,
Type: setter.Type,
}
var err error
if !f.DryRun {
err = sd.AddToFile(f.OpenAPIPath)
}
if err != nil {
sfr.FailedSetters[setter.Name] = err
} else {
sfr.CreatedSetters = append(sfr.CreatedSetters, setter.Name)
}
}
// for each group of partial setters, create equivalent substitution
for _, subst := range l.Substitutions {
sc := settersutil.SubstitutionCreator{
Name: subst.Name,
FieldValue: subst.FieldVale,
Pattern: subst.Pattern,
ResourcesPath: f.PkgPath,
OpenAPIPath: f.OpenAPIPath,
}
var err error
if !f.DryRun {
err = applyWriteFilter(&sc, f.PkgPath)
}
if err != nil {
sfr.FailedSubst[subst.Name] = err
} else {
sfr.CreatedSubst = append(sfr.CreatedSubst, subst.Name)
}
}
return sfr, nil
}
func applyWriteFilter(f kio.Filter, pkgPath string) error {
rw := &kio.LocalPackageReadWriter{
PackagePath: pkgPath,
}
return kio.Pipeline{
Inputs: []kio.Reader{rw},
Filters: []kio.Filter{f},
Outputs: []kio.Writer{rw},
}.Execute()
}
func applyReadFilter(f kio.Filter, pkgPath string) error {
rw := &kio.LocalPackageReader{
PackagePath: pkgPath,
}
return kio.Pipeline{
Inputs: []kio.Reader{rw},
Filters: []kio.Filter{f},
}.Execute()
}

View File

@@ -0,0 +1,405 @@
// Copyright 2019 The Kubernetes Authors.
// SPDX-License-Identifier: Apache-2.0
package fixsetters
import (
"io/ioutil"
"os"
"path/filepath"
"testing"
"github.com/stretchr/testify/assert"
)
func TestFixSettersV1(t *testing.T) {
var tests = []struct {
name string
input string
err string
dryRun bool
openAPIFile string
expectedOutput string
expectedOpenAPI string
needFix bool
createdSetters []string
createdSubst []string
failedSetters map[string]error
failedSubst map[string]error
}{
{
name: "upgrade-delete-partial-setters",
input: `
apiVersion: install.istio.io/v1alpha2
kind: IstioControlPlane
metadata:
cluster: "someproj/someclus" # {"type":"string","x-kustomize":{"partialSetters":[{"name":"project","value":"someproj"},{"name":"cluster","value":"someclus"}]}}
spec:
profile: asm # {"type":"string","x-kustomize":{"setter":{"name":"profile","value":"asm"}}}
cluster: "someproj/someclus" # {"type":"string","x-kustomize":{"partialSetters":[{"name":"project","value":"someproj"},{"name":"cluster","value":"someclus"}]}}
`,
openAPIFile: `apiVersion: kustomization.dev/v1alpha1
kind: Kustomization`,
needFix: true,
createdSetters: []string{"cluster", "profile", "project"},
createdSubst: []string{"subst-project-cluster"},
failedSetters: map[string]error{},
failedSubst: map[string]error{},
expectedOutput: `apiVersion: install.istio.io/v1alpha2
kind: IstioControlPlane
metadata:
cluster: "someproj/someclus" # {"$openapi":"subst-project-cluster"}
spec:
profile: asm # {"$openapi":"profile"}
cluster: "someproj/someclus" # {"$openapi":"subst-project-cluster"}
`,
expectedOpenAPI: `apiVersion: kustomization.dev/v1alpha1
kind: Kustomization
openAPI:
definitions:
io.k8s.cli.setters.cluster:
type: string
x-k8s-cli:
setter:
name: cluster
value: someclus
io.k8s.cli.setters.profile:
type: string
x-k8s-cli:
setter:
name: profile
value: asm
io.k8s.cli.setters.project:
type: string
x-k8s-cli:
setter:
name: project
value: someproj
io.k8s.cli.substitutions.subst-project-cluster:
x-k8s-cli:
substitution:
name: subst-project-cluster
pattern: ${project}/${cluster}
values:
- marker: ${project}
ref: '#/definitions/io.k8s.cli.setters.project'
- marker: ${cluster}
ref: '#/definitions/io.k8s.cli.setters.cluster'
`,
},
{
name: "upgrade-delete-partial-setters-dryRun",
dryRun: true,
input: `apiVersion: install.istio.io/v1alpha2
kind: IstioControlPlane
metadata:
cluster: "someproj/someclus" # {"type":"string","x-kustomize":{"partialSetters":[{"name":"project","value":"someproj"},{"name":"cluster","value":"someclus"}]}}
spec:
profile: asm # {"type":"string","x-kustomize":{"setter":{"name":"profile","value":"asm"}}}
cluster: "someproj/someclus" # {"type":"string","x-kustomize":{"partialSetters":[{"name":"project","value":"someproj"},{"name":"cluster","value":"someclus"}]}}
`,
needFix: true,
createdSetters: []string{"cluster", "profile", "project"},
createdSubst: []string{"subst-project-cluster"},
failedSetters: map[string]error{},
failedSubst: map[string]error{},
expectedOutput: `apiVersion: install.istio.io/v1alpha2
kind: IstioControlPlane
metadata:
cluster: "someproj/someclus" # {"type":"string","x-kustomize":{"partialSetters":[{"name":"project","value":"someproj"},{"name":"cluster","value":"someclus"}]}}
spec:
profile: asm # {"type":"string","x-kustomize":{"setter":{"name":"profile","value":"asm"}}}
cluster: "someproj/someclus" # {"type":"string","x-kustomize":{"partialSetters":[{"name":"project","value":"someproj"},{"name":"cluster","value":"someclus"}]}}
`,
},
{
name: "partial-setters-same-value",
input: `
apiVersion: install.istio.io/v1alpha2
kind: IstioControlPlane
spec:
profile: asm # {"type":"string","x-kustomize":{"setter":{"name":"profile","value":"asm"}}}
team: asm # {"type":"string","x-kustomize":{"setter":{"name":"team","value":"asm"}}}
profile-team: asm/asm # {"type":"string","x-kustomize":{"partialSetters":[{"name":"profile","value":"asm"},{"name":"team","value":"asm"}]}}
`,
openAPIFile: `apiVersion: kustomization.dev/v1alpha1
kind: Kustomization`,
needFix: true,
createdSetters: []string{"profile", "team"},
createdSubst: []string{"subst-profile-team"},
failedSetters: map[string]error{},
failedSubst: map[string]error{},
expectedOutput: `apiVersion: install.istio.io/v1alpha2
kind: IstioControlPlane
spec:
profile: asm # {"$openapi":"profile"}
team: asm # {"$openapi":"team"}
profile-team: asm/asm # {"$openapi":"subst-profile-team"}
`,
expectedOpenAPI: `apiVersion: kustomization.dev/v1alpha1
kind: Kustomization
openAPI:
definitions:
io.k8s.cli.setters.profile:
type: string
x-k8s-cli:
setter:
name: profile
value: asm
io.k8s.cli.setters.team:
type: string
x-k8s-cli:
setter:
name: team
value: asm
io.k8s.cli.substitutions.subst-profile-team:
x-k8s-cli:
substitution:
name: subst-profile-team
pattern: ${profile}/${team}
values:
- marker: ${profile}
ref: '#/definitions/io.k8s.cli.setters.profile'
- marker: ${team}
ref: '#/definitions/io.k8s.cli.setters.team'
`,
},
{
name: "upgrade-with-both-versions",
input: `
apiVersion: install.istio.io/v1alpha2
kind: IstioControlPlane
metadata:
clusterName: "project-id/us-east1-d/cluster-name"
spec:
profile: asm # {"type":"string","x-kustomize":{"setter":{"name":"profilesetter","value":"asm"}}}
hub: gcr.io/asm-testing # {"$openapi":"hubsetter"}
`,
openAPIFile: `apiVersion: kustomization.dev/v1alpha1
kind: Kustomization
openAPI:
definitions:
io.k8s.cli.setters.hubsetter:
type: string
x-k8s-cli:
setter:
name: hubsetter
value: gcr.io/asm-testing`,
needFix: true,
createdSetters: []string{"profilesetter"},
failedSetters: map[string]error{},
failedSubst: map[string]error{},
expectedOutput: `apiVersion: install.istio.io/v1alpha2
kind: IstioControlPlane
metadata:
clusterName: "project-id/us-east1-d/cluster-name"
spec:
profile: asm # {"$openapi":"profilesetter"}
hub: gcr.io/asm-testing # {"$openapi":"hubsetter"}
`,
expectedOpenAPI: `apiVersion: kustomization.dev/v1alpha1
kind: Kustomization
openAPI:
definitions:
io.k8s.cli.setters.hubsetter:
type: string
x-k8s-cli:
setter:
name: hubsetter
value: gcr.io/asm-testing
io.k8s.cli.setters.profilesetter:
type: string
x-k8s-cli:
setter:
name: profilesetter
value: asm
`,
},
{
name: "setter-already-exists",
input: `
apiVersion: install.istio.io/v1alpha2
kind: IstioControlPlane
metadata:
clusterName: "project-id/us-east1-d/cluster-name"
spec:
profile: asm # {"type":"string","x-kustomize":{"setter":{"name":"profilesetter","value":"asm"}}}
hub: asm # {"$openapi":"profilesetter"}
`,
openAPIFile: `apiVersion: kustomization.dev/v1alpha1
kind: Kustomization
openAPI:
definitions:
io.k8s.cli.setters.profilesetter:
type: string
x-k8s-cli:
setter:
name: profilesetter
value: asm
`,
needFix: true,
createdSetters: []string{"profilesetter"},
failedSetters: map[string]error{},
failedSubst: map[string]error{},
expectedOutput: `apiVersion: install.istio.io/v1alpha2
kind: IstioControlPlane
metadata:
clusterName: "project-id/us-east1-d/cluster-name"
spec:
profile: asm # {"$openapi":"profilesetter"}
hub: asm # {"$openapi":"profilesetter"}
`,
expectedOpenAPI: `apiVersion: kustomization.dev/v1alpha1
kind: Kustomization
openAPI:
definitions:
io.k8s.cli.setters.profilesetter:
type: string
x-k8s-cli:
setter:
name: profilesetter
value: asm
`,
},
{
name: "do-not-delete-latest setters",
input: `
apiVersion: install.istio.io/v1alpha2
kind: IstioControlPlane
metadata:
clusterName: "project-id/us-east1-d/cluster-name"
spec:
profile: asm # {"$openapi":"profilesetter"}
hub: gcr.io/asm-testing
`,
failedSetters: map[string]error{},
failedSubst: map[string]error{},
openAPIFile: `apiVersion: kustomization.dev/v1alpha1
kind: Kustomization
openAPI:
definitions:
io.k8s.cli.setters.profilesetter:
type: string
x-k8s-cli:
setter:
name: profilesetter
value: asm
`,
expectedOutput: `apiVersion: install.istio.io/v1alpha2
kind: IstioControlPlane
metadata:
clusterName: "project-id/us-east1-d/cluster-name"
spec:
profile: asm # {"$openapi":"profilesetter"}
hub: gcr.io/asm-testing
`,
expectedOpenAPI: `apiVersion: kustomization.dev/v1alpha1
kind: Kustomization
openAPI:
definitions:
io.k8s.cli.setters.profilesetter:
type: string
x-k8s-cli:
setter:
name: profilesetter
value: asm
`,
},
{
name: "no-openAPI-file-error",
input: `
apiVersion: install.istio.io/v1alpha2
kind: IstioControlPlane
metadata:
clusterName: "project-id/us-east1-d/cluster-name"
spec:
profile: asm # {"type":"string","x-kustomize":{"setter":{"name":"profilesetter","value":"asm"}}}
hub: gcr.io/asm-testing
`,
err: "Krmfile:",
},
}
for i := range tests {
test := tests[i]
t.Run(test.name, func(t *testing.T) {
openAPIFileName := "Krmfile"
dir, err := ioutil.TempDir("", "")
if !assert.NoError(t, err) {
t.FailNow()
}
defer os.RemoveAll(dir)
err = ioutil.WriteFile(filepath.Join(dir, "deploy.yaml"), []byte(test.input), 0600)
if !assert.NoError(t, err) {
t.FailNow()
}
if test.openAPIFile != "" {
err = ioutil.WriteFile(filepath.Join(dir, openAPIFileName), []byte(test.openAPIFile), 0600)
if !assert.NoError(t, err) {
t.FailNow()
}
}
sf := SetterFixer{
PkgPath: dir,
DryRun: test.dryRun,
OpenAPIPath: filepath.Join(dir, "Krmfile"),
}
sfr, err := sf.FixV1Setters()
if test.err == "" {
if !assert.NoError(t, err) {
t.FailNow()
}
} else {
if !assert.Contains(t, err.Error(), test.err) {
t.FailNow()
}
return
}
if test.expectedOpenAPI != "" {
actualOpenAPI, err := ioutil.ReadFile(filepath.Join(dir, openAPIFileName))
if !assert.NoError(t, err) {
t.FailNow()
}
assert.Equal(t, test.expectedOpenAPI, string(actualOpenAPI))
}
assert.Equal(t, test.needFix, sfr.NeedFix)
assert.Equal(t, test.createdSetters, sfr.CreatedSetters)
assert.Equal(t, test.createdSubst, sfr.CreatedSubst)
assert.Equal(t, test.failedSubst, sfr.FailedSubst)
assert.Equal(t, test.failedSetters, sfr.FailedSetters)
})
}
}

View File

@@ -0,0 +1,115 @@
// Copyright 2019 The Kubernetes Authors.
// SPDX-License-Identifier: Apache-2.0
package fixsetters
import (
"strings"
"sigs.k8s.io/kustomize/kyaml/yaml"
)
var _ yaml.Filter = &upgradeV1Setters{}
// upgradeV1Setters looks up v1 setters in a Resource and upgrades
// all the setters related comments
type upgradeV1Setters struct {
// Name of the setter to lookup. Optional
Name string
// Setters is a list of setters that were found
Setters []setter
// Substitutions is a list of substitutions that were found
Substitutions []substitution
}
type substitution struct {
Name string
FieldVale string
Pattern string
}
type setter struct {
PartialFieldSetter
Description string
Type string
SetBy string
SubstName string
}
func (ls *upgradeV1Setters) Filter(object *yaml.RNode) (*yaml.RNode, error) {
switch object.YNode().Kind {
case yaml.DocumentNode:
// skip the document node
return ls.Filter(yaml.NewRNode(object.YNode().Content[0]))
case yaml.MappingNode:
return object, object.VisitFields(func(node *yaml.MapNode) error {
return node.Value.PipeE(ls)
})
case yaml.SequenceNode:
return object, object.VisitElements(func(node *yaml.RNode) error {
return node.PipeE(ls)
})
case yaml.ScalarNode:
return object, ls.lookupAndUpgrade(object)
default:
return object, nil
}
}
// lookupAndUpgrade finds any setters for a field and upgrades the setters comment
func (ls *upgradeV1Setters) lookupAndUpgrade(field *yaml.RNode) error {
// check if there is a substitution for this field
var fm = FieldMetaV1{}
if err := fm.UpgradeV1SetterComment(field); err != nil {
return err
}
if fm.Extensions.FieldSetter != nil {
if ls.Name != "" && ls.Name != fm.Extensions.FieldSetter.Name {
// skip this setter, it doesn't match the specified setter
return nil
}
// full setter
ls.Setters = append(ls.Setters, setter{
PartialFieldSetter: *fm.Extensions.FieldSetter,
Description: fm.Schema.Description,
Type: fm.Schema.Type[0],
SetBy: fm.Extensions.SetBy,
})
return nil
}
if len(fm.Extensions.PartialFieldSetters) > 0 {
substName := "subst"
filedValue := field.YNode().Value
pattern := filedValue
// derive substitution pattern from partial setters
for i := range fm.Extensions.PartialFieldSetters {
substName += "-" + fm.Extensions.PartialFieldSetters[i].Name
pattern = strings.Replace(pattern, fm.Extensions.PartialFieldSetters[i].Value, `${`+fm.Extensions.PartialFieldSetters[i].Name+"}", 1)
}
ls.Substitutions = append(ls.Substitutions, substitution{
Name: substName,
FieldVale: filedValue,
Pattern: pattern,
})
}
for i := range fm.Extensions.PartialFieldSetters {
if ls.Name != "" && ls.Name != fm.Extensions.PartialFieldSetters[i].Name {
// skip this setter
continue
}
ls.Setters = append(ls.Setters, setter{
PartialFieldSetter: fm.Extensions.PartialFieldSetters[i],
Description: fm.Schema.Description,
Type: fm.Schema.Type[0],
SetBy: fm.Extensions.SetBy,
})
}
return nil
}

View File

@@ -0,0 +1,81 @@
// Copyright 2019 The Kubernetes Authors.
// SPDX-License-Identifier: Apache-2.0
package fixsetters
import (
"sort"
"sigs.k8s.io/kustomize/kyaml/kio"
"sigs.k8s.io/kustomize/kyaml/yaml"
)
var _ kio.Filter = &UpgradeV1Setters{}
// UpgradeV1Setters identifies setters for a collection of Resources to upgrade
type UpgradeV1Setters struct {
// Name is the name of the setter to match. Optional.
Name string
// SetterCounts is populated by Filter and contains the count of fields matching each setter.
SetterCounts []setterCount
// Substitutions are groups of partial setters
Substitutions []substitution
}
// setterCount records the identified setters and number of fields matching those setters
type setterCount struct {
// Count is the number of substitutions possible to perform
Count int
// setter is the substitution found
setter
}
// Filter implements kio.Filter
func (l *UpgradeV1Setters) Filter(input []*yaml.RNode) ([]*yaml.RNode, error) {
setters := map[string]*setterCount{}
substitutions := map[string]*substitution{}
for i := range input {
// lookup substitutions for this object
ls := &upgradeV1Setters{Name: l.Name}
if err := input[i].PipeE(ls); err != nil {
return nil, err
}
// aggregate counts for each setter by name. takes the description and value from
// the first setter for each name encountered.
for j := range ls.Setters {
setter := ls.Setters[j]
curr, found := setters[setter.Name]
if !found {
curr = &setterCount{setter: setter}
setters[setter.Name] = curr
}
curr.Count++
}
for j := range ls.Substitutions {
subst := ls.Substitutions[j]
_, found := substitutions[subst.Name]
if !found {
substitutions[subst.Name] = &subst
}
}
}
// pull out and sort the results by setter name
for _, v := range setters {
l.SetterCounts = append(l.SetterCounts, *v)
}
for _, subst := range substitutions {
l.Substitutions = append(l.Substitutions, *subst)
}
sort.Slice(l.SetterCounts, func(i, j int) bool {
return l.SetterCounts[i].Name < l.SetterCounts[j].Name
})
return input, nil
}