kubernetes-sigs/kustomize
1
0
Fork 0
mirror of https://github.com/kubernetes-sigs/kustomize.git synced 2026-06-30 01:46:23 +00:00
Code Issues Packages Projects Releases 1 Wiki Activity

Merge pull request #2421 from Shell32-Natsu/2392-codeClean

Browse Source 
Remove inventory transformer
This commit is contained in:
Jeff Regan
2020-04-29 11:03:54 -07:00
committed by GitHub
parent c95a89c1f1 9195f05bba
commit d67c03af12
19 changed files with 20 additions and 1425 deletions
Download Patch File Download Diff File Expand all files Collapse all files

125
api/builtins/InventoryTransformer.go
View File

@@ -1,125 +0,0 @@
// Code generated by pluginator on InventoryTransformer; DO NOT EDIT.
// pluginator {unknown 1970-01-01T00:00:00Z }
package builtins
import (
"fmt"
"sigs.k8s.io/kustomize/api/hasher"
"sigs.k8s.io/kustomize/api/inventory"
"sigs.k8s.io/kustomize/api/kv"
"sigs.k8s.io/kustomize/api/resid"
"sigs.k8s.io/kustomize/api/resmap"
"sigs.k8s.io/kustomize/api/resource"
"sigs.k8s.io/kustomize/api/types"
"sigs.k8s.io/yaml"
)
type InventoryTransformerPlugin struct {
h *resmap.PluginHelpers
types.ObjectMeta `json:"metadata,omitempty" yaml:"metadata,omitempty" protobuf:"bytes,1,opt,name=metadata"`
Policy string `json:"policy,omitempty" yaml:"policy,omitempty"`
}
func (p *InventoryTransformerPlugin) Config(
h *resmap.PluginHelpers, c []byte) (err error) {
p.h = h
err = yaml.Unmarshal(c, p)
if err != nil {
return err
}
if p.Policy == "" {
p.Policy = types.GarbageIgnore.String()
}
if p.Policy != types.GarbageCollect.String() &&
p.Policy != types.GarbageIgnore.String() {
return fmt.Errorf(
"unrecognized garbagePolicy '%s'", p.Policy)
}
return nil
}
// Transform generates an inventory object from the input ResMap.
// This ConfigMap supports the pruning command in
// the client side tool proposed here:
// https://github.com/kubernetes/enhancements/pull/810
//
// The inventory data is written to the ConfigMap's
// annotations, rather than to the key-value pairs in
// the ConfigMap's data field, since
// 1. Keys in a ConfigMap's data field are too
// constrained for this purpose.
// 2. Using annotations allow any object to be used,
// not just a ConfigMap, should some other object
// (e.g. some App object) become more desirable
// for this purpose.
func (p *InventoryTransformerPlugin) Transform(m resmap.ResMap) error {
inv, h, err := makeInventory(m)
if err != nil {
return err
}
args := types.ConfigMapArgs{}
args.Name = p.Name
args.Namespace = p.Namespace
args.Options = &types.GeneratorOptions{
Annotations: map[string]string{inventory.HashAnnotation: h},
}
err = inv.UpdateAnnotations(args.Options.Annotations)
if err != nil {
return err
}
cm, err := p.h.ResmapFactory().RF().MakeConfigMap(
kv.NewLoader(p.h.Loader(), p.h.Validator()), &args)
if err != nil {
return err
}
if p.Policy == types.GarbageCollect.String() {
for _, byeBye := range m.AllIds() {
m.Remove(byeBye)
}
}
return m.Append(cm)
}
func makeInventory(m resmap.ResMap) (
inv *inventory.Inventory, hash string, err error) {
inv = inventory.NewInventory()
var keys []string
for _, r := range m.Resources() {
ns := r.GetNamespace()
item := resid.NewResIdWithNamespace(r.GetGvk(), r.GetName(), ns)
if _, ok := inv.Current[item]; ok {
return nil, "", fmt.Errorf(
"item '%v' already in inventory", item)
}
inv.Current[item], err = computeRefs(r, m)
if err != nil {
return nil, "", err
}
keys = append(keys, item.String())
}
h, err := hasher.SortArrayAndComputeHash(keys)
return inv, h, err
}
func computeRefs(
r *resource.Resource, m resmap.ResMap) (refs []resid.ResId, err error) {
for _, refid := range r.GetRefBy() {
ref, err := m.GetByCurrentId(refid)
if err != nil {
return nil, err
}
refs = append(
refs,
resid.NewResIdWithNamespace(
ref.GetGvk(), ref.GetName(), ref.GetNamespace()))
}
return
}
func NewInventoryTransformerPlugin() resmap.TransformerPlugin {
return &InventoryTransformerPlugin{}
}

23
api/internal/plugins/builtinhelpers/builtinplugintype_string.go
View File

@@ -13,21 +13,20 @@ func _() {
_ = x[ConfigMapGenerator-2]
_ = x[HashTransformer-3]
_ = x[ImageTagTransformer-4]
_ = x[InventoryTransformer-5]
_ = x[LabelTransformer-6]
_ = x[LegacyOrderTransformer-7]
_ = x[NamespaceTransformer-8]
_ = x[PatchJson6902Transformer-9]
_ = x[PatchStrategicMergeTransformer-10]
_ = x[PatchTransformer-11]
_ = x[PrefixSuffixTransformer-12]
_ = x[ReplicaCountTransformer-13]
_ = x[SecretGenerator-14]
_ = x[LabelTransformer-5]
_ = x[LegacyOrderTransformer-6]
_ = x[NamespaceTransformer-7]
_ = x[PatchJson6902Transformer-8]
_ = x[PatchStrategicMergeTransformer-9]
_ = x[PatchTransformer-10]
_ = x[PrefixSuffixTransformer-11]
_ = x[ReplicaCountTransformer-12]
_ = x[SecretGenerator-13]
}
const _BuiltinPluginType_name = "UnknownAnnotationsTransformerConfigMapGeneratorHashTransformerImageTagTransformerInventoryTransformerLabelTransformerLegacyOrderTransformerNamespaceTransformerPatchJson6902TransformerPatchStrategicMergeTransformerPatchTransformerPrefixSuffixTransformerReplicaCountTransformerSecretGenerator"
const _BuiltinPluginType_name = "UnknownAnnotationsTransformerConfigMapGeneratorHashTransformerImageTagTransformerLabelTransformerLegacyOrderTransformerNamespaceTransformerPatchJson6902TransformerPatchStrategicMergeTransformerPatchTransformerPrefixSuffixTransformerReplicaCountTransformerSecretGenerator"
var _BuiltinPluginType_index = [...]uint16{0, 7, 29, 47, 62, 81, 101, 117, 139, 159, 183, 213, 229, 252, 275, 290}
var _BuiltinPluginType_index = [...]uint16{0, 7, 29, 47, 62, 81, 97, 119, 139, 163, 193, 209, 232, 255, 270}
func (i BuiltinPluginType) String() string {
if i < 0 || i >= BuiltinPluginType(len(_BuiltinPluginType_index)-1) {

2
api/internal/plugins/builtinhelpers/builtins.go
View File

@@ -17,7 +17,6 @@ const (
ConfigMapGenerator
HashTransformer
ImageTagTransformer
InventoryTransformer
LabelTransformer
LegacyOrderTransformer
NamespaceTransformer
@@ -63,7 +62,6 @@ var TransformerFactories = map[BuiltinPluginType]func() resmap.TransformerPlugin
AnnotationsTransformer: builtins.NewAnnotationsTransformerPlugin,
HashTransformer: builtins.NewHashTransformerPlugin,
ImageTagTransformer: builtins.NewImageTagTransformerPlugin,
InventoryTransformer: builtins.NewInventoryTransformerPlugin,
LabelTransformer: builtins.NewLabelTransformerPlugin,
LegacyOrderTransformer: builtins.NewLegacyOrderTransformerPlugin,
NamespaceTransformer: builtins.NewNamespaceTransformerPlugin,

43
api/internal/target/kusttarget.go
View File

@@ -106,15 +106,10 @@ func unmarshal(y []byte, o interface{}) error {
// MakeCustomizedResMap creates a fully customized ResMap
// per the instructions contained in its kustomiztion instance.
func (kt *KustTarget) MakeCustomizedResMap() (resmap.ResMap, error) {
return kt.makeCustomizedResMap(types.GarbageIgnore)
return kt.makeCustomizedResMap()
}
func (kt *KustTarget) MakePruneConfigMap() (resmap.ResMap, error) {
return kt.makeCustomizedResMap(types.GarbageCollect)
}
func (kt *KustTarget) makeCustomizedResMap(
garbagePolicy types.GarbagePolicy) (resmap.ResMap, error) {
func (kt *KustTarget) makeCustomizedResMap() (resmap.ResMap, error) {
ra, err := kt.AccumulateTarget()
if err != nil {
return nil, err
@@ -141,11 +136,6 @@ func (kt *KustTarget) makeCustomizedResMap(
return nil, err
}
err = kt.computeInventory(ra, garbagePolicy)
if err != nil {
return nil, err
}
return ra.ResMap(), nil
}
@@ -159,35 +149,6 @@ func (kt *KustTarget) addHashesToNames(
return ra.Transform(p)
}
func (kt *KustTarget) computeInventory(
ra *accumulator.ResAccumulator, garbagePolicy types.GarbagePolicy) error {
inv := kt.kustomization.Inventory
if inv == nil {
return nil
}
if inv.Type != "ConfigMap" {
return fmt.Errorf("don't know how to do that")
}
if inv.ConfigMap.Namespace != kt.kustomization.Namespace {
return fmt.Errorf("namespace mismatch")
}
var c struct {
Policy string
types.ObjectMeta `json:"metadata,omitempty" yaml:"metadata,omitempty"`
}
c.Name = inv.ConfigMap.Name
c.Namespace = inv.ConfigMap.Namespace
c.Policy = garbagePolicy.String()
p := builtins.NewInventoryTransformerPlugin()
err := kt.configureBuiltinPlugin(p, c, builtinhelpers.InventoryTransformer)
if err != nil {
return err
}
return ra.Transform(p)
}
// AccumulateTarget returns a new ResAccumulator,
// holding customized resources and the data/rules used
// to do so. The name back references and vars are

12
api/inventory/constants.go
View File

@@ -1,12 +0,0 @@
// Copyright 2019 The Kubernetes Authors.
// SPDX-License-Identifier: Apache-2.0
package inventory
const (
// Annotation that contains the inventory content.
ContentAnnotation = "kustomize.config.k8s.io/Inventory"
// Annotation for inventory content hash.
HashAnnotation = "kustomize.config.k8s.io/InventoryHash"
)

235
api/inventory/inventory.go
View File

@@ -1,235 +0,0 @@
// Copyright 2019 The Kubernetes Authors.
// SPDX-License-Identifier: Apache-2.0
package inventory
import (
"encoding/json"
"sigs.k8s.io/kustomize/api/resid"
)
//Refs is a reference map. Each key is the id
//of a k8s resource, and each value is a list of
//object ids that refer back to the object in the
//key.
//For example, the key could correspond to a
//ConfigMap, and the list of values might include
//several different Deployments that get data from
//that ConfigMap (and thus refer to it).
//References are important in inventory management
//because one may not delete an object before all
//objects referencing it have been removed.
type Refs map[resid.ResId][]resid.ResId
func NewRefs() Refs {
return Refs{}
}
// Merge merges a Refs into an existing Refs
func (rf Refs) Merge(b Refs) Refs {
for key, value := range b {
_, ok := rf[key]
if ok {
rf[key] = append(rf[key], value...)
} else {
rf[key] = value
}
}
return rf
}
// removeIfContains removes the reference relationship
// a --> b
// from the Refs if it exists
func (rf Refs) RemoveIfContains(a, b resid.ResId) {
refs, ok := rf[a]
if !ok {
return
}
for i, ref := range refs {
if ref.Equals(b) {
rf[a] = append(refs[:i], refs[i+1:]...)
break
}
}
}
//Inventory is a an object intended for
//serialization into the annotations of a so-called
//apply-root object (a ConfigMap, an Application,
//etc.) living in the cluster. This apply-root
//object is written as part of an apply operation as
//a means to record overall cluster state changes.
//At the end of a successful apply, the "current"
//field in Inventory will be a map whose keys all
//correspond to an object in the cluster, and
//"previous" will be the previous such set (an empty
//set on the first apply).
//An Inventory allows the Prune method to work.
type Inventory struct {
Current Refs `json:"current,omitempty"`
Previous Refs `json:"previous,omitempty"`
}
// NewInventory returns an Inventory object
func NewInventory() *Inventory {
return &Inventory{
Current: NewRefs(),
Previous: NewRefs(),
}
}
// UpdateCurrent updates the Inventory given a
// new current Refs
// The existing Current refs is merged into
// the Previous refs
func (a *Inventory) UpdateCurrent(curref Refs) *Inventory {
if len(a.Previous) > 0 {
a.Previous.Merge(a.Current)
} else {
a.Previous = a.Current
}
a.Current = curref
return a
}
func (a *Inventory) removeNewlyOrphanedItemsFromPrevious() []resid.ResId {
var results []resid.ResId
for item, refs := range a.Previous {
if _, ok := a.Current[item]; ok {
delete(a.Previous, item)
continue
}
var newRefs []resid.ResId
toDelete := true
for _, ref := range refs {
if _, ok := a.Current[ref]; ok {
toDelete = false
newRefs = append(newRefs, ref)
}
}
if toDelete {
results = append(results, item)
delete(a.Previous, item)
} else {
a.Previous[item] = newRefs
}
}
return results
}
func (a *Inventory) removeOrphanedItemsFromPreviousThatAreNotInCurrent() []resid.ResId {
var results []resid.ResId
for item, refs := range a.Previous {
if _, ok := a.Current[item]; ok {
continue
}
if len(refs) == 0 {
results = append(results, item)
delete(a.Previous, item)
}
}
return results
}
func (a *Inventory) removeOrphanedItemsFromPreviousThatAreInCurrent() {
//Remove references from Previous that are already in Current refs
for item, refs := range a.Current {
for _, ref := range refs {
a.Previous.RemoveIfContains(item, ref)
}
}
//Remove items from Previous that are already in Current refs
for item, refs := range a.Previous {
if len(refs) == 0 {
if _, ok := a.Current[item]; ok {
delete(a.Previous, item)
}
}
}
}
// Prune computes the diff of Current refs and Previous refs
// and returns a list of Items that can be pruned.
// An item that can be pruned shows up only in Previous refs.
// Prune also updates the Previous refs with those items removed
func (a *Inventory) Prune() []resid.ResId {
a.removeOrphanedItemsFromPreviousThatAreInCurrent()
// These are candidates for deletion from the cluster.
removable1 := a.removeOrphanedItemsFromPreviousThatAreNotInCurrent()
removable2 := a.removeNewlyOrphanedItemsFromPrevious()
return append(removable1, removable2...)
}
// inventory is the internal type used for serialization
type inventory struct {
Current map[string][]resid.ResId `json:"current,omitempty"`
Previous map[string][]resid.ResId `json:"previous,omitempty"`
}
func (a *Inventory) toInternalType() inventory {
prev := map[string][]resid.ResId{}
curr := map[string][]resid.ResId{}
for id, refs := range a.Current {
curr[id.String()] = refs
}
for id, refs := range a.Previous {
prev[id.String()] = refs
}
return inventory{
Current: curr,
Previous: prev,
}
}
func (a *Inventory) fromInternalType(i *inventory) {
for s, refs := range i.Previous {
a.Previous[resid.FromString(s)] = refs
}
for s, refs := range i.Current {
a.Current[resid.FromString(s)] = refs
}
}
func (a *Inventory) marshal() ([]byte, error) {
return json.Marshal(a.toInternalType())
}
func (a *Inventory) unMarshal(data []byte) error {
inv := &inventory{
Current: map[string][]resid.ResId{},
Previous: map[string][]resid.ResId{},
}
err := json.Unmarshal(data, inv)
if err != nil {
return err
}
a.fromInternalType(inv)
return nil
}
// UpdateAnnotations update the annotation map
func (a *Inventory) UpdateAnnotations(annot map[string]string) error {
data, err := a.marshal()
if err != nil {
return err
}
annot[ContentAnnotation] = string(data)
return nil
}
// LoadFromAnnotation loads the Inventory date from the annotation map
func (a *Inventory) LoadFromAnnotation(annot map[string]string) error {
value, ok := annot[ContentAnnotation]
if ok {
return a.unMarshal([]byte(value))
}
return nil
}

60
api/inventory/inventory_test.go
View File

@@ -1,60 +0,0 @@
// Copyright 2019 The Kubernetes Authors.
// SPDX-License-Identifier: Apache-2.0
package inventory_test
import (
"testing"
. "sigs.k8s.io/kustomize/api/inventory"
"sigs.k8s.io/kustomize/api/resid"
)
func makeRefs() (Refs, Refs) {
a := resid.FromString("G1_V1_K1|ns1|nm1")
b := resid.FromString("G2_V2_K2|ns2|nm2")
c := resid.FromString("G3_V3_K3|ns3|nm3")
current := NewRefs()
current[a] = []resid.ResId{b, c}
current[b] = []resid.ResId{}
current[c] = []resid.ResId{}
newRefs := NewRefs()
newRefs[a] = []resid.ResId{b}
newRefs[b] = []resid.ResId{}
return current, newRefs
}
func TestInventory(t *testing.T) {
inventory := NewInventory()
curref, _ := makeRefs()
inventory.UpdateCurrent(curref)
if len(inventory.Current) != 3 {
t.Fatalf("not getting the correct inventory %v", inventory)
}
curref, newref := makeRefs()
inventory.UpdateCurrent(curref)
if len(inventory.Current) != 3 {
t.Fatalf("not getting the corrent inventory %v", inventory)
}
if len(inventory.Previous) != 3 {
t.Fatalf("not getting the corrent inventory %v", inventory)
}
items := inventory.Prune()
if len(items) != 0 {
t.Fatalf("not getting the corrent items %v", items)
}
if len(inventory.Previous) != 0 {
t.Fatalf("not getting the corrent inventory %v", inventory)
}
inventory.UpdateCurrent(newref)
items = inventory.Prune()
if len(items) != 1 {
t.Fatalf("not getting the corrent items %v", items)
}
if len(inventory.Previous) != 0 {
t.Fatalf("not getting the corrent inventory %v", inventory.Previous)
}
}

6
api/krusty/kustomizer.go
View File

@@ -71,11 +71,7 @@ func (b *Kustomizer) Run(path string) (resmap.ResMap, error) {
return nil, err
}
var m resmap.ResMap
if b.options.DoPrune {
m, err = kt.MakePruneConfigMap()
} else {
m, err = kt.MakeCustomizedResMap()
}
m, err = kt.MakeCustomizedResMap()
if err != nil {
return nil, err
}

163
api/krusty/pruneconfigmap_test.go
View File

@@ -1,163 +0,0 @@
// Copyright 2019 The Kubernetes Authors.
// SPDX-License-Identifier: Apache-2.0
package krusty_test
import (
"testing"
kusttest_test "sigs.k8s.io/kustomize/api/testutils/kusttest"
)
func TestPruneConfigMap(t *testing.T) {
th := kusttest_test.MakeHarness(t)
th.WriteK("/app/base", `
resources:
- deployment.yaml
- service.yaml
- secret.yaml
inventory:
type: ConfigMap
configMap:
name: haha
namespace: default
namePrefix: my-
namespace: default
`)
th.WriteF("/app/base/deployment.yaml", `
apiVersion: apps/v1beta2
kind: Deployment
metadata:
name: mysql
labels:
app: mysql
spec:
selector:
matchLabels:
app: mysql
strategy:
type: Recreate
template:
metadata:
labels:
app: mysql
spec:
containers:
- image: mysql:5.6
name: mysql
env:
- name: MYSQL_ROOT_PASSWORD
valueFrom:
secretKeyRef:
name: pass
key: password
ports:
- containerPort: 3306
name: mysql
volumeMounts:
- name: mysql-persistent-storage
mountPath: /var/lib/mysql
volumes:
- name: mysql-persistent-storage
emptyDir: {}
`)
th.WriteF("/app/base/service.yaml", `
apiVersion: v1
kind: Service
metadata:
name: mmmysql
labels:
app: mysql
spec:
ports:
- port: 3306
selector:
app: mysql
`)
th.WriteF("/app/base/secret.yaml", `
apiVersion: v1
kind: Secret
metadata:
name: pass
type: Opaque
data:
# Default password is "admin".
password: YWRtaW4=
username: jingfang
`)
m := th.Run("/app/base", th.MakeDefaultOptions())
//nolint
th.AssertActualEqualsExpected(m, `
apiVersion: apps/v1beta2
kind: Deployment
metadata:
labels:
app: mysql
name: my-mysql
namespace: default
spec:
selector:
matchLabels:
app: mysql
strategy:
type: Recreate
template:
metadata:
labels:
app: mysql
spec:
containers:
- env:
- name: MYSQL_ROOT_PASSWORD
valueFrom:
secretKeyRef:
key: password
name: my-pass
image: mysql:5.6
name: mysql
ports:
- containerPort: 3306
name: mysql
volumeMounts:
- mountPath: /var/lib/mysql
name: mysql-persistent-storage
volumes:
- emptyDir: {}
name: mysql-persistent-storage
---
apiVersion: v1
kind: Service
metadata:
labels:
app: mysql
name: my-mmmysql
namespace: default
spec:
ports:
- port: 3306
selector:
app: mysql
---
apiVersion: v1
data:
password: YWRtaW4=
username: jingfang
kind: Secret
metadata:
name: my-pass
namespace: default
type: Opaque
---
apiVersion: v1
kind: ConfigMap
metadata:
annotations:
kustomize.config.k8s.io/Inventory: '{"current":{"apps_v1beta2_Deployment|default|my-mysql":null,"~G_v1_Secret|default|my-pass":[{"group":"apps","version":"v1beta2","kind":"Deployment","name":"my-mysql","namespace":"default"}],"~G_v1_Service|default|my-mmmysql":null}}'
kustomize.config.k8s.io/InventoryHash: kd67f7ht8t
name: haha
namespace: default
`)
}

12
api/types/garbagepolicy.go
View File

@@ -1,12 +0,0 @@
// Copyright 2019 The Kubernetes Authors.
// SPDX-License-Identifier: Apache-2.0
package types
//go:generate stringer -type=GarbagePolicy
type GarbagePolicy int
const (
GarbageIgnore GarbagePolicy = iota + 1
GarbageCollect
)

25
api/types/garbagepolicy_string.go
View File

@@ -1,25 +0,0 @@
// Code generated by "stringer -type=GarbagePolicy"; DO NOT EDIT.
package types
import "strconv"
func _() {
// An "invalid array index" compiler error signifies that the constant values have changed.
// Re-run the stringer command to generate them again.
var x [1]struct{}
_ = x[GarbageIgnore-1]
_ = x[GarbageCollect-2]
}
const _GarbagePolicy_name = "GarbageIgnoreGarbageCollect"
var _GarbagePolicy_index = [...]uint8{0, 13, 27}
func (i GarbagePolicy) String() string {
i -= 1
if i < 0 || i >= GarbagePolicy(len(_GarbagePolicy_index)-1) {
return "GarbagePolicy(" + strconv.FormatInt(int64(i+1), 10) + ")"
}
return _GarbagePolicy_name[_GarbagePolicy_index[i]:_GarbagePolicy_index[i+1]]
}