kubernetes-sigs/kustomize
1
0
Fork 0
mirror of https://github.com/kubernetes-sigs/kustomize.git synced 2026-06-30 01:46:23 +00:00
Code Issues Packages Projects Releases 1 Wiki Activity

Remove inventory transformer #2392

Browse Source
This commit is contained in:
Donny Xia
2020-04-22 16:56:16 -07:00
parent c771f24626
commit 63f7495e88
17 changed files with 14 additions and 1422 deletions
Download Patch File Download Diff File Expand all files Collapse all files

125
api/builtins/InventoryTransformer.go
View File

@@ -1,125 +0,0 @@
// Code generated by pluginator on InventoryTransformer; DO NOT EDIT.
// pluginator {unknown 1970-01-01T00:00:00Z }
package builtins
import (
"fmt"
"sigs.k8s.io/kustomize/api/hasher"
"sigs.k8s.io/kustomize/api/inventory"
"sigs.k8s.io/kustomize/api/kv"
"sigs.k8s.io/kustomize/api/resid"
"sigs.k8s.io/kustomize/api/resmap"
"sigs.k8s.io/kustomize/api/resource"
"sigs.k8s.io/kustomize/api/types"
"sigs.k8s.io/yaml"
)
type InventoryTransformerPlugin struct {
h *resmap.PluginHelpers
types.ObjectMeta `json:"metadata,omitempty" yaml:"metadata,omitempty" protobuf:"bytes,1,opt,name=metadata"`
Policy string `json:"policy,omitempty" yaml:"policy,omitempty"`
}
func (p *InventoryTransformerPlugin) Config(
h *resmap.PluginHelpers, c []byte) (err error) {
p.h = h
err = yaml.Unmarshal(c, p)
if err != nil {
return err
}
if p.Policy == "" {
p.Policy = types.GarbageIgnore.String()
}
if p.Policy != types.GarbageCollect.String() &&
p.Policy != types.GarbageIgnore.String() {
return fmt.Errorf(
"unrecognized garbagePolicy '%s'", p.Policy)
}
return nil
}
// Transform generates an inventory object from the input ResMap.
// This ConfigMap supports the pruning command in
// the client side tool proposed here:
// https://github.com/kubernetes/enhancements/pull/810
//
// The inventory data is written to the ConfigMap's
// annotations, rather than to the key-value pairs in
// the ConfigMap's data field, since
// 1. Keys in a ConfigMap's data field are too
// constrained for this purpose.
// 2. Using annotations allow any object to be used,
// not just a ConfigMap, should some other object
// (e.g. some App object) become more desirable
// for this purpose.
func (p *InventoryTransformerPlugin) Transform(m resmap.ResMap) error {
inv, h, err := makeInventory(m)
if err != nil {
return err
}
args := types.ConfigMapArgs{}
args.Name = p.Name
args.Namespace = p.Namespace
args.Options = &types.GeneratorOptions{
Annotations: map[string]string{inventory.HashAnnotation: h},
}
err = inv.UpdateAnnotations(args.Options.Annotations)
if err != nil {
return err
}
cm, err := p.h.ResmapFactory().RF().MakeConfigMap(
kv.NewLoader(p.h.Loader(), p.h.Validator()), &args)
if err != nil {
return err
}
if p.Policy == types.GarbageCollect.String() {
for _, byeBye := range m.AllIds() {
m.Remove(byeBye)
}
}
return m.Append(cm)
}
func makeInventory(m resmap.ResMap) (
inv *inventory.Inventory, hash string, err error) {
inv = inventory.NewInventory()
var keys []string
for _, r := range m.Resources() {
ns := r.GetNamespace()
item := resid.NewResIdWithNamespace(r.GetGvk(), r.GetName(), ns)
if _, ok := inv.Current[item]; ok {
return nil, "", fmt.Errorf(
"item '%v' already in inventory", item)
}
inv.Current[item], err = computeRefs(r, m)
if err != nil {
return nil, "", err
}
keys = append(keys, item.String())
}
h, err := hasher.SortArrayAndComputeHash(keys)
return inv, h, err
}
func computeRefs(
r *resource.Resource, m resmap.ResMap) (refs []resid.ResId, err error) {
for _, refid := range r.GetRefBy() {
ref, err := m.GetByCurrentId(refid)
if err != nil {
return nil, err
}
refs = append(
refs,
resid.NewResIdWithNamespace(
ref.GetGvk(), ref.GetName(), ref.GetNamespace()))
}
return
}
func NewInventoryTransformerPlugin() resmap.TransformerPlugin {
return &InventoryTransformerPlugin{}
}

23
api/internal/plugins/builtinhelpers/builtinplugintype_string.go
View File

@@ -13,21 +13,20 @@ func _() {
_ = x[ConfigMapGenerator-2]
_ = x[HashTransformer-3]
_ = x[ImageTagTransformer-4]
_ = x[InventoryTransformer-5]
_ = x[LabelTransformer-6]
_ = x[LegacyOrderTransformer-7]
_ = x[NamespaceTransformer-8]
_ = x[PatchJson6902Transformer-9]
_ = x[PatchStrategicMergeTransformer-10]
_ = x[PatchTransformer-11]
_ = x[PrefixSuffixTransformer-12]
_ = x[ReplicaCountTransformer-13]
_ = x[SecretGenerator-14]
_ = x[LabelTransformer-5]
_ = x[LegacyOrderTransformer-6]
_ = x[NamespaceTransformer-7]
_ = x[PatchJson6902Transformer-8]
_ = x[PatchStrategicMergeTransformer-9]
_ = x[PatchTransformer-10]
_ = x[PrefixSuffixTransformer-11]
_ = x[ReplicaCountTransformer-12]
_ = x[SecretGenerator-13]
}
const _BuiltinPluginType_name = "UnknownAnnotationsTransformerConfigMapGeneratorHashTransformerImageTagTransformerInventoryTransformerLabelTransformerLegacyOrderTransformerNamespaceTransformerPatchJson6902TransformerPatchStrategicMergeTransformerPatchTransformerPrefixSuffixTransformerReplicaCountTransformerSecretGenerator"
const _BuiltinPluginType_name = "UnknownAnnotationsTransformerConfigMapGeneratorHashTransformerImageTagTransformerLabelTransformerLegacyOrderTransformerNamespaceTransformerPatchJson6902TransformerPatchStrategicMergeTransformerPatchTransformerPrefixSuffixTransformerReplicaCountTransformerSecretGenerator"
var _BuiltinPluginType_index = [...]uint16{0, 7, 29, 47, 62, 81, 101, 117, 139, 159, 183, 213, 229, 252, 275, 290}
var _BuiltinPluginType_index = [...]uint16{0, 7, 29, 47, 62, 81, 97, 119, 139, 163, 193, 209, 232, 255, 270}
func (i BuiltinPluginType) String() string {
if i < 0 || i >= BuiltinPluginType(len(_BuiltinPluginType_index)-1) {

2
api/internal/plugins/builtinhelpers/builtins.go
View File

@@ -17,7 +17,6 @@ const (
ConfigMapGenerator
HashTransformer
ImageTagTransformer
InventoryTransformer
LabelTransformer
LegacyOrderTransformer
NamespaceTransformer
@@ -63,7 +62,6 @@ var TransformerFactories = map[BuiltinPluginType]func() resmap.TransformerPlugin
AnnotationsTransformer: builtins.NewAnnotationsTransformerPlugin,
HashTransformer: builtins.NewHashTransformerPlugin,
ImageTagTransformer: builtins.NewImageTagTransformerPlugin,
InventoryTransformer: builtins.NewInventoryTransformerPlugin,
LabelTransformer: builtins.NewLabelTransformerPlugin,
LegacyOrderTransformer: builtins.NewLegacyOrderTransformerPlugin,
NamespaceTransformer: builtins.NewNamespaceTransformerPlugin,

43
api/internal/target/kusttarget.go
View File

@@ -106,15 +106,10 @@ func unmarshal(y []byte, o interface{}) error {
// MakeCustomizedResMap creates a fully customized ResMap
// per the instructions contained in its kustomiztion instance.
func (kt *KustTarget) MakeCustomizedResMap() (resmap.ResMap, error) {
return kt.makeCustomizedResMap(types.GarbageIgnore)
return kt.makeCustomizedResMap()
}
func (kt *KustTarget) MakePruneConfigMap() (resmap.ResMap, error) {
return kt.makeCustomizedResMap(types.GarbageCollect)
}
func (kt *KustTarget) makeCustomizedResMap(
garbagePolicy types.GarbagePolicy) (resmap.ResMap, error) {
func (kt *KustTarget) makeCustomizedResMap() (resmap.ResMap, error) {
ra, err := kt.AccumulateTarget()
if err != nil {
return nil, err
@@ -141,11 +136,6 @@ func (kt *KustTarget) makeCustomizedResMap(
return nil, err
}
err = kt.computeInventory(ra, garbagePolicy)
if err != nil {
return nil, err
}
return ra.ResMap(), nil
}
@@ -159,35 +149,6 @@ func (kt *KustTarget) addHashesToNames(
return ra.Transform(p)
}
func (kt *KustTarget) computeInventory(
ra *accumulator.ResAccumulator, garbagePolicy types.GarbagePolicy) error {
inv := kt.kustomization.Inventory
if inv == nil {
return nil
}
if inv.Type != "ConfigMap" {
return fmt.Errorf("don't know how to do that")
}
if inv.ConfigMap.Namespace != kt.kustomization.Namespace {
return fmt.Errorf("namespace mismatch")
}
var c struct {
Policy string
types.ObjectMeta `json:"metadata,omitempty" yaml:"metadata,omitempty"`
}
c.Name = inv.ConfigMap.Name
c.Namespace = inv.ConfigMap.Namespace
c.Policy = garbagePolicy.String()
p := builtins.NewInventoryTransformerPlugin()
err := kt.configureBuiltinPlugin(p, c, builtinhelpers.InventoryTransformer)
if err != nil {
return err
}
return ra.Transform(p)
}
// AccumulateTarget returns a new ResAccumulator,
// holding customized resources and the data/rules used
// to do so. The name back references and vars are

12
api/inventory/constants.go
View File

@@ -1,12 +0,0 @@
// Copyright 2019 The Kubernetes Authors.
// SPDX-License-Identifier: Apache-2.0
package inventory
const (
// Annotation that contains the inventory content.
ContentAnnotation = "kustomize.config.k8s.io/Inventory"
// Annotation for inventory content hash.
HashAnnotation = "kustomize.config.k8s.io/InventoryHash"
)

235
api/inventory/inventory.go
View File

@@ -1,235 +0,0 @@
// Copyright 2019 The Kubernetes Authors.
// SPDX-License-Identifier: Apache-2.0
package inventory
import (
"encoding/json"
"sigs.k8s.io/kustomize/api/resid"
)
//Refs is a reference map. Each key is the id
//of a k8s resource, and each value is a list of
//object ids that refer back to the object in the
//key.
//For example, the key could correspond to a
//ConfigMap, and the list of values might include
//several different Deployments that get data from
//that ConfigMap (and thus refer to it).
//References are important in inventory management
//because one may not delete an object before all
//objects referencing it have been removed.
type Refs map[resid.ResId][]resid.ResId
func NewRefs() Refs {
return Refs{}
}
// Merge merges a Refs into an existing Refs
func (rf Refs) Merge(b Refs) Refs {
for key, value := range b {
_, ok := rf[key]
if ok {
rf[key] = append(rf[key], value...)
} else {
rf[key] = value
}
}
return rf
}
// removeIfContains removes the reference relationship
// a --> b
// from the Refs if it exists
func (rf Refs) RemoveIfContains(a, b resid.ResId) {
refs, ok := rf[a]
if !ok {
return
}
for i, ref := range refs {
if ref.Equals(b) {
rf[a] = append(refs[:i], refs[i+1:]...)
break
}
}
}
//Inventory is a an object intended for
//serialization into the annotations of a so-called
//apply-root object (a ConfigMap, an Application,
//etc.) living in the cluster. This apply-root
//object is written as part of an apply operation as
//a means to record overall cluster state changes.
//At the end of a successful apply, the "current"
//field in Inventory will be a map whose keys all
//correspond to an object in the cluster, and
//"previous" will be the previous such set (an empty
//set on the first apply).
//An Inventory allows the Prune method to work.
type Inventory struct {
Current Refs `json:"current,omitempty"`
Previous Refs `json:"previous,omitempty"`
}
// NewInventory returns an Inventory object
func NewInventory() *Inventory {
return &Inventory{
Current: NewRefs(),
Previous: NewRefs(),
}
}
// UpdateCurrent updates the Inventory given a
// new current Refs
// The existing Current refs is merged into
// the Previous refs
func (a *Inventory) UpdateCurrent(curref Refs) *Inventory {
if len(a.Previous) > 0 {
a.Previous.Merge(a.Current)
} else {
a.Previous = a.Current
}
a.Current = curref
return a
}
func (a *Inventory) removeNewlyOrphanedItemsFromPrevious() []resid.ResId {
var results []resid.ResId
for item, refs := range a.Previous {
if _, ok := a.Current[item]; ok {
delete(a.Previous, item)
continue
}
var newRefs []resid.ResId
toDelete := true
for _, ref := range refs {
if _, ok := a.Current[ref]; ok {
toDelete = false
newRefs = append(newRefs, ref)
}
}
if toDelete {
results = append(results, item)
delete(a.Previous, item)
} else {
a.Previous[item] = newRefs
}
}
return results
}
func (a *Inventory) removeOrphanedItemsFromPreviousThatAreNotInCurrent() []resid.ResId {
var results []resid.ResId
for item, refs := range a.Previous {
if _, ok := a.Current[item]; ok {
continue
}
if len(refs) == 0 {
results = append(results, item)
delete(a.Previous, item)
}
}
return results
}
func (a *Inventory) removeOrphanedItemsFromPreviousThatAreInCurrent() {
//Remove references from Previous that are already in Current refs
for item, refs := range a.Current {
for _, ref := range refs {
a.Previous.RemoveIfContains(item, ref)
}
}
//Remove items from Previous that are already in Current refs
for item, refs := range a.Previous {
if len(refs) == 0 {
if _, ok := a.Current[item]; ok {
delete(a.Previous, item)
}
}
}
}
// Prune computes the diff of Current refs and Previous refs
// and returns a list of Items that can be pruned.
// An item that can be pruned shows up only in Previous refs.
// Prune also updates the Previous refs with those items removed
func (a *Inventory) Prune() []resid.ResId {
a.removeOrphanedItemsFromPreviousThatAreInCurrent()
// These are candidates for deletion from the cluster.
removable1 := a.removeOrphanedItemsFromPreviousThatAreNotInCurrent()
removable2 := a.removeNewlyOrphanedItemsFromPrevious()
return append(removable1, removable2...)
}
// inventory is the internal type used for serialization
type inventory struct {
Current map[string][]resid.ResId `json:"current,omitempty"`
Previous map[string][]resid.ResId `json:"previous,omitempty"`
}
func (a *Inventory) toInternalType() inventory {
prev := map[string][]resid.ResId{}
curr := map[string][]resid.ResId{}
for id, refs := range a.Current {
curr[id.String()] = refs
}
for id, refs := range a.Previous {
prev[id.String()] = refs
}
return inventory{
Current: curr,
Previous: prev,
}
}
func (a *Inventory) fromInternalType(i *inventory) {
for s, refs := range i.Previous {
a.Previous[resid.FromString(s)] = refs
}
for s, refs := range i.Current {
a.Current[resid.FromString(s)] = refs
}
}
func (a *Inventory) marshal() ([]byte, error) {
return json.Marshal(a.toInternalType())
}
func (a *Inventory) unMarshal(data []byte) error {
inv := &inventory{
Current: map[string][]resid.ResId{},
Previous: map[string][]resid.ResId{},
}
err := json.Unmarshal(data, inv)
if err != nil {
return err
}
a.fromInternalType(inv)
return nil
}
// UpdateAnnotations update the annotation map
func (a *Inventory) UpdateAnnotations(annot map[string]string) error {
data, err := a.marshal()
if err != nil {
return err
}
annot[ContentAnnotation] = string(data)
return nil
}
// LoadFromAnnotation loads the Inventory date from the annotation map
func (a *Inventory) LoadFromAnnotation(annot map[string]string) error {
value, ok := annot[ContentAnnotation]
if ok {
return a.unMarshal([]byte(value))
}
return nil
}

60
api/inventory/inventory_test.go
View File

@@ -1,60 +0,0 @@
// Copyright 2019 The Kubernetes Authors.
// SPDX-License-Identifier: Apache-2.0
package inventory_test
import (
"testing"
. "sigs.k8s.io/kustomize/api/inventory"
"sigs.k8s.io/kustomize/api/resid"
)
func makeRefs() (Refs, Refs) {
a := resid.FromString("G1_V1_K1|ns1|nm1")
b := resid.FromString("G2_V2_K2|ns2|nm2")
c := resid.FromString("G3_V3_K3|ns3|nm3")
current := NewRefs()
current[a] = []resid.ResId{b, c}
current[b] = []resid.ResId{}
current[c] = []resid.ResId{}
newRefs := NewRefs()
newRefs[a] = []resid.ResId{b}
newRefs[b] = []resid.ResId{}
return current, newRefs
}
func TestInventory(t *testing.T) {
inventory := NewInventory()
curref, _ := makeRefs()
inventory.UpdateCurrent(curref)
if len(inventory.Current) != 3 {
t.Fatalf("not getting the correct inventory %v", inventory)
}
curref, newref := makeRefs()
inventory.UpdateCurrent(curref)
if len(inventory.Current) != 3 {
t.Fatalf("not getting the corrent inventory %v", inventory)
}
if len(inventory.Previous) != 3 {
t.Fatalf("not getting the corrent inventory %v", inventory)
}
items := inventory.Prune()
if len(items) != 0 {
t.Fatalf("not getting the corrent items %v", items)
}
if len(inventory.Previous) != 0 {
t.Fatalf("not getting the corrent inventory %v", inventory)
}
inventory.UpdateCurrent(newref)
items = inventory.Prune()
if len(items) != 1 {
t.Fatalf("not getting the corrent items %v", items)
}
if len(inventory.Previous) != 0 {
t.Fatalf("not getting the corrent inventory %v", inventory.Previous)
}
}

6
api/krusty/kustomizer.go
View File

@@ -71,11 +71,7 @@ func (b *Kustomizer) Run(path string) (resmap.ResMap, error) {
return nil, err
}
var m resmap.ResMap
if b.options.DoPrune {
m, err = kt.MakePruneConfigMap()
} else {
m, err = kt.MakeCustomizedResMap()
}
m, err = kt.MakeCustomizedResMap()
if err != nil {
return nil, err
}

163
api/krusty/pruneconfigmap_test.go
View File

@@ -1,163 +0,0 @@
// Copyright 2019 The Kubernetes Authors.
// SPDX-License-Identifier: Apache-2.0
package krusty_test
import (
"testing"
kusttest_test "sigs.k8s.io/kustomize/api/testutils/kusttest"
)
func TestPruneConfigMap(t *testing.T) {
th := kusttest_test.MakeHarness(t)
th.WriteK("/app/base", `
resources:
- deployment.yaml
- service.yaml
- secret.yaml
inventory:
type: ConfigMap
configMap:
name: haha
namespace: default
namePrefix: my-
namespace: default
`)
th.WriteF("/app/base/deployment.yaml", `
apiVersion: apps/v1beta2
kind: Deployment
metadata:
name: mysql
labels:
app: mysql
spec:
selector:
matchLabels:
app: mysql
strategy:
type: Recreate
template:
metadata:
labels:
app: mysql
spec:
containers:
- image: mysql:5.6
name: mysql
env:
- name: MYSQL_ROOT_PASSWORD
valueFrom:
secretKeyRef:
name: pass
key: password
ports:
- containerPort: 3306
name: mysql
volumeMounts:
- name: mysql-persistent-storage
mountPath: /var/lib/mysql
volumes:
- name: mysql-persistent-storage
emptyDir: {}
`)
th.WriteF("/app/base/service.yaml", `
apiVersion: v1
kind: Service
metadata:
name: mmmysql
labels:
app: mysql
spec:
ports:
- port: 3306
selector:
app: mysql
`)
th.WriteF("/app/base/secret.yaml", `
apiVersion: v1
kind: Secret
metadata:
name: pass
type: Opaque
data:
# Default password is "admin".
password: YWRtaW4=
username: jingfang
`)
m := th.Run("/app/base", th.MakeDefaultOptions())
//nolint
th.AssertActualEqualsExpected(m, `
apiVersion: apps/v1beta2
kind: Deployment
metadata:
labels:
app: mysql
name: my-mysql
namespace: default
spec:
selector:
matchLabels:
app: mysql
strategy:
type: Recreate
template:
metadata:
labels:
app: mysql
spec:
containers:
- env:
- name: MYSQL_ROOT_PASSWORD
valueFrom:
secretKeyRef:
key: password
name: my-pass
image: mysql:5.6
name: mysql
ports:
- containerPort: 3306
name: mysql
volumeMounts:
- mountPath: /var/lib/mysql
name: mysql-persistent-storage
volumes:
- emptyDir: {}
name: mysql-persistent-storage
---
apiVersion: v1
kind: Service
metadata:
labels:
app: mysql
name: my-mmmysql
namespace: default
spec:
ports:
- port: 3306
selector:
app: mysql
---
apiVersion: v1
data:
password: YWRtaW4=
username: jingfang
kind: Secret
metadata:
name: my-pass
namespace: default
type: Opaque
---
apiVersion: v1
kind: ConfigMap
metadata:
annotations:
kustomize.config.k8s.io/Inventory: '{"current":{"apps_v1beta2_Deployment|default|my-mysql":null,"~G_v1_Secret|default|my-pass":[{"group":"apps","version":"v1beta2","kind":"Deployment","name":"my-mysql","namespace":"default"}],"~G_v1_Service|default|my-mmmysql":null}}'
kustomize.config.k8s.io/InventoryHash: kd67f7ht8t
name: haha
namespace: default
`)
}

12
api/types/garbagepolicy.go
View File

@@ -1,12 +0,0 @@
// Copyright 2019 The Kubernetes Authors.
// SPDX-License-Identifier: Apache-2.0
package types
//go:generate stringer -type=GarbagePolicy
type GarbagePolicy int
const (
GarbageIgnore GarbagePolicy = iota + 1
GarbageCollect
)

25
api/types/garbagepolicy_string.go
View File

@@ -1,25 +0,0 @@
// Code generated by "stringer -type=GarbagePolicy"; DO NOT EDIT.
package types
import "strconv"
func _() {
// An "invalid array index" compiler error signifies that the constant values have changed.
// Re-run the stringer command to generate them again.
var x [1]struct{}
_ = x[GarbageIgnore-1]
_ = x[GarbageCollect-2]
}
const _GarbagePolicy_name = "GarbageIgnoreGarbageCollect"
var _GarbagePolicy_index = [...]uint8{0, 13, 27}
func (i GarbagePolicy) String() string {
i -= 1
if i < 0 || i >= GarbagePolicy(len(_GarbagePolicy_index)-1) {
return "GarbagePolicy(" + strconv.FormatInt(int64(i+1), 10) + ")"
}
return _GarbagePolicy_name[_GarbagePolicy_index[i]:_GarbagePolicy_index[i+1]]
}