Merge pull request #1006 from Liujingfang1/inventory

add inventory package and refactor inventory transformer

k8sdeps/transformer/inventory/inventory.go

@@ -21,6 +21,7 @@ import (
 	"sigs.k8s.io/kustomize/k8sdeps/kunstruct"
 	"sigs.k8s.io/kustomize/k8sdeps/transformer/hash"
 	"sigs.k8s.io/kustomize/pkg/gvk"
+	"sigs.k8s.io/kustomize/pkg/inventory"
 	"sigs.k8s.io/kustomize/pkg/resid"
 	"sigs.k8s.io/kustomize/pkg/resmap"
 	"sigs.k8s.io/kustomize/pkg/resource"
@@ -28,10 +29,7 @@ import (
 	"sigs.k8s.io/kustomize/pkg/types"
 )
 
-//const PruneAnnotation = "kustomize.k8s.io/PruneRevision"
-const PruneAnnotation = "current"
-
-// inventoryTransformer compute the ConfigMap used in prune
+// inventoryTransformer compute the inventory object used in prune
 type inventoryTransformer struct {
 	append      bool
 	cmName      string
@@ -52,20 +50,29 @@ func NewInventoryTransformer(p *types.Inventory, namespace string, append bool)
 	}
 }
 
-// Transform generates an inventory ConfigMap based on the input ResMap.
-// this tranformer doesn't change existing resources -
+// Transform generates an inventory object based on the input ResMap.
+// this transformer doesn't change existing resources -
 // it just visits resources and accumulates information to make a new ConfigMap.
 // The prune ConfigMap is used to support the pruning command in the client side tool,
 // which is proposed in https://github.com/kubernetes/enhancements/pull/810
+// The inventory data is written to annotation since
+//   1. The key in data field is constrained and couldn't include arbitrary letters
+//   2. The annotation can be put into any kind of objects
 func (o *inventoryTransformer) Transform(m resmap.ResMap) error {
+	invty := inventory.NewInventory()
 	var keys []string
 	for _, r := range m {
-		s := r.PruneString()
-		keys = append(keys, s)
+		ns, _ := r.GetFieldValue("metadata.namespace")
+		item := resid.NewItemId(r.GetGvk(), ns, r.GetName())
+		var refs []resid.ItemId
+
 		for _, refid := range r.GetRefBy() {
 			ref := m[refid]
-			keys = append(keys, s+"---"+ref.PruneString())
+			ns, _ := ref.GetFieldValue("metadata.namespace")
+			refs = append(refs, resid.NewItemId(ref.GetGvk(), ns, ref.GetName()))
 		}
+		invty.Current[item] = refs
+		keys = append(keys, item.String())
 	}
 	h, err := hash.SortArrayAndComputeHash(keys)
 	if err != nil {
@@ -75,14 +82,14 @@ func (o *inventoryTransformer) Transform(m resmap.ResMap) error {
 	args := &types.ConfigMapArgs{}
 	args.Name = o.cmName
 	args.Namespace = o.cmNamespace
-	for _, key := range keys {
-		args.LiteralSources = append(args.LiteralSources,
-			key+"="+h)
-	}
 	opts := &types.GeneratorOptions{
 		Annotations: make(map[string]string),
 	}
-	opts.Annotations[PruneAnnotation] = h
+	opts.Annotations[inventory.InventoryHashAnnotation] = h
+	err = invty.UpdateAnnotations(opts.Annotations)
+	if err != nil {
+		return err
+	}
 
 	kf := kunstruct.NewKunstructuredFactoryImpl()
 	k, err := kf.MakeConfigMap(nil, opts, args)

k8sdeps/transformer/inventory/inventory_test.go

@@ -107,12 +107,18 @@ func TestInventoryTransformer(t *testing.T) {
 	rf := resource.NewFactory(
 		kunstruct.NewKunstructuredFactoryImpl())
 
-	// hash is derived based on all keys in the ConfigMap data field.
+	// hash is derived based on all keys in the Inventory
 	// It is added to annotations as
-	//   current: hash
+	//   kustomize.config.k8s.io/InventoryHash: hash
 	// When seeing the same annotation, prune binary assumes no
 	// clean up is needed
-	hash := "k777d7h45b"
+	hash := "h44788gt7g"
+
+	// inventory is the derived json string for an Inventory object
+	// It is added to annotations as
+	// kustomize.config.k8s.io/Inventory: inventory
+	inventory := "{\"current\":{\"apps_v1_Deployment|~X|deploy1\":null,\"~G_v1_ConfigMap|~X|cm1\":[{\"group\":\"apps\",\"version\":\"v1\",\"kind\":\"Deployment\",\"name\":\"deploy1\"}],\"~G_v1_Secret|~X|secret1\":[{\"group\":\"apps\",\"version\":\"v1\",\"kind\":\"Deployment\",\"name\":\"deploy1\"}]}}" // nolint
+
 	//  This is the root or inventory object which tracks all
 	// the applied resources - this is the thing we expect the transformer to create.
 	pruneMap := rf.FromMap(
@@ -123,16 +129,10 @@ func TestInventoryTransformer(t *testing.T) {
 				"name":      "pruneCM",
 				"namespace": "default",
 				"annotations": map[string]interface{}{
-					"current": hash,
+					"kustomize.config.k8s.io/Inventory":     inventory,
+					"kustomize.config.k8s.io/InventoryHash": hash,
 				},
 			},
-			"data": map[string]interface{}{
-				"_ConfigMap__cm1":                             hash,
-				"_Secret__secret1":                            hash,
-				"apps_Deployment__deploy1":                    hash,
-				"_ConfigMap__cm1---apps_Deployment__deploy1":  hash,
-				"_Secret__secret1---apps_Deployment__deploy1": hash,
-			},
 		})
 	expected := resmap.ResMap{
 		resid.NewResIdWithPrefixNamespace(cmap, "pruneCM", "", "default"): pruneMap,

pkg/inventory/constants.go

@@ -0,0 +1,25 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package inventory
+
+const (
+	// the annotation for inventory hash
+	InventoryHashAnnotation = "kustomize.config.k8s.io/InventoryHash"
+
+	// the annotation that contains the inventory information
+	InventoryAnnotation = "kustomize.config.k8s.io/Inventory"
+)

pkg/inventory/inventory.go

@@ -0,0 +1,248 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package inventory
+
+import (
+	"encoding/json"
+
+	"sigs.k8s.io/kustomize/pkg/resid"
+)
+
+//Refs is a reference map.  Each key is the id
+//of a k8s resource, and each value is a list of
+//object ids that refer back to the object in the
+//key.
+
+//For example, the key could correspond to a
+//ConfigMap, and the list of values might include
+//several different Deployments that get data from
+//that ConfigMap (and thus refer to it).
+
+//References are important in inventory management
+//because one may not delete an object before all
+//objects referencing it have been removed.
+type Refs map[resid.ItemId][]resid.ItemId
+
+func NewRefs() Refs {
+	return Refs{}
+}
+
+// Merge merges a Refs into an existing Refs
+func (rf Refs) Merge(b Refs) Refs {
+	for key, value := range b {
+		_, ok := rf[key]
+		if ok {
+			rf[key] = append(rf[key], value...)
+		} else {
+			rf[key] = value
+		}
+	}
+	return rf
+}
+
+// removeIfContains removes the reference relationship
+//  a --> b
+// from the Refs if it exists
+func (rf Refs) RemoveIfContains(a, b resid.ItemId) {
+	refs, ok := rf[a]
+	if !ok {
+		return
+	}
+	for i, ref := range refs {
+		if ref.Equals(b) {
+			rf[a] = append(refs[:i], refs[i+1:]...)
+			break
+		}
+	}
+}
+
+//Inventory is a an object intended for
+//serialization into the annotations of a so-called
+//apply-root object (a ConfigMap, an Application,
+//etc.) living in the cluster.  This apply-root
+//object is written as part of an apply operation as
+//a means to record overall cluster state changes.
+
+//At the end of a successful apply, the "current"
+//field in Inventory will be a map whose keys all
+//correspond to an object in the cluster, and
+//"previous" will be the previous such set (an empty
+//set on the first apply).
+
+//An Inventory allows the Prune method to work.
+type Inventory struct {
+	Current  Refs `json:"current,omitempty"`
+	Previous Refs `json:"previous,omitempty"`
+}
+
+// NewInventory returns an Inventory object
+func NewInventory() *Inventory {
+	return &Inventory{
+		Current:  NewRefs(),
+		Previous: NewRefs(),
+	}
+}
+
+// UpdateCurrent updates the Inventory given a
+// new current Refs
+// The existing Current refs is merged into
+// the Previous refs
+func (a *Inventory) UpdateCurrent(curref Refs) *Inventory {
+	if len(a.Previous) > 0 {
+		a.Previous.Merge(a.Current)
+	} else {
+		a.Previous = a.Current
+	}
+	a.Current = curref
+	return a
+}
+
+func (a *Inventory) removeNewlyOrphanedItemsFromPrevious() []resid.ItemId {
+	var results []resid.ItemId
+	for item, refs := range a.Previous {
+		if _, ok := a.Current[item]; ok {
+			delete(a.Previous, item)
+			continue
+		}
+
+		var newRefs []resid.ItemId
+		toDelete := true
+		for _, ref := range refs {
+			if _, ok := a.Current[ref]; ok {
+				toDelete = false
+				newRefs = append(newRefs, ref)
+			}
+		}
+		if toDelete {
+			results = append(results, item)
+			delete(a.Previous, item)
+		} else {
+			a.Previous[item] = newRefs
+		}
+	}
+	return results
+}
+
+func (a *Inventory) removeOrphanedItemsFromPreviousThatAreNotInCurrent() []resid.ItemId {
+	var results []resid.ItemId
+	for item, refs := range a.Previous {
+		if _, ok := a.Current[item]; ok {
+			continue
+		}
+		if len(refs) == 0 {
+			results = append(results, item)
+			delete(a.Previous, item)
+		}
+	}
+	return results
+}
+
+func (a *Inventory) removeOrphanedItemsFromPreviousThatAreInCurrent() {
+	//Remove references from Previous that are already in Current refs
+	for item, refs := range a.Current {
+		for _, ref := range refs {
+			a.Previous.RemoveIfContains(item, ref)
+		}
+	}
+	//Remove items from Previous that are already in Current refs
+	for item, refs := range a.Previous {
+		if len(refs) == 0 {
+			if _, ok := a.Current[item]; ok {
+				delete(a.Previous, item)
+			}
+		}
+	}
+}
+
+// Prune computes the diff of Current refs and Previous refs
+// and returns a list of Items that can be pruned.
+// An item that can be pruned shows up only in Previous refs.
+// Prune also updates the Previous refs with those items removed
+func (a *Inventory) Prune() []resid.ItemId {
+	a.removeOrphanedItemsFromPreviousThatAreInCurrent()
+
+	// These are candidates for deletion from the cluster.
+	removable1 := a.removeOrphanedItemsFromPreviousThatAreNotInCurrent()
+	removable2 := a.removeNewlyOrphanedItemsFromPrevious()
+	return append(removable1, removable2...)
+}
+
+// inventory is the internal type used for serialization
+type inventory struct {
+	Current  map[string][]resid.ItemId `json:"current,omitempty"`
+	Previous map[string][]resid.ItemId `json:"previous,omitempty"`
+}
+
+func (a *Inventory) toInternalType() inventory {
+	prev := map[string][]resid.ItemId{}
+	curr := map[string][]resid.ItemId{}
+	for id, refs := range a.Current {
+		curr[id.String()] = refs
+	}
+	for id, refs := range a.Previous {
+		prev[id.String()] = refs
+	}
+	return inventory{
+		Current:  curr,
+		Previous: prev,
+	}
+}
+
+func (a *Inventory) fromInternalType(i *inventory) {
+	for s, refs := range i.Previous {
+		a.Previous[resid.FromString(s)] = refs
+	}
+	for s, refs := range i.Current {
+		a.Current[resid.FromString(s)] = refs
+	}
+}
+
+func (a *Inventory) marshal() ([]byte, error) {
+	return json.Marshal(a.toInternalType())
+}
+
+func (a *Inventory) unMarshal(data []byte) error {
+	inv := &inventory{
+		Current:  map[string][]resid.ItemId{},
+		Previous: map[string][]resid.ItemId{},
+	}
+	err := json.Unmarshal(data, inv)
+	if err != nil {
+		return err
+	}
+	a.fromInternalType(inv)
+	return nil
+}
+
+// UpdateAnnotations update the annotation map
+func (a *Inventory) UpdateAnnotations(annot map[string]string) error {
+	data, err := a.marshal()
+	if err != nil {
+		return err
+	}
+	annot[InventoryAnnotation] = string(data)
+	return nil
+}
+
+// LoadFromAnnotation loads the Inventory date from the annotation map
+func (a *Inventory) LoadFromAnnotation(annot map[string]string) error {
+	value, ok := annot[InventoryAnnotation]
+	if ok {
+		return a.unMarshal([]byte(value))
+	}
+	return nil
+}

pkg/inventory/inventory_test.go

@@ -0,0 +1,72 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package inventory
+
+import (
+	"testing"
+
+	"sigs.k8s.io/kustomize/pkg/resid"
+)
+
+func makeRefs() (Refs, Refs) {
+	a := resid.FromString("G1_V1_K1|ns1|nm1")
+	b := resid.FromString("G2_V2_K2|ns2|nm2")
+	c := resid.FromString("G3_V3_K3|ns3|nm3")
+	current := NewRefs()
+	current[a] = []resid.ItemId{b, c}
+	current[b] = []resid.ItemId{}
+	current[c] = []resid.ItemId{}
+	new := NewRefs()
+	new[a] = []resid.ItemId{b}
+	new[b] = []resid.ItemId{}
+	return current, new
+}
+
+func TestInventory(t *testing.T) {
+	inventory := NewInventory()
+	curref, _ := makeRefs()
+
+	inventory.UpdateCurrent(curref)
+	if len(inventory.Current) != 3 {
+		t.Fatalf("not getting the correct inventory %v", inventory)
+	}
+	curref, newref := makeRefs()
+	inventory.UpdateCurrent(curref)
+	if len(inventory.Current) != 3 {
+		t.Fatalf("not getting the corrent inventory %v", inventory)
+	}
+	if len(inventory.Previous) != 3 {
+		t.Fatalf("not getting the corrent inventory %v", inventory)
+	}
+
+	items := inventory.Prune()
+	if len(items) != 0 {
+		t.Fatalf("not getting the corrent items %v", items)
+	}
+	if len(inventory.Previous) != 0 {
+		t.Fatalf("not getting the corrent inventory %v", inventory)
+	}
+
+	inventory.UpdateCurrent(newref)
+	items = inventory.Prune()
+	if len(items) != 1 {
+		t.Fatalf("not getting the corrent items %v", items)
+	}
+	if len(inventory.Previous) != 0 {
+		t.Fatalf("not getting the corrent inventory %v", inventory.Previous)
+	}
+}

pkg/resid/itemid.go

@@ -53,7 +53,11 @@ func (i ItemId) String() string {
 		[]string{i.Gvk.String(), ns, nm}, separator)
 }
 
-func New(g gvk.Gvk, ns, nm string) ItemId {
+func (i ItemId) Equals(b ItemId) bool {
+	return i.String() == b.String()
+}
+
+func NewItemId(g gvk.Gvk, ns, nm string) ItemId {
 	return ItemId{
 		Gvk:       g,
 		Namespace: ns,

pkg/resource/resource.go

@@ -88,14 +88,6 @@ func (r *Resource) Merge(other *Resource) {
 	mergeConfigmap(r.Map(), other.Map(), r.Map())
 }
 
-func (r *Resource) PruneString() string {
-	namespace, _ := r.GetFieldValue("metadata.namespace")
-	return r.GetGvk().Group +
-		"_" + r.GetGvk().Kind +
-		"_" + namespace +
-		"_" + r.GetName()
-}
-
 // Replace performs replace with other resource.
 func (r *Resource) Replace(other *Resource) {
 	r.SetLabels(mergeStringMaps(other.GetLabels(), r.GetLabels()))

pkg/target/pruneconfigmap_test.go

@@ -103,17 +103,14 @@ data:
 	if err != nil {
 		t.Fatalf("Err: %v", err)
 	}
+	//nolint
 	th.assertActualEqualsExpected(m, `
 apiVersion: v1
-data:
-  _Secret_default_my-pass: 54f87m6fd6
-  _Secret_default_my-pass---apps_Deployment_default_my-mysql: 54f87m6fd6
-  _Service_default_my-mmmysql: 54f87m6fd6
-  apps_Deployment_default_my-mysql: 54f87m6fd6
 kind: ConfigMap
 metadata:
   annotations:
-    current: 54f87m6fd6
+    kustomize.config.k8s.io/Inventory: '{"current":{"apps_v1beta2_Deployment|default|my-mysql":null,"~G_v1_Secret|default|my-pass":[{"group":"apps","version":"v1beta2","kind":"Deployment","name":"my-mysql","namespace":"default"}],"~G_v1_Service|default|my-mmmysql":null}}'
+    kustomize.config.k8s.io/InventoryHash: kd67f7ht8t
   name: haha
   namespace: default
 ---