add inventory package and refactor inventory transformer

k8sdeps/transformer/inventory/inventory.go

@@ -21,6 +21,7 @@ import (
 	"sigs.k8s.io/kustomize/k8sdeps/kunstruct"
 	"sigs.k8s.io/kustomize/k8sdeps/transformer/hash"
 	"sigs.k8s.io/kustomize/pkg/gvk"
+	"sigs.k8s.io/kustomize/pkg/inventory"
 	"sigs.k8s.io/kustomize/pkg/resid"
 	"sigs.k8s.io/kustomize/pkg/resmap"
 	"sigs.k8s.io/kustomize/pkg/resource"
@@ -28,9 +29,6 @@ import (
 	"sigs.k8s.io/kustomize/pkg/types"
 )
 
-//const PruneAnnotation = "kustomize.k8s.io/PruneRevision"
-const PruneAnnotation = "current"
-
 // inventoryTransformer compute the ConfigMap used in prune
 type inventoryTransformer struct {
 	append      bool
@@ -58,14 +56,20 @@ func NewInventoryTransformer(p *types.Inventory, namespace string, append bool)
 // The prune ConfigMap is used to support the pruning command in the client side tool,
 // which is proposed in https://github.com/kubernetes/enhancements/pull/810
 func (o *inventoryTransformer) Transform(m resmap.ResMap) error {
+	invty := inventory.NewInventory()
 	var keys []string
 	for _, r := range m {
-		s := r.PruneString()
-		keys = append(keys, s)
+		ns, _ := r.GetFieldValue("metadata.namespace")
+		item := resid.New(r.GetGvk(), ns, r.GetName())
+		var refs []resid.ItemId
+
 		for _, refid := range r.GetRefBy() {
 			ref := m[refid]
-			keys = append(keys, s+"---"+ref.PruneString())
+			ns, _ := ref.GetFieldValue("metadata.namespace")
+			refs = append(refs, resid.New(ref.GetGvk(), ns, ref.GetName()))
 		}
+		invty.Current[item.String()] = refs
+		keys = append(keys, item.String())
 	}
 	h, err := hash.SortArrayAndComputeHash(keys)
 	if err != nil {
@@ -75,14 +79,14 @@ func (o *inventoryTransformer) Transform(m resmap.ResMap) error {
 	args := &types.ConfigMapArgs{}
 	args.Name = o.cmName
 	args.Namespace = o.cmNamespace
-	for _, key := range keys {
-		args.LiteralSources = append(args.LiteralSources,
-			key+"="+h)
-	}
 	opts := &types.GeneratorOptions{
 		Annotations: make(map[string]string),
 	}
-	opts.Annotations[PruneAnnotation] = h
+	opts.Annotations[inventory.InventoryHashAnnotation] = h
+	err = invty.UpdateAnnotations(opts.Annotations)
+	if err != nil {
+		return err
+	}
 
 	kf := kunstruct.NewKunstructuredFactoryImpl()
 	k, err := kf.MakeConfigMap(nil, opts, args)

k8sdeps/transformer/inventory/inventory_test.go

@@ -107,12 +107,18 @@ func TestInventoryTransformer(t *testing.T) {
 	rf := resource.NewFactory(
 		kunstruct.NewKunstructuredFactoryImpl())
 
-	// hash is derived based on all keys in the ConfigMap data field.
+	// hash is derived based on all keys in the Inventory
 	// It is added to annotations as
-	//   current: hash
+	//   kustomize.config.k8s.io/InventoryHash: hash
 	// When seeing the same annotation, prune binary assumes no
 	// clean up is needed
-	hash := "k777d7h45b"
+	hash := "h44788gt7g"
+
+	// inventory is the derived json string for an Inventory object
+	// It is added to annotations as
+	// kustomize.config.k8s.io/Inventory: inventory
+	inventory := "{\"current\":{\"apps_v1_Deployment|~X|deploy1\":null,\"~G_v1_ConfigMap|~X|cm1\":[{\"group\":\"apps\",\"version\":\"v1\",\"kind\":\"Deployment\",\"name\":\"deploy1\"}],\"~G_v1_Secret|~X|secret1\":[{\"group\":\"apps\",\"version\":\"v1\",\"kind\":\"Deployment\",\"name\":\"deploy1\"}]}}" // nolint
+
 	//  This is the root or inventory object which tracks all
 	// the applied resources - this is the thing we expect the transformer to create.
 	pruneMap := rf.FromMap(
@@ -123,16 +129,10 @@ func TestInventoryTransformer(t *testing.T) {
 				"name":      "pruneCM",
 				"namespace": "default",
 				"annotations": map[string]interface{}{
-					"current": hash,
+					"kustomize.config.k8s.io/Inventory":     inventory,
+					"kustomize.config.k8s.io/InventoryHash": hash,
 				},
 			},
-			"data": map[string]interface{}{
-				"_ConfigMap__cm1":                             hash,
-				"_Secret__secret1":                            hash,
-				"apps_Deployment__deploy1":                    hash,
-				"_ConfigMap__cm1---apps_Deployment__deploy1":  hash,
-				"_Secret__secret1---apps_Deployment__deploy1": hash,
-			},
 		})
 	expected := resmap.ResMap{
 		resid.NewResIdWithPrefixNamespace(cmap, "pruneCM", "", "default"): pruneMap,

pkg/inventory/constants.go

@@ -0,0 +1,25 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package inventory
+
+const (
+	// the annotation for inventory hash
+	InventoryHashAnnotation = "kustomize.config.k8s.io/InventoryHash"
+
+	// the annotation that contains the inventory information
+	InventoryAnnotation = "kustomize.config.k8s.io/Inventory"
+)

pkg/inventory/inventory.go

@@ -0,0 +1,173 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package inventory
+
+import (
+	"encoding/json"
+
+	"sigs.k8s.io/kustomize/pkg/resid"
+)
+
+// A Refs is a map from an Item string to a list of Items
+// that it is referred
+type Refs map[string][]resid.ItemId
+
+func NewRefs() Refs {
+	return Refs{}
+}
+
+// Merge merges a Refs into an existing Refs
+func (rf Refs) Merge(b Refs) Refs {
+	for key, value := range b {
+		_, ok := rf[key]
+		if ok {
+			rf[key] = append(rf[key], value...)
+		} else {
+			rf[key] = value
+		}
+	}
+	return rf
+}
+
+// RemoveIfContains removes the reference relationship
+//  a --> b
+// from the Refs if it exists
+func (rf Refs) RemoveIfContains(a string, b resid.ItemId) {
+	refs, ok := rf[a]
+	if !ok {
+		return
+	}
+	for i, ref := range refs {
+		if ref.String() == b.String() {
+			rf[a] = append(refs[:i], refs[i+1:]...)
+			break
+		}
+	}
+}
+
+// An Inventory contains current refs
+// and previous refs
+type Inventory struct {
+	Current  Refs `json:"current,omitempty"`
+	Previous Refs `json:"previous,omitempty"`
+}
+
+// NewInventory returns an Inventory object
+func NewInventory() *Inventory {
+	return &Inventory{
+		Current:  NewRefs(),
+		Previous: NewRefs(),
+	}
+}
+
+// UpdateCurrent updates the Inventory given a
+// new current Refs
+// The existing Current refs is merged into
+// the Previous refs
+func (a *Inventory) UpdateCurrent(curref Refs) *Inventory {
+	if len(a.Previous) > 0 {
+		a.Previous.Merge(a.Current)
+	} else {
+		a.Previous = a.Current
+	}
+	a.Current = curref
+	return a
+}
+
+// Prune returns a list of Items that can be pruned
+// as well as updates the Inventory
+func (a *Inventory) Prune() []resid.ItemId {
+	var results []resid.ItemId
+	curref := a.Current
+
+	// Remove references that are already in Current refs
+	for item, refs := range curref {
+		for _, ref := range refs {
+			a.Previous.RemoveIfContains(item, ref)
+		}
+	}
+	// Remove items that are already in Current refs
+	for item, refs := range a.Previous {
+		if len(refs) == 0 {
+			if _, ok := curref[item]; ok {
+				delete(a.Previous, item)
+			}
+		}
+	}
+
+	// Remove items from the Previous refs
+	// that are not referred by others
+	for item, refs := range a.Previous {
+		if _, ok := curref[item]; ok {
+			continue
+		}
+		if len(refs) == 0 {
+			results = append(results, resid.FromString(item))
+			delete(a.Previous, item)
+		}
+	}
+
+	// Remove items from the Previous refs
+	// that are referred only by to be deleted items
+	for item, refs := range a.Previous {
+		if _, ok := curref[item]; ok {
+			delete(a.Previous, item)
+			continue
+		}
+
+		var newRefs []resid.ItemId
+		toDelete := true
+		for _, ref := range refs {
+			if _, ok := curref[ref.String()]; ok {
+				toDelete = false
+				newRefs = append(newRefs, ref)
+			}
+		}
+		if toDelete {
+			results = append(results, resid.FromString(item))
+			delete(a.Previous, item)
+		} else {
+			a.Previous[item] = newRefs
+		}
+	}
+	return results
+}
+
+func (a *Inventory) marshal() ([]byte, error) {
+	return json.Marshal(a)
+}
+
+func (a *Inventory) unMarshal(data []byte) error {
+	return json.Unmarshal(data, a)
+}
+
+func (a *Inventory) UpdateAnnotations(annot map[string]string) error {
+	data, err := a.marshal()
+	if err != nil {
+		return err
+	}
+	annot[InventoryAnnotation] = string(data)
+	return nil
+}
+
+func (a *Inventory) LoadFromAnnotation(annot map[string]string) error {
+	value, ok := annot[InventoryAnnotation]
+	if ok {
+		return a.unMarshal([]byte(value))
+	}
+	return nil
+}

pkg/inventory/inventory_test.go

@@ -0,0 +1,72 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package inventory
+
+import (
+	"testing"
+
+	"sigs.k8s.io/kustomize/pkg/resid"
+)
+
+func makeRefs() (Refs, Refs) {
+	a := resid.FromString("G1_V1_K1|ns1|nm1")
+	b := resid.FromString("G2_V2_K2|ns2|nm2")
+	c := resid.FromString("G3_V3_K3|ns3|nm3")
+	current := NewRefs()
+	current[a.String()] = []resid.ItemId{b, c}
+	current[b.String()] = []resid.ItemId{}
+	current[c.String()] = []resid.ItemId{}
+	new := NewRefs()
+	new[a.String()] = []resid.ItemId{b}
+	new[b.String()] = []resid.ItemId{}
+	return current, new
+}
+
+func TestInventory(t *testing.T) {
+	inventory := NewInventory()
+	curref, _ := makeRefs()
+
+	inventory.UpdateCurrent(curref)
+	if len(inventory.Current) != 3 {
+		t.Fatalf("not getting the correct inventory %v", inventory)
+	}
+	curref, newref := makeRefs()
+	inventory.UpdateCurrent(curref)
+	if len(inventory.Current) != 3 {
+		t.Fatalf("not getting the corrent inventory %v", inventory)
+	}
+	if len(inventory.Previous) != 3 {
+		t.Fatalf("not getting the corrent inventory %v", inventory)
+	}
+
+	items := inventory.Prune()
+	if len(items) != 0 {
+		t.Fatalf("not getting the corrent items %v", items)
+	}
+	if len(inventory.Previous) != 0 {
+		t.Fatalf("not getting the corrent inventory %v", inventory)
+	}
+
+	inventory.UpdateCurrent(newref)
+	items = inventory.Prune()
+	if len(items) != 1 {
+		t.Fatalf("not getting the corrent items %v", items)
+	}
+	if len(inventory.Previous) != 0 {
+		t.Fatalf("not getting the corrent inventory %v", inventory.Previous)
+	}
+}

pkg/resource/resource.go

@@ -88,14 +88,6 @@ func (r *Resource) Merge(other *Resource) {
 	mergeConfigmap(r.Map(), other.Map(), r.Map())
 }
 
-func (r *Resource) PruneString() string {
-	namespace, _ := r.GetFieldValue("metadata.namespace")
-	return r.GetGvk().Group +
-		"_" + r.GetGvk().Kind +
-		"_" + namespace +
-		"_" + r.GetName()
-}
-
 // Replace performs replace with other resource.
 func (r *Resource) Replace(other *Resource) {
 	r.SetLabels(mergeStringMaps(other.GetLabels(), r.GetLabels()))

pkg/target/pruneconfigmap_test.go

@@ -103,17 +103,14 @@ data:
 	if err != nil {
 		t.Fatalf("Err: %v", err)
 	}
+	//nolint
 	th.assertActualEqualsExpected(m, `
 apiVersion: v1
-data:
-  _Secret_default_my-pass: 54f87m6fd6
-  _Secret_default_my-pass---apps_Deployment_default_my-mysql: 54f87m6fd6
-  _Service_default_my-mmmysql: 54f87m6fd6
-  apps_Deployment_default_my-mysql: 54f87m6fd6
 kind: ConfigMap
 metadata:
   annotations:
-    current: 54f87m6fd6
+    kustomize.config.k8s.io/Inventory: '{"current":{"apps_v1beta2_Deployment|default|my-mysql":null,"~G_v1_Secret|default|my-pass":[{"group":"apps","version":"v1beta2","kind":"Deployment","name":"my-mysql","namespace":"default"}],"~G_v1_Service|default|my-mmmysql":null}}'
+    kustomize.config.k8s.io/InventoryHash: kd67f7ht8t
   name: haha
   namespace: default
 ---