diff --git a/k8sdeps/transformer/inventory/inventory.go b/k8sdeps/transformer/inventory/inventory.go index cb33a9fd6..79fb94dd4 100644 --- a/k8sdeps/transformer/inventory/inventory.go +++ b/k8sdeps/transformer/inventory/inventory.go @@ -21,6 +21,7 @@ import ( "sigs.k8s.io/kustomize/k8sdeps/kunstruct" "sigs.k8s.io/kustomize/k8sdeps/transformer/hash" "sigs.k8s.io/kustomize/pkg/gvk" + "sigs.k8s.io/kustomize/pkg/inventory" "sigs.k8s.io/kustomize/pkg/resid" "sigs.k8s.io/kustomize/pkg/resmap" "sigs.k8s.io/kustomize/pkg/resource" @@ -28,9 +29,6 @@ import ( "sigs.k8s.io/kustomize/pkg/types" ) -//const PruneAnnotation = "kustomize.k8s.io/PruneRevision" -const PruneAnnotation = "current" - // inventoryTransformer compute the ConfigMap used in prune type inventoryTransformer struct { append bool @@ -58,14 +56,20 @@ func NewInventoryTransformer(p *types.Inventory, namespace string, append bool) // The prune ConfigMap is used to support the pruning command in the client side tool, // which is proposed in https://github.com/kubernetes/enhancements/pull/810 func (o *inventoryTransformer) Transform(m resmap.ResMap) error { + invty := inventory.NewInventory() var keys []string for _, r := range m { - s := r.PruneString() - keys = append(keys, s) + ns, _ := r.GetFieldValue("metadata.namespace") + item := resid.New(r.GetGvk(), ns, r.GetName()) + var refs []resid.ItemId + for _, refid := range r.GetRefBy() { ref := m[refid] - keys = append(keys, s+"---"+ref.PruneString()) + ns, _ := ref.GetFieldValue("metadata.namespace") + refs = append(refs, resid.New(ref.GetGvk(), ns, ref.GetName())) } + invty.Current[item.String()] = refs + keys = append(keys, item.String()) } h, err := hash.SortArrayAndComputeHash(keys) if err != nil { @@ -75,14 +79,14 @@ func (o *inventoryTransformer) Transform(m resmap.ResMap) error { args := &types.ConfigMapArgs{} args.Name = o.cmName args.Namespace = o.cmNamespace - for _, key := range keys { - args.LiteralSources = append(args.LiteralSources, - key+"="+h) - } opts := &types.GeneratorOptions{ Annotations: make(map[string]string), } - opts.Annotations[PruneAnnotation] = h + opts.Annotations[inventory.InventoryHashAnnotation] = h + err = invty.UpdateAnnotations(opts.Annotations) + if err != nil { + return err + } kf := kunstruct.NewKunstructuredFactoryImpl() k, err := kf.MakeConfigMap(nil, opts, args) diff --git a/k8sdeps/transformer/inventory/inventory_test.go b/k8sdeps/transformer/inventory/inventory_test.go index c7811269d..6d8ee6978 100644 --- a/k8sdeps/transformer/inventory/inventory_test.go +++ b/k8sdeps/transformer/inventory/inventory_test.go @@ -107,12 +107,18 @@ func TestInventoryTransformer(t *testing.T) { rf := resource.NewFactory( kunstruct.NewKunstructuredFactoryImpl()) - // hash is derived based on all keys in the ConfigMap data field. + // hash is derived based on all keys in the Inventory // It is added to annotations as - // current: hash + // kustomize.config.k8s.io/InventoryHash: hash // When seeing the same annotation, prune binary assumes no // clean up is needed - hash := "k777d7h45b" + hash := "h44788gt7g" + + // inventory is the derived json string for an Inventory object + // It is added to annotations as + // kustomize.config.k8s.io/Inventory: inventory + inventory := "{\"current\":{\"apps_v1_Deployment|~X|deploy1\":null,\"~G_v1_ConfigMap|~X|cm1\":[{\"group\":\"apps\",\"version\":\"v1\",\"kind\":\"Deployment\",\"name\":\"deploy1\"}],\"~G_v1_Secret|~X|secret1\":[{\"group\":\"apps\",\"version\":\"v1\",\"kind\":\"Deployment\",\"name\":\"deploy1\"}]}}" // nolint + // This is the root or inventory object which tracks all // the applied resources - this is the thing we expect the transformer to create. pruneMap := rf.FromMap( @@ -123,16 +129,10 @@ func TestInventoryTransformer(t *testing.T) { "name": "pruneCM", "namespace": "default", "annotations": map[string]interface{}{ - "current": hash, + "kustomize.config.k8s.io/Inventory": inventory, + "kustomize.config.k8s.io/InventoryHash": hash, }, }, - "data": map[string]interface{}{ - "_ConfigMap__cm1": hash, - "_Secret__secret1": hash, - "apps_Deployment__deploy1": hash, - "_ConfigMap__cm1---apps_Deployment__deploy1": hash, - "_Secret__secret1---apps_Deployment__deploy1": hash, - }, }) expected := resmap.ResMap{ resid.NewResIdWithPrefixNamespace(cmap, "pruneCM", "", "default"): pruneMap, diff --git a/pkg/inventory/constants.go b/pkg/inventory/constants.go new file mode 100644 index 000000000..47439d437 --- /dev/null +++ b/pkg/inventory/constants.go @@ -0,0 +1,25 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package inventory + +const ( + // the annotation for inventory hash + InventoryHashAnnotation = "kustomize.config.k8s.io/InventoryHash" + + // the annotation that contains the inventory information + InventoryAnnotation = "kustomize.config.k8s.io/Inventory" +) diff --git a/pkg/inventory/inventory.go b/pkg/inventory/inventory.go new file mode 100644 index 000000000..da868bbee --- /dev/null +++ b/pkg/inventory/inventory.go @@ -0,0 +1,173 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package inventory + +import ( + "encoding/json" + + "sigs.k8s.io/kustomize/pkg/resid" +) + +// A Refs is a map from an Item string to a list of Items +// that it is referred +type Refs map[string][]resid.ItemId + +func NewRefs() Refs { + return Refs{} +} + +// Merge merges a Refs into an existing Refs +func (rf Refs) Merge(b Refs) Refs { + for key, value := range b { + _, ok := rf[key] + if ok { + rf[key] = append(rf[key], value...) + } else { + rf[key] = value + } + } + return rf +} + +// RemoveIfContains removes the reference relationship +// a --> b +// from the Refs if it exists +func (rf Refs) RemoveIfContains(a string, b resid.ItemId) { + refs, ok := rf[a] + if !ok { + return + } + for i, ref := range refs { + if ref.String() == b.String() { + rf[a] = append(refs[:i], refs[i+1:]...) + break + } + } +} + +// An Inventory contains current refs +// and previous refs +type Inventory struct { + Current Refs `json:"current,omitempty"` + Previous Refs `json:"previous,omitempty"` +} + +// NewInventory returns an Inventory object +func NewInventory() *Inventory { + return &Inventory{ + Current: NewRefs(), + Previous: NewRefs(), + } +} + +// UpdateCurrent updates the Inventory given a +// new current Refs +// The existing Current refs is merged into +// the Previous refs +func (a *Inventory) UpdateCurrent(curref Refs) *Inventory { + if len(a.Previous) > 0 { + a.Previous.Merge(a.Current) + } else { + a.Previous = a.Current + } + a.Current = curref + return a +} + +// Prune returns a list of Items that can be pruned +// as well as updates the Inventory +func (a *Inventory) Prune() []resid.ItemId { + var results []resid.ItemId + curref := a.Current + + // Remove references that are already in Current refs + for item, refs := range curref { + for _, ref := range refs { + a.Previous.RemoveIfContains(item, ref) + } + } + // Remove items that are already in Current refs + for item, refs := range a.Previous { + if len(refs) == 0 { + if _, ok := curref[item]; ok { + delete(a.Previous, item) + } + } + } + + // Remove items from the Previous refs + // that are not referred by others + for item, refs := range a.Previous { + if _, ok := curref[item]; ok { + continue + } + if len(refs) == 0 { + results = append(results, resid.FromString(item)) + delete(a.Previous, item) + } + } + + // Remove items from the Previous refs + // that are referred only by to be deleted items + for item, refs := range a.Previous { + if _, ok := curref[item]; ok { + delete(a.Previous, item) + continue + } + + var newRefs []resid.ItemId + toDelete := true + for _, ref := range refs { + if _, ok := curref[ref.String()]; ok { + toDelete = false + newRefs = append(newRefs, ref) + } + } + if toDelete { + results = append(results, resid.FromString(item)) + delete(a.Previous, item) + } else { + a.Previous[item] = newRefs + } + } + return results +} + +func (a *Inventory) marshal() ([]byte, error) { + return json.Marshal(a) +} + +func (a *Inventory) unMarshal(data []byte) error { + return json.Unmarshal(data, a) +} + +func (a *Inventory) UpdateAnnotations(annot map[string]string) error { + data, err := a.marshal() + if err != nil { + return err + } + annot[InventoryAnnotation] = string(data) + return nil +} + +func (a *Inventory) LoadFromAnnotation(annot map[string]string) error { + value, ok := annot[InventoryAnnotation] + if ok { + return a.unMarshal([]byte(value)) + } + return nil +} diff --git a/pkg/inventory/inventory_test.go b/pkg/inventory/inventory_test.go new file mode 100644 index 000000000..116ef7e7d --- /dev/null +++ b/pkg/inventory/inventory_test.go @@ -0,0 +1,72 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package inventory + +import ( + "testing" + + "sigs.k8s.io/kustomize/pkg/resid" +) + +func makeRefs() (Refs, Refs) { + a := resid.FromString("G1_V1_K1|ns1|nm1") + b := resid.FromString("G2_V2_K2|ns2|nm2") + c := resid.FromString("G3_V3_K3|ns3|nm3") + current := NewRefs() + current[a.String()] = []resid.ItemId{b, c} + current[b.String()] = []resid.ItemId{} + current[c.String()] = []resid.ItemId{} + new := NewRefs() + new[a.String()] = []resid.ItemId{b} + new[b.String()] = []resid.ItemId{} + return current, new +} + +func TestInventory(t *testing.T) { + inventory := NewInventory() + curref, _ := makeRefs() + + inventory.UpdateCurrent(curref) + if len(inventory.Current) != 3 { + t.Fatalf("not getting the correct inventory %v", inventory) + } + curref, newref := makeRefs() + inventory.UpdateCurrent(curref) + if len(inventory.Current) != 3 { + t.Fatalf("not getting the corrent inventory %v", inventory) + } + if len(inventory.Previous) != 3 { + t.Fatalf("not getting the corrent inventory %v", inventory) + } + + items := inventory.Prune() + if len(items) != 0 { + t.Fatalf("not getting the corrent items %v", items) + } + if len(inventory.Previous) != 0 { + t.Fatalf("not getting the corrent inventory %v", inventory) + } + + inventory.UpdateCurrent(newref) + items = inventory.Prune() + if len(items) != 1 { + t.Fatalf("not getting the corrent items %v", items) + } + if len(inventory.Previous) != 0 { + t.Fatalf("not getting the corrent inventory %v", inventory.Previous) + } +} diff --git a/pkg/resource/resource.go b/pkg/resource/resource.go index 4fadd57e6..697959cbf 100644 --- a/pkg/resource/resource.go +++ b/pkg/resource/resource.go @@ -88,14 +88,6 @@ func (r *Resource) Merge(other *Resource) { mergeConfigmap(r.Map(), other.Map(), r.Map()) } -func (r *Resource) PruneString() string { - namespace, _ := r.GetFieldValue("metadata.namespace") - return r.GetGvk().Group + - "_" + r.GetGvk().Kind + - "_" + namespace + - "_" + r.GetName() -} - // Replace performs replace with other resource. func (r *Resource) Replace(other *Resource) { r.SetLabels(mergeStringMaps(other.GetLabels(), r.GetLabels())) diff --git a/pkg/target/chartinflatorexecplugin_test.go b/pkg/target/chartinflatorexecplugin_test.go index b66803abc..2a3009fd8 100644 --- a/pkg/target/chartinflatorexecplugin_test.go +++ b/pkg/target/chartinflatorexecplugin_test.go @@ -27,7 +27,7 @@ import ( // This is an example of using a helm chart as a base, // inflating it and then customizing it with a nameprefix // applied to all its resources. -// +// // The helm chart used is downloaded from // https://github.com/helm/charts/tree/master/stable/minecraft // with each test run, so it's a bit brittle as that @@ -36,7 +36,7 @@ import ( // This test requires having the helm binary on the PATH. // // TODO: Download and inflate the chart, and check that -// in for the test. +// in for the test. func TestChartInflatorExecPlugin(t *testing.T) { tc := plugintest_test.NewPluginTestEnv(t).Set() defer tc.Reset() diff --git a/pkg/target/pruneconfigmap_test.go b/pkg/target/pruneconfigmap_test.go index 66eeda593..5bed65ae8 100644 --- a/pkg/target/pruneconfigmap_test.go +++ b/pkg/target/pruneconfigmap_test.go @@ -103,17 +103,14 @@ data: if err != nil { t.Fatalf("Err: %v", err) } + //nolint th.assertActualEqualsExpected(m, ` apiVersion: v1 -data: - _Secret_default_my-pass: 54f87m6fd6 - _Secret_default_my-pass---apps_Deployment_default_my-mysql: 54f87m6fd6 - _Service_default_my-mmmysql: 54f87m6fd6 - apps_Deployment_default_my-mysql: 54f87m6fd6 kind: ConfigMap metadata: annotations: - current: 54f87m6fd6 + kustomize.config.k8s.io/Inventory: '{"current":{"apps_v1beta2_Deployment|default|my-mysql":null,"~G_v1_Secret|default|my-pass":[{"group":"apps","version":"v1beta2","kind":"Deployment","name":"my-mysql","namespace":"default"}],"~G_v1_Service|default|my-mmmysql":null}}' + kustomize.config.k8s.io/InventoryHash: kd67f7ht8t name: haha namespace: default ---