Files
kustomize/api/internal/accumulator/resaccumulator.go
Carl Henrik Lunde 4842d8be60 perf: Intersection: Avoid callid AllIds inside inner loop
This shaves of another 8.5 seconds (one third) of the remaining execution
time for a kustomization tree with 4000 documents, reducing the execution
time from 27.46s to 18.94s

     0.02s 0.062% 11.14%      8.45s 26.36%  sigs.k8s.io/kustomize/api/internal/accumulator.(*ResAccumulator).Intersection
     0.06s  0.19% 11.32%      8.32s 25.95%  sigs.k8s.io/kustomize/api/resmap.(*resWrangler).AllIds

before

    (pprof) top25 -cum
    Showing nodes accounting for 3.63s, 11.32% of 32.06s total
    Dropped 614 nodes (cum <= 0.16s)
    Showing top 25 nodes out of 171
        flat  flat%   sum%        cum   cum%
            0     0%     0%     27.46s 85.65%  github.com/spf13/cobra.(*Command).Execute
            0     0%     0%     27.46s 85.65%  github.com/spf13/cobra.(*Command).ExecuteC
            0     0%     0%     27.46s 85.65%  github.com/spf13/cobra.(*Command).execute
            0     0%     0%     27.46s 85.65%  main.main
            0     0%     0%     27.46s 85.65%  runtime.main
            0     0%     0%     27.46s 85.65%  sigs.k8s.io/kustomize/kustomize/v5/commands/build.NewCmdBuild.func1
            0     0%     0%     26.95s 84.06%  sigs.k8s.io/kustomize/api/krusty.(*Kustomizer).Run
            0     0%     0%     22.09s 68.90%  sigs.k8s.io/kustomize/api/internal/target.(*KustTarget).MakeCustomizedResMap (inline)
            0     0%     0%     22.09s 68.90%  sigs.k8s.io/kustomize/api/internal/target.(*KustTarget).makeCustomizedResMap
        0.29s   0.9%   0.9%     20.96s 65.38%  sigs.k8s.io/kustomize/api/resource.(*Resource).CurId
            0     0%   0.9%     13.61s 42.45%  sigs.k8s.io/kustomize/api/resmap.(*resWrangler).Append
            0     0%   0.9%     13.60s 42.42%  sigs.k8s.io/kustomize/api/resmap.(*resWrangler).GetMatchingResourcesByCurrentId (partial-inline)
        0.14s  0.44%  1.34%     13.60s 42.42%  sigs.k8s.io/kustomize/api/resmap.(*resWrangler).filteredById
        0.05s  0.16%  1.50%     12.91s 40.27%  sigs.k8s.io/kustomize/api/resmap.GetCurrentId
        0.25s  0.78%  2.28%     12.48s 38.93%  sigs.k8s.io/kustomize/api/resource.(*Resource).GetGvk (inline)
        0.49s  1.53%  3.81%     12.23s 38.15%  sigs.k8s.io/kustomize/kyaml/resid.GvkFromNode
            0     0%  3.81%     11.61s 36.21%  sigs.k8s.io/kustomize/api/internal/target.(*KustTarget).IgnoreLocal
            0     0%  3.81%     10.47s 32.66%  sigs.k8s.io/kustomize/api/internal/target.(*KustTarget).AccumulateTarget
            0     0%  3.81%     10.47s 32.66%  sigs.k8s.io/kustomize/api/internal/target.(*KustTarget).accumulateTarget
        0.01s 0.031%  3.84%     10.46s 32.63%  sigs.k8s.io/kustomize/api/internal/target.(*KustTarget).accumulateResources
            0     0%  3.84%     10.43s 32.53%  sigs.k8s.io/kustomize/api/internal/target.(*KustTarget).accumulateDirectory
        0.64s  2.00%  5.83%     10.12s 31.57%  sigs.k8s.io/kustomize/kyaml/yaml.visitMappingNodeFields
        1.68s  5.24% 11.07%      9.48s 29.57%  sigs.k8s.io/kustomize/kyaml/yaml.visitFieldsWhileTrue
        0.02s 0.062% 11.14%      8.45s 26.36%  sigs.k8s.io/kustomize/api/internal/accumulator.(*ResAccumulator).Intersection
        0.06s  0.19% 11.32%      8.32s 25.95%  sigs.k8s.io/kustomize/api/resmap.(*resWrangler).AllIds

after

    (pprof) top30 -cum
    Showing nodes accounting for 5.04s, 22.63% of 22.27s total
    Dropped 540 nodes (cum <= 0.11s)
    Showing top 30 nodes out of 209
        flat  flat%   sum%        cum   cum%
            0     0%     0%     18.94s 85.05%  github.com/spf13/cobra.(*Command).Execute
            0     0%     0%     18.94s 85.05%  github.com/spf13/cobra.(*Command).ExecuteC
            0     0%     0%     18.94s 85.05%  github.com/spf13/cobra.(*Command).execute
            0     0%     0%     18.94s 85.05%  main.main
            0     0%     0%     18.94s 85.05%  runtime.main
            0     0%     0%     18.94s 85.05%  sigs.k8s.io/kustomize/kustomize/v5/commands/build.NewCmdBuild.func1
            0     0%     0%     18.40s 82.62%  sigs.k8s.io/kustomize/api/krusty.(*Kustomizer).Run
            0     0%     0%     13.65s 61.29%  sigs.k8s.io/kustomize/api/internal/target.(*KustTarget).MakeCustomizedResMap (inline)
            0     0%     0%     13.65s 61.29%  sigs.k8s.io/kustomize/api/internal/target.(*KustTarget).makeCustomizedResMap
            0     0%     0%     13.52s 60.71%  sigs.k8s.io/kustomize/api/resmap.(*resWrangler).Append
            0     0%     0%     13.44s 60.35%  sigs.k8s.io/kustomize/api/resmap.(*resWrangler).GetMatchingResourcesByCurrentId (inline)
        0.16s  0.72%  0.72%     13.44s 60.35%  sigs.k8s.io/kustomize/api/resmap.(*resWrangler).filteredById
        0.04s  0.18%   0.9%     12.54s 56.31%  sigs.k8s.io/kustomize/api/resmap.GetCurrentId
        0.19s  0.85%  1.75%     12.49s 56.08%  sigs.k8s.io/kustomize/api/resource.(*Resource).CurId
            0     0%  1.75%     10.37s 46.56%  sigs.k8s.io/kustomize/api/internal/target.(*KustTarget).AccumulateTarget
            0     0%  1.75%     10.37s 46.56%  sigs.k8s.io/kustomize/api/internal/target.(*KustTarget).accumulateResources
            0     0%  1.75%     10.37s 46.56%  sigs.k8s.io/kustomize/api/internal/target.(*KustTarget).accumulateTarget
            0     0%  1.75%     10.34s 46.43%  sigs.k8s.io/kustomize/api/internal/target.(*KustTarget).accumulateDirectory
        0.19s  0.85%  2.60%      7.82s 35.11%  sigs.k8s.io/kustomize/api/resource.(*Resource).GetGvk (inline)
        0.42s  1.89%  4.49%      7.63s 34.26%  sigs.k8s.io/kustomize/kyaml/resid.GvkFromNode
        0.26s  1.17%  5.66%      6.01s 26.99%  sigs.k8s.io/kustomize/kyaml/yaml.visitMappingNodeFields
            0     0%  5.66%      5.76s 25.86%  sigs.k8s.io/kustomize/api/internal/accumulator.(*ResAccumulator).MergeAccumulator
        1.12s  5.03% 10.69%      5.75s 25.82%  sigs.k8s.io/kustomize/kyaml/yaml.visitFieldsWhileTrue
            0     0% 10.69%      5.57s 25.01%  sigs.k8s.io/kustomize/api/resmap.(*resWrangler).appendAll (inline)
            0     0% 10.69%      5.55s 24.92%  sigs.k8s.io/kustomize/api/internal/accumulator.(*ResAccumulator).AppendAll (inline)
            0     0% 10.69%      5.55s 24.92%  sigs.k8s.io/kustomize/api/resmap.(*resWrangler).AppendAll
            0     0% 10.69%      4.73s 21.24%  sigs.k8s.io/kustomize/api/internal/builtins.(*SortOrderTransformerPlugin).Transform
            0     0% 10.69%      4.73s 21.24%  sigs.k8s.io/kustomize/api/krusty.(*Kustomizer).applySortOrder
            0     0% 10.69%      4.72s 21.19%  sigs.k8s.io/kustomize/api/internal/builtins.applyOrdering
        2.66s 11.94% 22.63%      4.63s 20.79%  sigs.k8s.io/kustomize/kyaml/yaml.visitMappingNodeFields.func2
2023-03-12 15:50:11 +01:00

191 lines
5.2 KiB
Go

// Copyright 2019 The Kubernetes Authors.
// SPDX-License-Identifier: Apache-2.0
package accumulator
import (
"fmt"
"log"
"strings"
"sigs.k8s.io/kustomize/api/internal/plugins/builtinconfig"
"sigs.k8s.io/kustomize/api/resmap"
"sigs.k8s.io/kustomize/api/types"
"sigs.k8s.io/kustomize/kyaml/resid"
)
// ResAccumulator accumulates resources and the rules
// used to customize those resources. It's a ResMap
// plus stuff needed to modify the ResMap.
type ResAccumulator struct {
resMap resmap.ResMap
tConfig *builtinconfig.TransformerConfig
varSet types.VarSet
}
func MakeEmptyAccumulator() *ResAccumulator {
ra := &ResAccumulator{}
ra.resMap = resmap.New()
ra.tConfig = &builtinconfig.TransformerConfig{}
ra.varSet = types.NewVarSet()
return ra
}
// ResMap returns a copy of the internal resMap.
func (ra *ResAccumulator) ResMap() resmap.ResMap {
return ra.resMap.ShallowCopy()
}
// Vars returns a copy of underlying vars.
func (ra *ResAccumulator) Vars() []types.Var {
return ra.varSet.AsSlice()
}
func (ra *ResAccumulator) AppendAll(resources resmap.ResMap) error {
return ra.resMap.AppendAll(resources)
}
func (ra *ResAccumulator) AbsorbAll(resources resmap.ResMap) error {
return ra.resMap.AbsorbAll(resources)
}
func (ra *ResAccumulator) MergeConfig(
tConfig *builtinconfig.TransformerConfig) (err error) {
ra.tConfig, err = ra.tConfig.Merge(tConfig)
return err
}
func (ra *ResAccumulator) GetTransformerConfig() *builtinconfig.TransformerConfig {
return ra.tConfig
}
// MergeVars accumulates vars into ResAccumulator.
// A Var is a tuple of name, object reference and field reference.
// This func takes a list of vars from the current kustomization file and
// annotates the accumulated resources with the names of the vars that match
// those resources. E.g. if there's a var named "sam" that wants to get
// its data from a ConfigMap named "james", and the resource list contains a
// ConfigMap named "james", then that ConfigMap will be annotated with the
// var name "sam". Later this annotation is used to find the data for "sam"
// by digging into a particular fieldpath of "james".
func (ra *ResAccumulator) MergeVars(incoming []types.Var) error {
for _, v := range incoming {
targetId := resid.NewResIdWithNamespace(v.ObjRef.GVK(), v.ObjRef.Name, v.ObjRef.Namespace)
idMatcher := targetId.GvknEquals
if targetId.Namespace != "" || targetId.IsClusterScoped() {
// Preserve backward compatibility. An empty namespace means
// wildcard search on the namespace hence we still use GvknEquals
idMatcher = targetId.Equals
}
matched := ra.resMap.GetMatchingResourcesByAnyId(idMatcher)
if len(matched) > 1 {
return fmt.Errorf(
"found %d resId matches for var %s "+
"(unable to disambiguate)",
len(matched), v)
}
if len(matched) == 1 {
matched[0].AppendRefVarName(v)
}
}
return ra.varSet.MergeSlice(incoming)
}
func (ra *ResAccumulator) MergeAccumulator(other *ResAccumulator) (err error) {
err = ra.AppendAll(other.resMap)
if err != nil {
return err
}
err = ra.MergeConfig(other.tConfig)
if err != nil {
return err
}
return ra.varSet.MergeSet(other.varSet)
}
func (ra *ResAccumulator) findVarValueFromResources(v types.Var) (interface{}, error) {
for _, res := range ra.resMap.Resources() {
for _, varName := range res.GetRefVarNames() {
if varName == v.Name {
s, err := res.GetFieldValue(v.FieldRef.FieldPath)
if err != nil {
return "", fmt.Errorf(
"field specified in var '%v' "+
"not found in corresponding resource", v)
}
return s, nil
}
}
}
return "", fmt.Errorf(
"var '%v' cannot be mapped to a field "+
"in the set of known resources", v)
}
// makeVarReplacementMap returns a map of Var names to
// their final values. The values are strings intended
// for substitution wherever the $(var.Name) occurs.
func (ra *ResAccumulator) makeVarReplacementMap() (map[string]interface{}, error) {
result := map[string]interface{}{}
for _, v := range ra.Vars() {
s, err := ra.findVarValueFromResources(v)
if err != nil {
return nil, err
}
result[v.Name] = s
}
return result, nil
}
func (ra *ResAccumulator) Transform(t resmap.Transformer) error {
return t.Transform(ra.resMap)
}
func (ra *ResAccumulator) ResolveVars() error {
replacementMap, err := ra.makeVarReplacementMap()
if err != nil {
return err
}
if len(replacementMap) == 0 {
return nil
}
t := newRefVarTransformer(
replacementMap, ra.tConfig.VarReference)
err = ra.Transform(t)
if len(t.UnusedVars()) > 0 {
log.Printf(
"well-defined vars that were never replaced: %s\n",
strings.Join(t.UnusedVars(), ","))
}
return err
}
func (ra *ResAccumulator) FixBackReferences() (err error) {
if ra.tConfig.NameReference == nil {
return nil
}
return ra.Transform(
newNameReferenceTransformer(ra.tConfig.NameReference))
}
// Intersection drops the resources which "other" does not have.
func (ra *ResAccumulator) Intersection(other resmap.ResMap) error {
otherIds := other.AllIds()
for _, curId := range ra.resMap.AllIds() {
toDelete := true
for _, otherId := range otherIds {
if otherId == curId {
toDelete = false
break
}
}
if toDelete {
err := ra.resMap.Remove(curId)
if err != nil {
return err
}
}
}
return nil
}