diff --git a/api/internal/plugins/fnplugin/fnplugin.go b/api/internal/plugins/fnplugin/fnplugin.go new file mode 100644 index 000000000..89063d7d0 --- /dev/null +++ b/api/internal/plugins/fnplugin/fnplugin.go @@ -0,0 +1,287 @@ +// Copyright 2019 The Kubernetes Authors. +// SPDX-License-Identifier: Apache-2.0 + +package fnplugin + +import ( + "bytes" + "fmt" + "log" + "strconv" + + "github.com/pkg/errors" + + "sigs.k8s.io/kustomize/api/resid" + "sigs.k8s.io/kustomize/api/resmap" + "sigs.k8s.io/kustomize/api/resource" + "sigs.k8s.io/kustomize/api/types" + "sigs.k8s.io/yaml" + + kyaml "sigs.k8s.io/kustomize/kyaml/yaml" + "sigs.k8s.io/kustomize/kyaml/kio" + + "sigs.k8s.io/kustomize/kyaml/runfn" + "sigs.k8s.io/kustomize/kyaml/fn/runtime/runtimeutil" +) + +const ( + idAnnotation = "kustomize.config.k8s.io/id" + HashAnnotation = "kustomize.config.k8s.io/needs-hash" + BehaviorAnnotation = "kustomize.config.k8s.io/behavior" +) + +type FnPlugin struct { + // Function runner + RunFns runfn.RunFns + + // Plugin configuration data. + cfg []byte + + // PluginHelpers + h *resmap.PluginHelpers +} + +func bytesToRNode(yml []byte) (*kyaml.RNode, error) { + rnode, err := kyaml.Parse(string(yml)) + if err != nil { + return nil, err + } + return rnode, nil +} + +func resourceToRNode(res *resource.Resource) (*kyaml.RNode, error) { + yml, err := res.AsYAML() + if err != nil { + return nil, err + } + + return bytesToRNode(yml) +} + +func GetFunctionSpec(res *resource.Resource) (*runtimeutil.FunctionSpec, error) { + rnode, err := resourceToRNode(res) + if err != nil { + return nil, err + } + + fSpec := runtimeutil.GetFunctionSpec(rnode) + if fSpec == nil { + return nil, fmt.Errorf("resource %v doesn't contain function spec", res.GetGvk()) + } + + return fSpec, nil +} + +func toStorageMounts(mounts []string) []runtimeutil.StorageMount { + var sms []runtimeutil.StorageMount + for _, mount := range mounts { + sms = append(sms, runtimeutil.StringToStorageMount(mount)) + } + return sms +} + +func NewFnPlugin(o *types.FnPluginLoadingOptions) *FnPlugin { + log.Printf("options: %v\n", o) + return &FnPlugin{ + RunFns: runfn.RunFns{ + Functions: []*kyaml.RNode{}, + Network: o.Network, + NetworkName: o.NetworkName, + EnableStarlark: o.EnableStar, + EnableExec: o.EnableExec, + StorageMounts: toStorageMounts(o.Mounts), + }, + } +} + +func (p *FnPlugin) Cfg() []byte { + return p.cfg +} + +func (p *FnPlugin) Config(h *resmap.PluginHelpers, config []byte) error { + p.h = h + p.cfg = config + + rnode, err := bytesToRNode(config) + if err != nil { + return err + } + + p.RunFns.Functions = append(p.RunFns.Functions, rnode) + + return nil +} + +func (p *FnPlugin) Generate() (resmap.ResMap, error) { + output, err := p.invokePlugin(nil) + if err != nil { + return nil, err + } + rm, err := p.h.ResmapFactory().NewResMapFromBytes(output) + if err != nil { + return nil, err + } + return p.UpdateResourceOptions(rm) +} + +func (p *FnPlugin) Transform(rm resmap.ResMap) error { + // add ResIds as annotations to all objects so that we can add them back + inputRM, err := p.getResMapWithIdAnnotation(rm) + if err != nil { + return err + } + + // encode the ResMap so it can be fed to the plugin + resources, err := inputRM.AsYaml() + if err != nil { + return err + } + + // invoke the plugin with resources as the input + output, err := p.invokePlugin(resources) + if err != nil { + return fmt.Errorf("%v %s", err, string(output)) + } + + // update the original ResMap based on the output + return p.updateResMapValues(output, rm) +} + +// invokePlugin uses Function runner to run function as plugin +func (p *FnPlugin) invokePlugin(input []byte) ([]byte, error) { + // Transform to ResourceList + var inOut bytes.Buffer + inIn := bytes.NewReader(input) + + err := kio.Pipeline{ + Inputs: []kio.Reader{&kio.ByteReader{Reader: inIn}}, + Outputs: []kio.Writer{kio.ByteWriter{ + Writer: &inOut, + WrappingKind: kio.ResourceListKind, + WrappingAPIVersion: kio.ResourceListAPIVersion,}}, + }.Execute() + if err != nil { + return nil, errors.Wrap( + err, "couldn't transform to ResourceList") + } + + //log.Printf("converted to:\n%s\n", inOut.String()) + + // Execute Fn (it's configured - see Config()) + var runFnsOut bytes.Buffer + p.RunFns.Input = bytes.NewReader(inOut.Bytes()) + p.RunFns.Output = &runFnsOut + + err = p.RunFns.Execute() + if err != nil { + return nil, errors.Wrap( + err, "couln't execute function") + } + + //log.Printf("fn returned:\n%s\n", runFnsOut.String()) + + // Convert back to a single multi-yaml doc + var outOut bytes.Buffer + outIn := bytes.NewReader(runFnsOut.Bytes()) + + err = kio.Pipeline{ + Inputs: []kio.Reader{&kio.ByteReader{Reader: outIn}}, + Outputs: []kio.Writer{kio.ByteWriter{Writer: &outOut}}, + }.Execute() + if err != nil { + return nil, errors.Wrap( + err, "couldn't transform from ResourceList") + } + + //log.Printf("converted back to:\n%s\n", outOut.String()) + + return outOut.Bytes(), nil +} + +// Returns a new copy of the given ResMap with the ResIds annotated in each Resource +func (p *FnPlugin) getResMapWithIdAnnotation(rm resmap.ResMap) (resmap.ResMap, error) { + inputRM := rm.DeepCopy() + for _, r := range inputRM.Resources() { + idString, err := yaml.Marshal(r.CurId()) + if err != nil { + return nil, err + } + annotations := r.GetAnnotations() + if annotations == nil { + annotations = make(map[string]string) + } + annotations[idAnnotation] = string(idString) + r.SetAnnotations(annotations) + } + return inputRM, nil +} + +// updateResMapValues updates the Resource value in the given ResMap +// with the emitted Resource values in output. +func (p *FnPlugin) updateResMapValues(output []byte, rm resmap.ResMap) error { + outputRM, err := p.h.ResmapFactory().NewResMapFromBytes(output) + if err != nil { + return err + } + for _, r := range outputRM.Resources() { + // for each emitted Resource, find the matching Resource in the original ResMap + // using its id + annotations := r.GetAnnotations() + idString, ok := annotations[idAnnotation] + if !ok { + return fmt.Errorf("the transformer should not remove annotation %s", + idAnnotation) + } + id := resid.ResId{} + err := yaml.Unmarshal([]byte(idString), &id) + if err != nil { + return err + } + res, err := rm.GetByCurrentId(id) + if err != nil { + return fmt.Errorf("unable to find unique match to %s", id.String()) + } + // remove the annotation set by Kustomize to track the resource + delete(annotations, idAnnotation) + if len(annotations) == 0 { + annotations = nil + } + r.SetAnnotations(annotations) + + // update the ResMap resource value with the transformed object + res.Kunstructured = r.Kunstructured + } + return nil +} + +// updateResourceOptions updates the generator options for each resource in the +// given ResMap based on plugin provided annotations. +func (p *FnPlugin) UpdateResourceOptions(rm resmap.ResMap) (resmap.ResMap, error) { + for _, r := range rm.Resources() { + // Disable name hashing by default and require plugin to explicitly + // request it for each resource. + annotations := r.GetAnnotations() + behavior := annotations[BehaviorAnnotation] + var needsHash bool + if val, ok := annotations[HashAnnotation]; ok { + b, err := strconv.ParseBool(val) + if err != nil { + return nil, fmt.Errorf( + "the annotation %q contains an invalid value (%q)", + HashAnnotation, val) + } + needsHash = b + } + delete(annotations, HashAnnotation) + delete(annotations, BehaviorAnnotation) + if len(annotations) == 0 { + annotations = nil + } + r.SetAnnotations(annotations) + r.SetOptions(types.NewGenArgs( + &types.GeneratorArgs{ + Behavior: behavior, + Options: &types.GeneratorOptions{DisableNameSuffixHash: !needsHash}})) + } + return rm, nil +} diff --git a/api/internal/plugins/loader/loader.go b/api/internal/plugins/loader/loader.go index 09dc3a167..25cb12ebf 100644 --- a/api/internal/plugins/loader/loader.go +++ b/api/internal/plugins/loader/loader.go @@ -16,6 +16,7 @@ import ( "sigs.k8s.io/kustomize/api/ifc" "sigs.k8s.io/kustomize/api/internal/plugins/builtinhelpers" "sigs.k8s.io/kustomize/api/internal/plugins/execplugin" + "sigs.k8s.io/kustomize/api/internal/plugins/fnplugin" "sigs.k8s.io/kustomize/api/internal/plugins/utils" "sigs.k8s.io/kustomize/api/konfig" "sigs.k8s.io/kustomize/api/resid" @@ -116,7 +117,7 @@ func (l *Loader) loadAndConfigurePlugin( if isBuiltinPlugin(res) { switch l.pc.BpLoadingOptions { case types.BploLoadFromFileSys: - c, err = l.loadPlugin(res.OrgId()) + c, err = l.loadPlugin(res) case types.BploUseStaticallyLinked: // Instead of looking for and loading a .so file, // instantiate the plugin from a generated factory @@ -131,7 +132,7 @@ func (l *Loader) loadAndConfigurePlugin( } else { switch l.pc.PluginRestrictions { case types.PluginRestrictionsNone: - c, err = l.loadPlugin(res.OrgId()) + c, err = l.loadPlugin(res) case types.PluginRestrictionsBuiltinsOnly: err = types.NewErrOnlyBuiltinPluginsAllowed(res.OrgId().Kind) default: @@ -166,7 +167,15 @@ func (l *Loader) makeBuiltinPlugin(r resid.Gvk) (resmap.Configurable, error) { return nil, errors.Errorf("unable to load builtin %s", r) } -func (l *Loader) loadPlugin(resId resid.ResId) (resmap.Configurable, error) { +func (l *Loader) loadPlugin(res *resource.Resource) (resmap.Configurable, error) { + _, err := fnplugin.GetFunctionSpec(res) + if err == nil { + return fnplugin.NewFnPlugin(&l.pc.FnpLoadingOptions), nil + } + return l.loadExecOrGoPlugin(res.OrgId()) +} + +func (l *Loader) loadExecOrGoPlugin(resId resid.ResId) (resmap.Configurable, error) { // First try to load the plugin as an executable. p := execplugin.NewExecPlugin(l.absolutePluginPath(resId)) err := p.ErrIfNotExecutable() diff --git a/api/types/pluginconfig.go b/api/types/pluginconfig.go index 88c0ade77..9b48c6771 100644 --- a/api/types/pluginconfig.go +++ b/api/types/pluginconfig.go @@ -29,4 +29,7 @@ type PluginConfig struct { // BpLoadingOptions distinguishes builtin plugin behaviors. BpLoadingOptions BuiltinPluginLoadingOptions + + // FnpLoadingOpeions sets the way function-based plugin behaviors. + FnpLoadingOptions FnPluginLoadingOptions } diff --git a/api/types/pluginrestrictions.go b/api/types/pluginrestrictions.go index a9953c00f..81478a8da 100644 --- a/api/types/pluginrestrictions.go +++ b/api/types/pluginrestrictions.go @@ -41,3 +41,16 @@ const ( // to generate static code. BploLoadFromFileSys ) + +// FnPluginLoadingOptions set way functions-based pluing are restricted +type FnPluginLoadingOptions struct { + // Allow to run executables + EnableExec bool + // Allow to run starlark + EnableStar bool + // Allow container access to network + Network bool + NetworkName string + // list of mounts + Mounts []string +} diff --git a/kustomize/internal/commands/build/build.go b/kustomize/internal/commands/build/build.go index 8cadb15ea..76219b8ef 100644 --- a/kustomize/internal/commands/build/build.go +++ b/kustomize/internal/commands/build/build.go @@ -25,6 +25,7 @@ type Options struct { kustomizationPath string outputPath string outOrder reorderOutput + fnOptions types.FnPluginLoadingOptions } // NewOptions creates a Options object @@ -74,10 +75,27 @@ func NewCmdBuild(out io.Writer) *cobra.Command { &o.outputPath, "output", "o", "", "If specified, write the build output to this path.") + cmd.Flags().BoolVar( + &o.fnOptions.EnableExec, "enable-exec", false /*do not change!*/, + "enable support for exec functions -- note: exec functions run arbitrary code -- do not use for untrusted configs!!! (Alpha)") + cmd.Flags().BoolVar( + &o.fnOptions.EnableStar, "enable-star", false, + "enable support for starlark functions. (Alpha)") + cmd.Flags().BoolVar( + &o.fnOptions.Network, "network", false, + "enable network access for functions that declare it") + cmd.Flags().StringVar( + &o.fnOptions.NetworkName, "network-name", "bridge", + "the docker network to run the container in") + cmd.Flags().StringArrayVar( + &o.fnOptions.Mounts, "mount", []string{}, + "a list of storage options read from the filesystem") + addFlagLoadRestrictor(cmd.Flags()) addFlagEnablePlugins(cmd.Flags()) addFlagReorderOutput(cmd.Flags()) addFlagEnableManagedbyLabel(cmd.Flags()) + return cmd } @@ -111,6 +129,9 @@ func (o *Options) makeOptions() *krusty.Options { if err != nil { log.Fatal(err) } + + c.FnpLoadingOptions = o.fnOptions + opts.PluginConfig = c } else { opts.PluginConfig = konfig.DisabledPluginConfig()