add support for exec plugins

bin/pre-commit.sh

@@ -11,14 +11,6 @@ cd "$base_dir" || {
 
 rc=0
 
-function buildPlugins {
-  go build \
-     -buildmode plugin \
-     -tags=plugin \
-     -o ./pkg/plugins/builtin/executable.so \
-        ./pkg/plugins/builtin/executable.go
-}
-
 function runTest {
   local name=$1
   local result="SUCCESS"
@@ -88,7 +80,6 @@ echo pwd=`pwd`
 echo " "
 echo "Beginning tests..."
 
-runTest buildPlugins
 runTest testGoLangCILint
 runTest testGoTest
 

k8sdeps/kunstruct/kunstruct.go

@@ -19,7 +19,7 @@ package kunstruct
 
 import (
 	"encoding/json"
-	"fmt"
+	"sigs.k8s.io/kustomize/pkg/types"
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
@@ -88,7 +88,7 @@ func (fs *UnstructAdapter) GetFieldValue(path string) (string, error) {
 	if found || err != nil {
 		return s, err
 	}
-	return "", fmt.Errorf("no field named '%s'", path)
+	return "", types.NoFieldError{Field: path}
 }
 
 // GetStringSlice returns value at the given fieldpath.
@@ -102,5 +102,5 @@ func (fs *UnstructAdapter) GetStringSlice(path string) ([]string, error) {
 	if found || err != nil {
 		return s, err
 	}
-	return []string{}, fmt.Errorf("no field named '%s'", path)
+	return []string{}, types.NoFieldError{Field: path}
 }

pkg/plugins/builtin/executable.go

@@ -1,72 +0,0 @@
-// +build plugin
-
-package main
-
-import (
-	"bytes"
-	"os"
-	"os/exec"
-	"path/filepath"
-
-	"github.com/ghodss/yaml"
-	"sigs.k8s.io/kustomize/pkg/ifc"
-	"sigs.k8s.io/kustomize/pkg/pgmconfig"
-	"sigs.k8s.io/kustomize/pkg/resmap"
-)
-
-type plugin struct {
-	name  string
-	input string
-	rf    *resmap.Factory
-}
-
-var KustomizePlugin plugin
-
-func (p *plugin) Config(
-	ldr ifc.Loader, rf *resmap.Factory, k ifc.Kunstructured) error {
-	dir := filepath.Join(pgmconfig.ConfigRoot(), "plugins")
-	id := k.GetGvk()
-	p.name = filepath.Join(dir, id.Group, id.Version, id.Kind)
-	content, err := yaml.Marshal(k)
-	if err != nil {
-		return err
-	}
-	p.input = string(content)
-	p.rf = rf
-	return nil
-}
-
-func (p *plugin) Generate() (resmap.ResMap, error) {
-	return p.run(nil)
-}
-
-func (p *plugin) Transformer(rm resmap.ResMap) error {
-	result, err := p.run(rm)
-	if err != nil {
-		return err
-	}
-	for id := range rm {
-		delete(rm, id)
-	}
-	for id, r := range result {
-		rm[id] = r
-	}
-	return nil
-}
-
-func (p *plugin) run(rm resmap.ResMap) (resmap.ResMap, error) {
-	cmd := exec.Command(p.name, p.input)
-	cmd.Env = os.Environ()
-	if rm != nil {
-		content, err := rm.EncodeAsYaml()
-		if err != nil {
-			return nil, err
-		}
-		cmd.Stdin = bytes.NewReader(content)
-	}
-	output, err := cmd.Output()
-	if err != nil {
-		return nil, err
-	}
-	return p.rf.NewResMapFromBytes(output)
-}

pkg/plugins/compiler.go

@@ -17,7 +17,9 @@ import (
 	"fmt"
 	"os"
 	"os/exec"
+	"path"
 	"path/filepath"
+	"runtime"
 	"time"
 
 	"sigs.k8s.io/kustomize/k8sdeps/kv/plugin"
@@ -51,6 +53,15 @@ func DefaultSrcRoot() (string, error) {
 	}
 	nope = append(nope, root)
 
+	// get the root kustomize source directory when
+	// GOPATH is not set
+	_, filename, _, _ := runtime.Caller(1)
+	root = path.Join(path.Dir(filename), "../..", plugin.PluginRoot)
+	if FileExists(root) {
+		return root, nil
+	}
+	nope = append(nope, root)
+
 	root = filepath.Join(
 		pgmconfig.ConfigRoot(), plugin.PluginRoot)
 	if FileExists(root) {

pkg/plugins/executable.go

@@ -0,0 +1,140 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package plugins
+
+import (
+	"bytes"
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"sigs.k8s.io/kustomize/pkg/types"
+	"strings"
+
+	"github.com/ghodss/yaml"
+	"sigs.k8s.io/kustomize/pkg/ifc"
+	"sigs.k8s.io/kustomize/pkg/pgmconfig"
+	"sigs.k8s.io/kustomize/pkg/resmap"
+)
+
+// ExecPlugin record the name and args of an executable
+// It triggers the executable generator and transformer
+type ExecPlugin struct {
+	// name of the executable
+	name string
+
+	// one line of arguments for the executable
+	argOneLiner string
+
+	// relative file path to a file
+	// Each line of this file is treated as one argument
+	argsFromFile string
+
+	// resmap Factory to make resources
+	rf *resmap.Factory
+
+	// loader to load files
+	ldr ifc.Loader
+}
+
+func (p *ExecPlugin) Config(
+	ldr ifc.Loader, rf *resmap.Factory, k ifc.Kunstructured) error {
+	dir := filepath.Join(pgmconfig.ConfigRoot(), "plugins")
+	id := k.GetGvk()
+	p.name = filepath.Join(dir, id.Group, id.Version, id.Kind)
+	p.rf = rf
+	p.ldr = ldr
+
+	var err error
+	p.argOneLiner, err = k.GetFieldValue("arg")
+	if err != nil && !isNoFieldError(err) {
+		return err
+	}
+	p.argsFromFile, err = k.GetFieldValue("file")
+	if err != nil && !isNoFieldError(err) {
+		return err
+	}
+	return nil
+}
+
+func (p *ExecPlugin) Generate() (resmap.ResMap, error) {
+	args, err := p.getArgs()
+	if err != nil {
+		return nil, err
+	}
+	cmd := exec.Command(p.name, args...)
+	cmd.Env = os.Environ()
+	cmd.Stderr = os.Stderr
+	output, err := cmd.Output()
+	if err != nil {
+		return nil, err
+	}
+	return p.rf.NewResMapFromBytes(output)
+}
+
+func (p *ExecPlugin) Transform(rm resmap.ResMap) error {
+	args, err := p.getArgs()
+	if err != nil {
+		return err
+	}
+
+	for id, r := range rm {
+		content, err := yaml.Marshal(r.Kunstructured)
+		if err != nil {
+			return err
+		}
+		cmd := exec.Command(p.name, args...)
+		cmd.Env = os.Environ()
+		cmd.Stdin = bytes.NewReader(content)
+		cmd.Stderr = os.Stderr
+		output, err := cmd.Output()
+		if err != nil {
+			return err
+		}
+		tmpMap, err := p.rf.NewResMapFromBytes(output)
+		if err != nil {
+			return err
+		}
+		if len(tmpMap) != 1 {
+			return fmt.Errorf("Unable to put two resources into one")
+		}
+		for _, v := range tmpMap {
+			rm[id].Kunstructured = v.Kunstructured
+		}
+	}
+	return nil
+}
+
+func (p *ExecPlugin) getArgs() ([]string, error) {
+	args := strings.Split(p.argOneLiner, " ")
+	if p.argsFromFile != "" {
+		content, err := p.ldr.Load(p.argsFromFile)
+		if err != nil {
+			return nil, err
+		}
+		args = append(args, strings.Split(string(content), "\n")...)
+	}
+	return args, nil
+}
+
+func isNoFieldError(e error) bool {
+	_, ok := e.(types.NoFieldError)
+	if ok {
+		return true
+	}
+	return false
+}

pkg/plugins/transformers.go

@@ -18,12 +18,10 @@ package plugins
 
 import (
 	"fmt"
+	"github.com/pkg/errors"
 	"os"
 	"path/filepath"
 	"plugin"
-	"runtime"
-
-	"github.com/pkg/errors"
 	kplugin "sigs.k8s.io/kustomize/k8sdeps/kv/plugin"
 	"sigs.k8s.io/kustomize/pkg/ifc"
 	"sigs.k8s.io/kustomize/pkg/resid"
@@ -96,28 +94,30 @@ func loadAndConfigurePlugin(
 	ldr ifc.Loader,
 	rf *resmap.Factory, res *resource.Resource) (Configurable, error) {
 	var fileName string
+	var c Configurable
+
 	exec := execPluginFileName(dir, id)
 	if isExecAvailable(exec) {
-		_, f, _, _ := runtime.Caller(1)
+		c = &ExecPlugin{}
-		fileName = filepath.Join(filepath.Dir(f), "builtin", "executable.so")
 	} else {
 		fileName = goPluginFileName(dir, id)
+		goPlugin, err := plugin.Open(fileName)
+		if err != nil {
+			return nil, errors.Wrapf(err, "plugin %s fails to load", fileName)
+		}
+		symbol, err := goPlugin.Lookup(kplugin.PluginSymbol)
+		if err != nil {
+			return nil, errors.Wrapf(
+				err, "plugin %s doesn't have symbol %s",
+				fileName, kplugin.PluginSymbol)
+		}
+		var ok bool
+		c, ok = symbol.(Configurable)
+		if !ok {
+			return nil, fmt.Errorf("plugin %s not configurable", fileName)
+		}
 	}
-	goPlugin, err := plugin.Open(fileName)
+	err := c.Config(ldr, rf, res)
-	if err != nil {
-		return nil, errors.Wrapf(err, "plugin %s fails to load", fileName)
-	}
-	symbol, err := goPlugin.Lookup(kplugin.PluginSymbol)
-	if err != nil {
-		return nil, errors.Wrapf(
-			err, "plugin %s doesn't have symbol %s",
-			fileName, kplugin.PluginSymbol)
-	}
-	c, ok := symbol.(Configurable)
-	if !ok {
-		return nil, fmt.Errorf("plugin %s not configurable", fileName)
-	}
-	err = c.Config(ldr, rf, res)
 	if err != nil {
 		return nil, errors.Wrapf(err, "plugin %s fails configuration", fileName)
 	}

pkg/target/generatorplugin_test.go

@@ -128,7 +128,7 @@ type: Opaque
 `)
 }
 
-func xTestConfigMapGenerator(t *testing.T) {
+func TestConfigMapGenerator(t *testing.T) {
 	tc := NewTestEnvController(t).Set()
 	defer tc.Reset()
 
@@ -146,6 +146,7 @@ apiVersion: someteam.example.com/v1
 kind: ConfigMapGenerator
 metadata:
   name: some-random-name
+arg: "admin secret"
 `)
 	m, err := th.makeKustTarget().MakeCustomizedResMap()
 	if err != nil {

pkg/target/transformerplugin_test.go

@@ -98,6 +98,52 @@ spec:
 `)
 }
 
+func TestSedTransformer(t *testing.T) {
+	tc := NewTestEnvController(t).Set()
+	defer tc.Reset()
+
+	tc.BuildExecPlugin(
+		"someteam.example.com", "v1", "SedTransformer")
+
+	th := NewKustTestHarnessWithPluginConfig(
+		t, "/app", plugin.ActivePluginConfig())
+	th.writeK("/app", `
+transformers:
+- sed-transformer.yaml
+
+configMapGenerator:
+- name: test
+  literals:
+  - FOO=$FOO
+  - BAR=$BAR
+`)
+	th.writeF("/app/sed-transformer.yaml", `
+apiVersion: someteam.example.com/v1
+kind: SedTransformer
+metadata:
+  name: some-random-name
+file: sed-input.txt
+`)
+	th.writeF("/app/sed-input.txt", `
+s/$FOO/foo/g
+s/$BAR/bar/g
+`)
+
+	m, err := th.makeKustTarget().MakeCustomizedResMap()
+	if err != nil {
+		t.Fatalf("Err: %v", err)
+	}
+	th.assertActualEqualsExpected(m, `
+apiVersion: v1
+data:
+  BAR: bar
+  FOO: foo
+kind: ConfigMap
+metadata:
+  name: test-k4bkhftttd
+`)
+}
+
 func xTestTransformedTransformers(t *testing.T) {
 	th := NewKustTestHarnessWithPluginConfig(
 		t, "/app/overlay", plugin.ActivePluginConfig())

pkg/types/errors.go

@@ -0,0 +1,29 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package types
+
+import (
+	"fmt"
+)
+
+type NoFieldError struct {
+	Field string
+}
+
+func (e NoFieldError) Error() string {
+	return fmt.Sprintf("no field named '%s'", e.Field)
+}

plugins/someteam.example.com/v1/ConfigMapGenerator

@@ -6,6 +6,6 @@ apiVersion: v1
 metadata:
   name: example-configmap-test
 data:
-  username: admin
+  username: $1
-  password: secret
+  password: $2
 "

plugins/someteam.example.com/v1/SedTransformer

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+args=""
+
+for arg in $@; do
+  args="$args -e $arg"
+done
+
+cat - | sed $args