Merge pull request #760 from sethpollack/plugins

add secret and configmap generator plugins

k8sdeps/configmapandsecret/basefactory.go

@@ -24,6 +24,7 @@ import (
 	"github.com/pkg/errors"
 	"k8s.io/apimachinery/pkg/util/validation"
 	"sigs.k8s.io/kustomize/k8sdeps/kv"
+	"sigs.k8s.io/kustomize/k8sdeps/kv/plugin"
 	"sigs.k8s.io/kustomize/pkg/ifc"
 	"sigs.k8s.io/kustomize/pkg/types"
 )
@@ -32,11 +33,20 @@ import (
 type baseFactory struct {
 	ldr     ifc.Loader
 	options *types.GeneratorOptions
+	reg     plugin.Registry
 }
 
 func (bf baseFactory) loadKvPairs(
 	args types.GeneratorArgs) (all []kv.Pair, err error) {
-	pairs, err := bf.keyValuesFromEnvFile(args.EnvSource)
+	pairs, err := bf.keyValuesFromPlugins(args.KVSources)
+	if err != nil {
+		return nil, errors.Wrap(err, fmt.Sprintf(
+			"plugins: %s",
+			args.EnvSource))
+	}
+	all = append(all, pairs...)
+
+	pairs, err = bf.keyValuesFromEnvFile(args.EnvSource)
 	if err != nil {
 		return nil, errors.Wrap(err, fmt.Sprintf(
 			"env source file: %s",
@@ -81,6 +91,22 @@ func keyValuesFromLiteralSources(sources []string) ([]kv.Pair, error) {
 	return kvs, nil
 }
 
+func (bf baseFactory) keyValuesFromPlugins(sources []types.KVSource) ([]kv.Pair, error) {
+	var allKvs []kv.Pair
+	for _, s := range sources {
+		plug, err := bf.reg.Load(s.PluginType, s.Name)
+		if err != nil {
+			return nil, err
+		}
+		kvs, err := plug.Get(bf.reg.Root(), s.Args)
+		if err != nil {
+			return nil, err
+		}
+		allKvs = append(allKvs, kvs...)
+	}
+	return allKvs, nil
+}
+
 func (bf baseFactory) keyValuesFromFileSources(sources []string) ([]kv.Pair, error) {
 	var kvs []kv.Pair
 	for _, s := range sources {

k8sdeps/configmapandsecret/basefactory_test.go

@@ -21,8 +21,10 @@ import (
 	"testing"
 
 	"sigs.k8s.io/kustomize/k8sdeps/kv"
+	"sigs.k8s.io/kustomize/k8sdeps/kv/plugin"
 	"sigs.k8s.io/kustomize/pkg/fs"
 	"sigs.k8s.io/kustomize/pkg/loader"
+	"sigs.k8s.io/kustomize/pkg/types"
 )
 
 func TestKeyValuesFromFileSources(t *testing.T) {
@@ -45,7 +47,9 @@ func TestKeyValuesFromFileSources(t *testing.T) {
 
 	fSys := fs.MakeFakeFS()
 	fSys.WriteFile("/files/app-init.ini", []byte("FOO=bar"))
-	bf := baseFactory{loader.NewFileLoaderAtRoot(fSys), nil}
+	ldr := loader.NewFileLoaderAtRoot(fSys)
+	reg := plugin.NewRegistry(ldr)
+	bf := baseFactory{loader.NewFileLoaderAtRoot(fSys), nil, reg}
 	for _, tc := range tests {
 		kvs, err := bf.keyValuesFromFileSources(tc.sources)
 		if err != nil {
@@ -56,3 +60,51 @@ func TestKeyValuesFromFileSources(t *testing.T) {
 		}
 	}
 }
+
+func TestKeyValuesFromPlugins(t *testing.T) {
+	tests := []struct {
+		description string
+		sources     []types.KVSource
+		expected    []kv.Pair
+	}{
+		{
+			description: "Create kv.Pairs from plugin",
+			sources: []types.KVSource{
+				{
+					PluginType: "testonly",
+					Name:       "testonly",
+					Args:       []string{"FOO", "BAR", "BAZ"},
+				},
+			},
+			expected: []kv.Pair{
+				{
+					Key:   "k_FOO",
+					Value: "v_FOO",
+				},
+				{
+					Key:   "k_BAR",
+					Value: "v_BAR",
+				},
+				{
+					Key:   "k_BAZ",
+					Value: "v_BAZ",
+				},
+			},
+		},
+	}
+
+	fSys := fs.MakeFakeFS()
+	ldr := loader.NewFileLoaderAtRoot(fSys)
+	reg := plugin.NewRegistry(ldr)
+	bf := baseFactory{ldr, nil, reg}
+
+	for _, tc := range tests {
+		kvs, err := bf.keyValuesFromPlugins(tc.sources)
+		if err != nil {
+			t.Fatalf("unexpected error: %v", err)
+		}
+		if !reflect.DeepEqual(kvs, tc.expected) {
+			t.Fatalf("in testcase: %q updated:\n%#v\ndoesn't match expected:\n%#v\n", tc.description, kvs, tc.expected)
+		}
+	}
+}

k8sdeps/configmapandsecret/configmapfactory.go

@@ -21,8 +21,9 @@ import (
 	"fmt"
 	"unicode/utf8"
 
-	"k8s.io/api/core/v1"
 	corev1 "k8s.io/api/core/v1"
+	v1 "k8s.io/api/core/v1"
+	"sigs.k8s.io/kustomize/k8sdeps/kv/plugin"
 	"sigs.k8s.io/kustomize/pkg/ifc"
 	"sigs.k8s.io/kustomize/pkg/types"
 )
@@ -34,8 +35,8 @@ type Factory struct {
 
 // NewFactory returns a new Factory.
 func NewFactory(
-	l ifc.Loader, o *types.GeneratorOptions) *Factory {
-	return &Factory{baseFactory{ldr: l, options: o}}
+	l ifc.Loader, o *types.GeneratorOptions, reg plugin.Registry) *Factory {
+	return &Factory{baseFactory{ldr: l, options: o, reg: reg}}
 }
 
 func makeFreshConfigMap(

k8sdeps/configmapandsecret/configmapfactory_test.go

@@ -22,6 +22,7 @@ import (
 
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/kustomize/k8sdeps/kv/plugin"
 	"sigs.k8s.io/kustomize/pkg/fs"
 	"sigs.k8s.io/kustomize/pkg/loader"
 	"sigs.k8s.io/kustomize/pkg/types"
@@ -141,8 +142,10 @@ func TestConstructConfigMap(t *testing.T) {
 	fSys.WriteFile("/configmap/app.env", []byte("DB_USERNAME=admin\nDB_PASSWORD=somepw\n"))
 	fSys.WriteFile("/configmap/app-init.ini", []byte("FOO=bar\nBAR=baz\n"))
 	fSys.WriteFile("/configmap/app.bin", []byte{0xff, 0xfd})
+	ldr := loader.NewFileLoaderAtRoot(fSys)
+	reg := plugin.NewRegistry(ldr)
 	for _, tc := range testCases {
-		f := NewFactory(loader.NewFileLoaderAtRoot(fSys), tc.options)
+		f := NewFactory(ldr, tc.options, reg)
 		cm, err := f.MakeConfigMap(&tc.input)
 		if err != nil {
 			t.Fatalf("unexpected error: %v", err)

k8sdeps/configmapandsecret/secretfactory_test.go

@@ -22,6 +22,7 @@ import (
 
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/kustomize/k8sdeps/kv/plugin"
 	"sigs.k8s.io/kustomize/pkg/fs"
 	"sigs.k8s.io/kustomize/pkg/loader"
 	"sigs.k8s.io/kustomize/pkg/types"
@@ -138,8 +139,10 @@ func TestConstructSecret(t *testing.T) {
 	fSys := fs.MakeFakeFS()
 	fSys.WriteFile("/secret/app.env", []byte("DB_USERNAME=admin\nDB_PASSWORD=somepw\n"))
 	fSys.WriteFile("/secret/app-init.ini", []byte("FOO=bar\nBAR=baz\n"))
+	ldr := loader.NewFileLoaderAtRoot(fSys)
+	reg := plugin.NewRegistry(ldr)
 	for _, tc := range testCases {
-		f := NewFactory(loader.NewFileLoaderAtRoot(fSys), tc.options)
+		f := NewFactory(ldr, tc.options, reg)
 		cm, err := f.MakeSecret(&tc.input)
 		if err != nil {
 			t.Fatalf("unexpected error: %v", err)

k8sdeps/kunstruct/factory.go

@@ -25,6 +25,7 @@ import (
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/util/yaml"
 	"sigs.k8s.io/kustomize/k8sdeps/configmapandsecret"
+	"sigs.k8s.io/kustomize/k8sdeps/kv/plugin"
 	"sigs.k8s.io/kustomize/pkg/ifc"
 	"sigs.k8s.io/kustomize/pkg/types"
 )
@@ -81,7 +82,7 @@ func (kf *KunstructuredFactoryImpl) MakeConfigMap(
 	ldr ifc.Loader,
 	options *types.GeneratorOptions,
 	args *types.ConfigMapArgs) (ifc.Kunstructured, error) {
-	o, err := configmapandsecret.NewFactory(ldr, options).MakeConfigMap(args)
+	o, err := configmapandsecret.NewFactory(ldr, options, plugin.NewRegistry(ldr)).MakeConfigMap(args)
 	if err != nil {
 		return nil, err
 	}
@@ -93,7 +94,7 @@ func (kf *KunstructuredFactoryImpl) MakeSecret(
 	ldr ifc.Loader,
 	options *types.GeneratorOptions,
 	args *types.SecretArgs) (ifc.Kunstructured, error) {
-	o, err := configmapandsecret.NewFactory(ldr, options).MakeSecret(args)
+	o, err := configmapandsecret.NewFactory(ldr, options, plugin.NewRegistry(ldr)).MakeSecret(args)
 	if err != nil {
 		return nil, err
 	}

k8sdeps/kv/plugin/goplugin.go

@@ -0,0 +1,63 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package plugin
+
+import (
+	"fmt"
+	"os"
+	"plugin"
+)
+
+var _ Factory = &goFactory{}
+
+const (
+	dir = "$HOME/.config/kustomize/plugins/kvsource"
+)
+
+func newGoFactory() *goFactory {
+	return &goFactory{
+		plugins: make(map[string]KVSource),
+	}
+}
+
+type goFactory struct {
+	plugins map[string]KVSource
+}
+
+func (p *goFactory) load(name string) (KVSource, error) {
+	if plug, ok := p.plugins[name]; ok {
+		return plug, nil
+	}
+
+	goPlugin, err := plugin.Open(fmt.Sprintf("%s/kustomize-%s.so", os.ExpandEnv(dir), name))
+	if err != nil {
+		return nil, err
+	}
+
+	symbol, err := goPlugin.Lookup("Plugin")
+	if err != nil {
+		return nil, err
+	}
+
+	plug, ok := symbol.(KVSource)
+	if !ok {
+		return nil, fmt.Errorf("plugin %s not found", name)
+	}
+
+	p.plugins[name] = plug
+	return plug, nil
+}

k8sdeps/kv/plugin/plugin.go

@@ -0,0 +1,32 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package plugin provides a plugin abstraction layer.
+package plugin
+
+import (
+	"sigs.k8s.io/kustomize/k8sdeps/kv"
+)
+
+// KVSource is the interface for kv source plugins.
+type KVSource interface {
+	Get(root string, args []string) ([]kv.Pair, error)
+}
+
+// Factory is the interface for new kv source plugin implementations.
+type Factory interface {
+	load(string) (KVSource, error)
+}

k8sdeps/kv/plugin/registry.go

@@ -0,0 +1,54 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package plugin
+
+import (
+	"fmt"
+
+	"sigs.k8s.io/kustomize/pkg/ifc"
+)
+
+// Registry holds all the plugin factories.
+type Registry struct {
+	factories map[string]Factory
+	ldr       ifc.Loader
+}
+
+// NewRegistry returns a new Registry loaded with all the factories.
+func NewRegistry(ldr ifc.Loader) Registry {
+	return Registry{
+		ldr: ldr,
+		factories: map[string]Factory{
+			"go":       newGoFactory(),
+			"testonly": newTestonlyFactory(),
+		},
+	}
+}
+
+// Load returns a plugin by type and name,
+func (r *Registry) Load(pluginType, name string) (KVSource, error) {
+	factory, exists := r.factories[pluginType]
+	if !exists {
+		return nil, fmt.Errorf("%s is not a valid plugin type", pluginType)
+	}
+	return factory.load(name)
+}
+
+// Root returns the root of the plugins kustomization file.
+func (r *Registry) Root() string {
+	return r.ldr.Root()
+}

k8sdeps/kv/plugin/testonly.go

@@ -0,0 +1,43 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// testonly is temporary until we have builtin plugins to use in the tests.
+
+package plugin
+
+import (
+	"sigs.k8s.io/kustomize/k8sdeps/kv"
+)
+
+var _ Factory = &testonlyFactory{}
+
+func newTestonlyFactory() *testonlyFactory {
+	return &testonlyFactory{}
+}
+
+type testonlyFactory struct{}
+
+func (p testonlyFactory) Get(_ string, args []string) ([]kv.Pair, error) {
+	var kvs []kv.Pair
+	for _, arg := range args {
+		kvs = append(kvs, kv.Pair{Key: "k_" + arg, Value: "v_" + arg})
+	}
+	return kvs, nil
+}
+
+func (p *testonlyFactory) load(_ string) (KVSource, error) {
+	return p, nil
+}

pkg/target/configmaps_test.go

@@ -240,3 +240,31 @@ metadata:
   name: p2-com2-c4b8md75k9
 `)
 }
+
+func TestGeneratorPlugins(t *testing.T) {
+	th := NewKustTestHarness(t, "/app")
+	th.writeK("/app", `
+secretGenerator:
+- name: bob
+  kvSources:
+  - pluginType: testonly
+    name: testonly
+    args:
+    - FRUIT
+    - VEGETABLE
+`)
+	m, err := th.makeKustTarget().MakeCustomizedResMap()
+	if err != nil {
+		t.Fatalf("Err: %v", err)
+	}
+	th.assertActualEqualsExpected(m, `
+apiVersion: v1
+data:
+  k_FRUIT: dl9GUlVJVA==
+  k_VEGETABLE: dl9WRUdFVEFCTEU=
+kind: Secret
+metadata:
+  name: bob-cb9mhbh9gg
+type: Opaque
+`)
+}

pkg/types/kustomization.go

@@ -189,6 +189,9 @@ type GeneratorArgs struct {
 
 	// DataSources for the generator.
 	DataSources `json:",inline,omitempty" yaml:",inline,omitempty"`
+
+	// KVSources for the generator.
+	KVSources []KVSource `json:",inline,omitempty" yaml:",inline,omitempty"`
 }
 
 // ConfigMapArgs contains the metadata of how to generate a configmap.
@@ -248,3 +251,10 @@ type GeneratorOptions struct {
 	// resource contents.
 	DisableNameSuffixHash bool `json:"disableNameSuffixHash,omitempty" yaml:"disableNameSuffixHash,omitempty"`
 }
+
+// KVSource represents a KV plugin backend.
+type KVSource struct {
+	PluginType string   `json:"pluginType,omitempty" yaml:"pluginType,omitempty"`
+	Name       string   `json:"name,omitempty" yaml:"name,omitempty"`
+	Args       []string `json:"args,omitempty" yaml:"args,omitempty"`
+}