Helm chart generator exec plugin

.travis.yml

@@ -1,8 +1,8 @@
 os:
-# TODO: Enable this when we can get the tests to work
-#  - windows
   - linux
   - osx
+# TODO: Uncomment when tests running on Windows.
+#  - windows
 
 addons:
   apt:
@@ -30,6 +30,10 @@ before_install:
   - source ./bin/consider-early-travis-exit.sh
   - curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | sh -s -- -b $GOPATH/bin ${GOLANGCI_RELEASE}
   - go get -u github.com/monopole/mdrip
+  # The following would install Helm if needed for some reason.
+  # - wget https://storage.googleapis.com/kubernetes-helm/helm-v2.13.1-linux-amd64.tar.gz
+  # - tar -xvzf helm-v2.13.1-linux-amd64.tar.gz
+  # - sudo mv linux-amd64/helm /usr/local/bin/helm
 
 # Skip the install process; let pre-commit.sh do it.
 install: true

bin/pre-commit.sh

@@ -32,6 +32,11 @@ function testGoTest {
   go test -v ./...
 }
 
+function testNoTravisGoTest {
+  go test -v sigs.k8s.io/kustomize/pkg/target \
+      -run TestChartInflatorExecPlugin -tags=notravis
+}
+
 function testExamples {
   mdrip --mode test --label test README.md ./examples
 }
@@ -83,6 +88,11 @@ echo "Beginning tests..."
 runTest testGoLangCILint
 runTest testGoTest
 
+if [ -z ${TRAVIS+x} ]; then
+  echo Not on travis, so running the notravis tests
+  runTest testNoTravisGoTest
+fi
+
 PATH=$HOME/go/bin:$PATH
 runTest testExamples
 

pkg/plugins/execplugin.go

@@ -19,34 +19,38 @@ package plugins
 import (
 	"bytes"
 	"fmt"
+	"io/ioutil"
 	"os"
 	"os/exec"
 	"path/filepath"
-	"sigs.k8s.io/kustomize/pkg/types"
 	"strings"
+	"syscall"
 
 	"github.com/ghodss/yaml"
+	"sigs.k8s.io/kustomize/k8sdeps/kv/plugin"
 	"sigs.k8s.io/kustomize/pkg/ifc"
 	"sigs.k8s.io/kustomize/pkg/pgmconfig"
 	"sigs.k8s.io/kustomize/pkg/resmap"
 )
 
+const (
+	ArgsOneLiner = "argsOneLiner"
+	ArgsFromFile = "argsFromFile"
+)
+
 // ExecPlugin record the name and args of an executable
 // It triggers the executable generator and transformer
 type ExecPlugin struct {
 	// name of the executable
 	name string
 
-	// one line of arguments for the executable
+	// Optional command line arguments to the executable
-	argOneLiner string
+	// pulled from specially named fields in cfg.
+	// This is for executables that don't want to parse YAML.
+	args []string
 
-	// relative file path to a file
+	// Plugin configuration data.
-	// Each line of this file is treated as one argument
+	cfg []byte
-	argsFromFile string
-
-	// cfg hold the unstructured data which can be used
-	// to configure the plugin
-	cfg string
 
 	// resmap Factory to make resources
 	rf *resmap.Factory
@@ -57,30 +61,66 @@ type ExecPlugin struct {
 
 func (p *ExecPlugin) Config(
 	ldr ifc.Loader, rf *resmap.Factory, k ifc.Kunstructured) error {
-	dir := filepath.Join(pgmconfig.ConfigRoot(), "plugins")
+	dir := filepath.Join(pgmconfig.ConfigRoot(), plugin.PluginRoot)
 	id := k.GetGvk()
 	p.name = filepath.Join(dir, id.Group, id.Version, id.Kind)
 	p.rf = rf
 	p.ldr = ldr
 
 	var err error
-	data, err := yaml.Marshal(k)
+	p.cfg, err = yaml.Marshal(k)
 	if err != nil {
 		return err
 	}
-	p.cfg = string(data)
+	err = p.processOptionalArgsFields(k)
-
+	if err != nil {
-	p.argOneLiner, err = k.GetFieldValue("arg")
-	if err != nil && !isNoFieldError(err) {
-		return err
-	}
-	p.argsFromFile, err = k.GetFieldValue("file")
-	if err != nil && !isNoFieldError(err) {
 		return err
 	}
 	return nil
 }
 
+func (p *ExecPlugin) processOptionalArgsFields(k ifc.Kunstructured) error {
+	args, err := k.GetFieldValue(ArgsOneLiner)
+	if err == nil && args != "" {
+		p.args = strings.Split(args, " ")
+	}
+	fileName, err := k.GetFieldValue(ArgsFromFile)
+	if err == nil && fileName != "" {
+		content, err := p.ldr.Load(fileName)
+		if err != nil {
+			return err
+		}
+		for _, x := range strings.Split(string(content), "\n") {
+			x := strings.TrimLeft(x, " ")
+			if x != "" {
+				p.args = append(p.args, x)
+			}
+		}
+	}
+	return nil
+}
+
+func (p *ExecPlugin) writeConfig() (string, error) {
+	tmpFile, err := ioutil.TempFile("", "kust-pipe")
+	if err != nil {
+		return "", err
+	}
+	syscall.Mkfifo(tmpFile.Name(), 0600)
+	stdout, err := os.OpenFile(tmpFile.Name(), os.O_RDWR, 0600)
+	if err != nil {
+		return "", err
+	}
+	_, err = stdout.Write(p.cfg)
+	if err != nil {
+		return "", err
+	}
+	err = stdout.Close()
+	if err != nil {
+		return "", err
+	}
+	return tmpFile.Name(), nil
+}
+
 func (p *ExecPlugin) Generate() (resmap.ResMap, error) {
 	args, err := p.getArgs()
 	if err != nil {
@@ -101,7 +141,6 @@ func (p *ExecPlugin) Transform(rm resmap.ResMap) error {
 	if err != nil {
 		return err
 	}
-
 	for id, r := range rm {
 		content, err := yaml.Marshal(r.Kunstructured)
 		if err != nil {
@@ -129,28 +168,18 @@ func (p *ExecPlugin) Transform(rm resmap.ResMap) error {
 	return nil
 }
 
+// The first arg is always the absolute path to a temporary file
+// holding the YAML form of the plugin config.
 func (p *ExecPlugin) getArgs() ([]string, error) {
-	args := strings.Split(p.argOneLiner, " ")
+	configFileName, err := p.writeConfig()
-	if p.argsFromFile != "" {
-		content, err := p.ldr.Load(p.argsFromFile)
 	if err != nil {
 		return nil, err
 	}
-		args = append(args, strings.Split(string(content), "\n")...)
+	return append([]string{configFileName}, p.args...), nil
-	}
-	return args, nil
 }
 
 func (p *ExecPlugin) getEnv() []string {
 	env := os.Environ()
-	env = append(env, "KUSTOMIZE_PLUGIN_CONFIG_STRING="+p.cfg)
+	env = append(env, "KUSTOMIZE_PLUGIN_CONFIG_STRING="+string(p.cfg))
 	return env
 }
-
-func isNoFieldError(e error) bool {
-	_, ok := e.(types.NoFieldError)
-	if ok {
-		return true
-	}
-	return false
-}

pkg/plugins/execplugin_test.go

@@ -0,0 +1,79 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+ Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+     http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package plugins
+
+import (
+	"strings"
+	"testing"
+
+	"sigs.k8s.io/kustomize/k8sdeps/kunstruct"
+	"sigs.k8s.io/kustomize/pkg/internal/loadertest"
+	"sigs.k8s.io/kustomize/pkg/resmap"
+	"sigs.k8s.io/kustomize/pkg/resource"
+)
+
+func TestExecPluginConfig(t *testing.T) {
+	path := "/app"
+	kFactory := kunstruct.NewKunstructuredFactoryImpl()
+	rf := resmap.NewFactory(resource.NewFactory(kFactory))
+	ldr := loadertest.NewFakeLoader(path)
+
+	pluginConfig := kFactory.FromMap(
+		map[string]interface{}{
+			"apiVersion": "someteam.example.com/v1",
+			"kind":       "SedTransformer",
+			"metadata": map[string]interface{}{
+				"name": "some-random-name",
+			},
+			ArgsOneLiner: "one two",
+			ArgsFromFile: "sed-input.txt",
+		})
+
+	ldr.AddFile("/app/sed-input.txt", []byte(`
+s/$FOO/foo/g
+s/$BAR/bar/g
+ \ \ \ 
+`))
+
+	p := &ExecPlugin{}
+
+	p.Config(ldr, rf, pluginConfig)
+
+	expected := "/kustomize/plugins/someteam.example.com/v1/SedTransformer"
+	if !strings.HasSuffix(p.name, expected) {
+		t.Fatalf("expected suffix '%s', got '%s'", expected, p.name)
+	}
+
+	expected = `apiVersion: someteam.example.com/v1
+argsFromFile: sed-input.txt
+argsOneLiner: one two
+kind: SedTransformer
+metadata:
+  name: some-random-name
+`
+	if expected != string(p.cfg) {
+		t.Fatalf("expected cfg '%s', got '%s'", expected, string(p.cfg))
+
+	}
+	if len(p.args) != 5 {
+		t.Fatalf("unexpected arg len %d, %v", len(p.args), p.args)
+	}
+	if p.args[0] != "one" ||
+		p.args[1] != "two" ||
+		p.args[2] != "s/$FOO/foo/g" ||
+		p.args[3] != "s/$BAR/bar/g" ||
+		p.args[4] != "\\ \\ \\ " {
+		t.Fatalf("unexpected arg array: %v", p.args)
+	}
+}

pkg/target/chartinflatorexecplugin_test.go

@@ -0,0 +1,113 @@
+// +build notravis
+
+// Disabled on travis, because don't want to install helm on travis.
+
+/*
+Copyright 2019 The Kubernetes Authors.
+ Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+     http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package target_test
+
+import (
+	"testing"
+
+	"sigs.k8s.io/kustomize/k8sdeps/kv/plugin"
+)
+
+// TODO: Make this test less brittle.
+//
+// To test ChartInflatorExec, it downloads the latest
+// stable minecraft chart, inflates it with default values,
+// and demands an exact match.
+// Maybe just grep for particular strings instead.
+//
+// This test requires having the helm binary on the PATH.
+//
+func TestChartInflatorExecPlugin(t *testing.T) {
+	tc := NewTestEnvController(t).Set()
+	defer tc.Reset()
+
+	tc.BuildExecPlugin(
+		"kustomize.config.k8s.io", "v1", "ChartInflatorExec")
+
+	th := NewKustTestHarnessWithPluginConfig(
+		t, "/app", plugin.ActivePluginConfig())
+	th.writeK("/app", `
+generators:
+- chartInflatorExec.yaml
+namePrefix: LOOOOOOOONG-
+`)
+
+	th.writeF("/app/chartInflatorExec.yaml", `
+apiVersion: kustomize.config.k8s.io/v1
+kind: ChartInflatorExec
+metadata:
+  name: notImportantHere
+chart: minecraft
+`)
+
+	m, err := th.makeKustTarget().MakeCustomizedResMap()
+	if err != nil {
+		t.Fatalf("Err: %v", err)
+	}
+	th.assertActualEqualsExpected(m, `
+apiVersion: v1
+data:
+  rcon-password: Q0hBTkdFTUUh
+kind: Secret
+metadata:
+  labels:
+    app: release-name-minecraft
+    chart: minecraft-0.3.2
+    heritage: Tiller
+    release: release-name
+  name: LOOOOOOOONG-release-name-minecraft
+type: Opaque
+---
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app: release-name-minecraft
+    chart: minecraft-0.3.2
+    heritage: Tiller
+    release: release-name
+  name: LOOOOOOOONG-release-name-minecraft
+spec:
+  ports:
+  - name: minecraft
+    port: 25565
+    protocol: TCP
+    targetPort: minecraft
+  selector:
+    app: release-name-minecraft
+  type: LoadBalancer
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  annotations:
+    volume.alpha.kubernetes.io/storage-class: default
+  labels:
+    app: release-name-minecraft
+    chart: minecraft-0.3.2
+    heritage: Tiller
+    release: release-name
+  name: LOOOOOOOONG-release-name-minecraft-datadir
+spec:
+  accessModes:
+  - ReadWriteOnce
+  resources:
+    requests:
+      storage: 1Gi
+`)
+}

pkg/target/generatorplugin_test.go

@@ -146,7 +146,7 @@ apiVersion: someteam.example.com/v1
 kind: ConfigMapGenerator
 metadata:
   name: some-random-name
-arg: "admin secret"
+argsOneLiner: "admin secret"
 `)
 	m, err := th.makeKustTarget().MakeCustomizedResMap()
 	if err != nil {

pkg/target/transformerplugin_test.go

@@ -122,7 +122,7 @@ apiVersion: someteam.example.com/v1
 kind: SedTransformer
 metadata:
   name: some-random-name
-file: sed-input.txt
+argsFromFile: sed-input.txt
 `)
 	th.writeF("/app/sed-input.txt", `
 s/$FOO/foo/g

plugins/kustomize.config.k8s.io/v1/ChartInflatorExec

@@ -0,0 +1,77 @@
+#!/bin/bash
+set -e
+
+# Helm chart inflator
+
+# Reads a file like this
+#
+#  apiVersion: kustomize.config.k8s.io/v1
+#  kind: ChartInflatorExec
+#  metadata:
+#    name: notImportantHere
+#  chart: chartName
+#  values: path/to/values/file
+#  helmBin: path/to/helmBin
+#
+# fetches the given chart from stable/$chartName,
+# and inflates it to stdout, using the given values file.
+#
+# Example execution:
+# ./plugins/kustomize.config.k8s.io/v1/ChartInflatorExec configFile.yaml
+
+
+
+# Yaml parsing is a ridiculous thing to do in bash,
+# but let's try:
+function parseYaml {
+  local file=$1
+  while read -r line
+  do
+    local k=${line%:*}
+    local v=${line#*:}
+
+    [ "$k" == "chart" ] && chartName=$v
+    [ "$k" == "values" ] && valuesFile=$v
+    [ "$k" == "helmBin" ] && helmBin=$v
+  done <"$file"
+
+  # Trim leading space
+  chartName="${chartName#"${chartName%%[![:space:]]*}"}"
+  valuesFile="${valuesFile#"${valuesFile%%[![:space:]]*}"}"
+  helmBin="${helmBin#"${helmBin%%[![:space:]]*}"}"
+}
+
+TMP_DIR=$(mktemp -d)
+
+# Where all the files generated by 'helm init' live.
+HELM_HOME=$TMP_DIR/dotHelm
+
+# Where helm charts are unpacked.
+CHART_HOME=$TMP_DIR/charts
+
+parseYaml $1
+
+if [ -z "$helmBin" ]; then
+  helmBin=/usr/local/bin/helm
+fi
+
+if [ -z "$valuesFile" ]; then
+  valuesFile=$CHART_HOME/$chartName/values.yaml
+fi
+
+function doHelm {
+  $helmBin --home $HELM_HOME $@
+}
+
+# The init command is extremely chatty
+doHelm init --client-only >& /dev/null
+
+doHelm fetch --untar \
+    --untardir $CHART_HOME \
+    stable/$chartName
+
+doHelm template \
+    --values $valuesFile \
+    $CHART_HOME/$chartName
+
+/bin/rm -rf $TMP_DIR

plugins/someteam.example.com/v1/ConfigMapGenerator

@@ -1,5 +1,8 @@
 #!/bin/bash
 
+# Skip the config file name argument.
+shift
+
 echo "
 kind: ConfigMap
 apiVersion: v1

plugins/someteam.example.com/v1/SedTransformer

@@ -1,7 +1,9 @@
 #!/bin/bash
 
-args=""
+# Skip the config file name argument.
+shift
 
+args=""
 for arg in $@; do
   args="$args -e $arg"
 done