Merge branch 'master' into container-filter-network

kyaml/kio/byteio_reader.go

@@ -17,7 +17,7 @@ import (
 
 const (
 	ResourceListKind       = "ResourceList"
-	ResourceListApiVersion = "kyaml.kustomize.dev/v1alpha1"
+	ResourceListApiVersion = "config.kubernetes.io/v1alpha1"
 )
 
 // ByteReadWriter reads from an input and writes to an output.

kyaml/kio/byteio_reader_test.go

@@ -34,7 +34,7 @@ i: j
 }
 
 func TestByteReader_Read_wrappedResourceßßList(t *testing.T) {
-	r := &ByteReader{Reader: bytes.NewBufferString(`apiVersion: kyaml.kustomize.dev/v1alpha1
+	r := &ByteReader{Reader: bytes.NewBufferString(`apiVersion: config.kubernetes.io/v1alpha1
 kind: ResourceList
 functionConfig:
   foo: bar

kyaml/kio/byteio_writer_test.go

@@ -45,7 +45,7 @@ g:
 	if !assert.NoError(t, err) {
 		return
 	}
-	assert.Equal(t, `apiVersion: kyaml.kustomize.dev/v1alpha1
+	assert.Equal(t, `apiVersion: config.kubernetes.io/v1alpha1
 kind: ResourceList
 items:
 - c: d # second

kyaml/kio/filters/container.go

@@ -5,12 +5,13 @@ package filters
 
 import (
 	"bytes"
+	"fmt"
 	"os"
 	"os/exec"
-	"regexp"
 	"strings"
 
 	"sigs.k8s.io/kustomize/kyaml/kio"
+	"sigs.k8s.io/kustomize/kyaml/kio/kioutil"
 
 	"sigs.k8s.io/kustomize/kyaml/yaml"
 )
@@ -24,6 +25,8 @@ import (
 // The full set of environment variables from the parent process
 // are passed to the container.
 type ContainerFilter struct {
+	mountPath string
+
 	// Image is the container image to use to create a container.
 	Image string `yaml:"image,omitempty"`
 
@@ -41,6 +44,10 @@ type ContainerFilter struct {
 	checkInput func(string)
 }
 
+func (c *ContainerFilter) SetMountPath(path string) {
+	c.mountPath = path
+}
+
 // GrepFilter implements kio.GrepFilter
 func (c *ContainerFilter) Filter(input []*yaml.RNode) ([]*yaml.RNode, error) {
 	// get the command to filter the Resources
@@ -98,6 +105,10 @@ func (c *ContainerFilter) getArgs() []string {
 		// don't make fs readonly because things like heredoc rely on writing tmp files
 		"--security-opt=no-new-privileges", // don't allow the user to escalate privileges
 	}
+	// mount the directory containing the function as read-only
+	if c.mountPath != "" {
+		args = append(args, "-v", fmt.Sprintf("%s:/local/:ro", c.mountPath))
+	}
 
 	// export the local environment vars to the container
 	for _, pair := range os.Environ() {
@@ -150,7 +161,8 @@ type IsReconcilerFilter struct {
 func (c *IsReconcilerFilter) Filter(inputs []*yaml.RNode) ([]*yaml.RNode, error) {
 	var out []*yaml.RNode
 	for i := range inputs {
-		isContainerResource := GetContainerName(inputs[i]) != ""
+		img, _ := GetContainerName(inputs[i])
+		isContainerResource := img != ""
 		if isContainerResource && !c.ExcludeReconcilers {
 			out = append(out, inputs[i])
 		}
@@ -162,19 +174,20 @@ func (c *IsReconcilerFilter) Filter(inputs []*yaml.RNode) ([]*yaml.RNode, error)
 }
 
 // GetContainerName returns the container image for an API if one exists
-func GetContainerName(n *yaml.RNode) string {
+func GetContainerName(n *yaml.RNode) (string, string) {
 	meta, _ := n.GetMeta()
-	container := meta.Annotations["kyaml.kustomize.dev/container"]
+
+	// path to the function, this will be mounted into the container
+	path := meta.Annotations[kioutil.PathAnnotation]
+
+	container := meta.Annotations["config.kubernetes.io/container"]
 	if container != "" {
-		return container
+		return container, path
 	}
 
-	if match.MatchString(meta.ApiVersion) {
-		return meta.ApiVersion
+	image, err := n.Pipe(yaml.Lookup("metadata", "configFn", "container", "image"))
+	if err != nil || yaml.IsMissingOrNull(image) {
+		return "", path
 	}
-
-	return ""
+	return image.YNode().Value, path
 }
-
-// match specifies the set of apiVersions to recognize as being container images
-var match = regexp.MustCompile(`(docker\.io|.*\.?gcr\.io)/.*(:.*)?`)

kyaml/kio/filters/container_test.go

@@ -5,7 +5,9 @@ package filters
 
 import (
 	"bytes"
+	"fmt"
 	"os"
+	"path/filepath"
 	"strings"
 	"testing"
 
@@ -60,6 +62,43 @@ metadata:
 	assert.True(t, foundKyaml)
 }
 
+func TestFilter_commandMountPath(t *testing.T) {
+	cfg, err := yaml.Parse(`apiversion: apps/v1
+kind: Deployment
+metadata:
+  name: foo
+`)
+	if !assert.NoError(t, err) {
+		return
+	}
+	instance := &ContainerFilter{
+		Image:     "example.com:version",
+		Config:    cfg,
+		mountPath: filepath.Join("mount", "path"),
+	}
+	os.Setenv("KYAML_TEST", "FOO")
+	cmd, err := instance.getCommand()
+	if !assert.NoError(t, err) {
+		return
+	}
+
+	expected := []string{
+		"docker", "run",
+		"--rm",
+		"-i", "-a", "STDIN", "-a", "STDOUT", "-a", "STDERR",
+		"--network", "none",
+		"--user", "nobody",
+		"--security-opt=no-new-privileges",
+		"-v", fmt.Sprintf("%s:/local/:ro", filepath.Join("mount", "path")),
+	}
+	for _, e := range os.Environ() {
+		// the process env
+		expected = append(expected, "-e", strings.Split(e, "=")[0])
+	}
+	expected = append(expected, "example.com:version")
+	assert.Equal(t, expected, cmd.Args)
+}
+
 func TestFilter_command_network(t *testing.T) {
 	cfg, err := yaml.Parse(`apiversion: apps/v1
 kind: Deployment
@@ -127,7 +166,7 @@ metadata:
 		args:   []string{"sed", "s/Deployment/StatefulSet/g"},
 		checkInput: func(s string) {
 			called = true
-			if !assert.Equal(t, `apiVersion: kyaml.kustomize.dev/v1alpha1
+			if !assert.Equal(t, `apiVersion: config.kubernetes.io/v1alpha1
 kind: ResourceList
 items:
 - apiVersion: apps/v1
@@ -209,7 +248,7 @@ metadata:
 		args:   []string{"sh", "-c", "cat <&0"},
 		checkInput: func(s string) {
 			called = true
-			if !assert.Equal(t, `apiVersion: kyaml.kustomize.dev/v1alpha1
+			if !assert.Equal(t, `apiVersion: config.kubernetes.io/v1alpha1
 kind: ResourceList
 items:
 - apiversion: apps/v1
@@ -261,36 +300,30 @@ metadata:
 
 func Test_GetContainerName(t *testing.T) {
 	// make sure gcr.io works
-	n, err := yaml.Parse(`apiVersion: gcr.io/foo/bar:something
+	n, err := yaml.Parse(`apiVersion: v1beta1
 kind: MyThing
+metadata:
+  configFn:
+    container:
+      image: gcr.io/foo/bar:something
 `)
 	if !assert.NoError(t, err) {
 		return
 	}
-	c := GetContainerName(n)
+	c, _ := GetContainerName(n)
 	assert.Equal(t, "gcr.io/foo/bar:something", c)
 
-	// make sure regional gcr.io works
-	n, err = yaml.Parse(`apiVersion: us.gcr.io/foo/bar:something
-kind: MyThing
-`)
-	if !assert.NoError(t, err) {
-		return
-	}
-	c = GetContainerName(n)
-	assert.Equal(t, "us.gcr.io/foo/bar:something", c)
-
 	// container from annotation
 	n, err = yaml.Parse(`apiVersion: v1
 kind: MyThing
 metadata:
   annotations:
-    kyaml.kustomize.dev/container: gcr.io/foo/bar:something
+    config.kubernetes.io/container: gcr.io/foo/bar:something
 `)
 	if !assert.NoError(t, err) {
 		return
 	}
-	c = GetContainerName(n)
+	c, _ = GetContainerName(n)
 	assert.Equal(t, "gcr.io/foo/bar:something", c)
 
 	// doesn't have a container
@@ -301,16 +334,6 @@ metadata:
 	if !assert.NoError(t, err) {
 		return
 	}
-	c = GetContainerName(n)
+	c, _ = GetContainerName(n)
 	assert.Equal(t, "", c)
-
-	// make sure docker.io works
-	n, err = yaml.Parse(`apiVersion: docker.io/foo/bar:something
-kind: MyThing
-`)
-	if !assert.NoError(t, err) {
-		return
-	}
-	c = GetContainerName(n)
-	assert.Equal(t, "docker.io/foo/bar:something", c)
 }

kyaml/kio/filters/local.go

@@ -0,0 +1,38 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package filters
+
+import (
+	"sigs.k8s.io/kustomize/kyaml/yaml"
+)
+
+const LocalConfigAnnotation = "config.kubernetes.io/local-config"
+
+// IsLocalConfig filters Resources using the config.kubernetes.io/local-config annotation
+type IsLocalConfig struct {
+	// IncludeLocalConfig will include local-config if set to true
+	IncludeLocalConfig bool `yaml:"includeLocalConfig,omitempty"`
+
+	// ExcludeNonLocalConfig will exclude non local-config if set to true
+	ExcludeNonLocalConfig bool `yaml:"excludeNonLocalConfig,omitempty"`
+}
+
+// Filter implements kio.Filter
+func (c *IsLocalConfig) Filter(inputs []*yaml.RNode) ([]*yaml.RNode, error) {
+	var out []*yaml.RNode
+	for i := range inputs {
+		meta, err := inputs[i].GetMeta()
+		if err != nil {
+			return nil, err
+		}
+		_, local := meta.Annotations[LocalConfigAnnotation]
+
+		if local && c.IncludeLocalConfig {
+			out = append(out, inputs[i])
+		} else if !local && !c.ExcludeNonLocalConfig {
+			out = append(out, inputs[i])
+		}
+	}
+	return out, nil
+}

kyaml/kio/filters/merge3.go

@@ -12,7 +12,7 @@ import (
 )
 
 const (
-	mergeSourceAnnotation = "kyaml.kustomize.dev/merge-source"
+	mergeSourceAnnotation = "config.kubernetes.io/merge-source"
 	mergeSourceOriginal   = "original"
 	mergeSourceUpdated    = "updated"
 	mergeSourceDest       = "dest"

kyaml/runfn/runfn.go

@@ -0,0 +1,98 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package runfn
+
+import (
+	"io"
+	"path/filepath"
+
+	"sigs.k8s.io/kustomize/kyaml/errors"
+	"sigs.k8s.io/kustomize/kyaml/kio"
+	"sigs.k8s.io/kustomize/kyaml/kio/filters"
+	"sigs.k8s.io/kustomize/kyaml/yaml"
+)
+
+// RunFns runs the set of configuration functions in a local directory against
+// the Resources in that directory
+type RunFns struct {
+	// Path is the path to the directory containing functions
+	Path string
+
+	// FunctionPaths Paths allows functions to be specified outside the configuration
+	// directory
+	FunctionPaths []string
+
+	// Output can be set to write the result to Output rather than back to the directory
+	Output io.Writer
+
+	// containerFilterProvider may be override by tests to fake invoking containers
+	containerFilterProvider func(string, string, *yaml.RNode) kio.Filter
+}
+
+// Execute runs the command
+func (r RunFns) Execute() error {
+	// make the path absolute so it works on mac
+	var err error
+	r.Path, err = filepath.Abs(r.Path)
+	if err != nil {
+		return errors.Wrap(err)
+	}
+
+	// default the containerFilterProvider if it hasn't been override.  Split out for testing.
+	(&r).init()
+
+	// identify the configuration functions in the directory
+	buff := &kio.PackageBuffer{}
+	err = kio.Pipeline{
+		Inputs:  []kio.Reader{kio.LocalPackageReader{PackagePath: r.Path}},
+		Filters: []kio.Filter{&filters.IsReconcilerFilter{}},
+		Outputs: []kio.Writer{buff},
+	}.Execute()
+	if err != nil {
+		return err
+	}
+
+	// accept a
+	for i := range r.FunctionPaths {
+		err := kio.Pipeline{
+			Inputs:  []kio.Reader{kio.LocalPackageReader{PackagePath: r.FunctionPaths[i]}},
+			Outputs: []kio.Writer{buff},
+		}.Execute()
+		if err != nil {
+			return err
+		}
+	}
+
+	// reconcile each local API
+	var fltrs []kio.Filter
+	for i := range buff.Nodes {
+		api := buff.Nodes[i]
+		img, path := filters.GetContainerName(api)
+		fltrs = append(fltrs, r.containerFilterProvider(img, path, api))
+	}
+
+	pkgIO := &kio.LocalPackageReadWriter{PackagePath: r.Path}
+	inputs := []kio.Reader{pkgIO}
+	var outputs []kio.Writer
+	if r.Output == nil {
+		// write back to the package
+		outputs = append(outputs, pkgIO)
+	} else {
+		// write to the output instead of the directory
+		outputs = append(outputs, kio.ByteWriter{Writer: r.Output})
+	}
+	return kio.Pipeline{Inputs: inputs, Filters: fltrs, Outputs: outputs}.Execute()
+}
+
+// init initializes the RunFns with a containerFilterProvider.
+func (r *RunFns) init() {
+	// if containerFilterProvider hasn't been set, use the default
+	if r.containerFilterProvider == nil {
+		r.containerFilterProvider = func(image, path string, api *yaml.RNode) kio.Filter {
+			cf := &filters.ContainerFilter{Image: image, Config: api}
+			cf.SetMountPath(filepath.Join(r.Path, path))
+			return cf
+		}
+	}
+}

kyaml/runfn/runfn_test.go

@@ -0,0 +1,248 @@
+// Copyright 2019 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package runfn
+
+import (
+	"bytes"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"runtime"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"sigs.k8s.io/kustomize/kyaml/copyutil"
+	"sigs.k8s.io/kustomize/kyaml/kio"
+	"sigs.k8s.io/kustomize/kyaml/kio/filters"
+	"sigs.k8s.io/kustomize/kyaml/yaml"
+)
+
+func TestRunFns_Execute(t *testing.T) {
+	instance := RunFns{}
+	instance.init()
+	api, err := yaml.Parse(`apiVersion: apps/v1
+kind: 
+`)
+	if !assert.NoError(t, err) {
+		return
+	}
+	filter := instance.containerFilterProvider("example.com:version", "", api)
+	assert.Equal(t, &filters.ContainerFilter{Image: "example.com:version", Config: api}, filter)
+}
+
+func TestCmd_Execute(t *testing.T) {
+	dir, err := ioutil.TempDir("", "kustomize-kyaml-test")
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
+	defer os.RemoveAll(dir)
+
+	_, filename, _, ok := runtime.Caller(0)
+	if !assert.True(t, ok) {
+		t.FailNow()
+	}
+	ds, err := filepath.Abs(filepath.Join(filepath.Dir(filename), "test", "testdata"))
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
+	if !assert.NoError(t, copyutil.CopyDir(ds, dir)) {
+		t.FailNow()
+	}
+	if !assert.NoError(t, os.Chdir(filepath.Dir(dir))) {
+		return
+	}
+
+	// write a test filter
+	f := `apiVersion: v1
+kind: ValueReplacer
+metadata:
+  configFn:
+    container:
+      image: gcr.io/example.com/image:version
+stringMatch: Deployment
+replace: StatefulSet
+`
+	if !assert.NoError(t, ioutil.WriteFile(
+		filepath.Join(dir, "filter.yaml"), []byte(f), 0600)) {
+		return
+	}
+
+	instance := RunFns{
+		Path: dir,
+		containerFilterProvider: func(s, _ string, node *yaml.RNode) kio.Filter {
+			// parse the filter from the input
+			filter := yaml.YFilter{}
+			b := &bytes.Buffer{}
+			e := yaml.NewEncoder(b)
+			if !assert.NoError(t, e.Encode(node.YNode())) {
+				t.FailNow()
+			}
+			e.Close()
+			d := yaml.NewDecoder(b)
+			if !assert.NoError(t, d.Decode(&filter)) {
+				t.FailNow()
+			}
+
+			return filters.Modifier{
+				Filters: []yaml.YFilter{{Filter: yaml.Lookup("kind")}, filter},
+			}
+		},
+	}
+	if !assert.NoError(t, instance.Execute()) {
+		t.FailNow()
+	}
+	b, err := ioutil.ReadFile(
+		filepath.Join(dir, "java", "java-deployment.resource.yaml"))
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
+	assert.Contains(t, string(b), "kind: StatefulSet")
+}
+
+func TestCmd_Execute_APIs(t *testing.T) {
+	dir, err := ioutil.TempDir("", "kustomize-kyaml-test")
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
+	defer os.RemoveAll(dir)
+
+	_, filename, _, ok := runtime.Caller(0)
+	if !assert.True(t, ok) {
+		t.FailNow()
+	}
+	ds, err := filepath.Abs(filepath.Join(filepath.Dir(filename), "test", "testdata"))
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
+	if !assert.NoError(t, copyutil.CopyDir(ds, dir)) {
+		t.FailNow()
+	}
+	if !assert.NoError(t, os.Chdir(filepath.Dir(dir))) {
+		return
+	}
+
+	// write a test filter
+	f := `apiVersion: v1
+kind: ValueReplacer
+metadata:
+  configFn:
+    container:
+      image: gcr.io/example.com/image:version
+stringMatch: Deployment
+replace: StatefulSet
+`
+	tmpF, err := ioutil.TempFile("", "filter*.yaml")
+	if !assert.NoError(t, err) {
+		return
+	}
+	os.RemoveAll(tmpF.Name())
+	if !assert.NoError(t, ioutil.WriteFile(tmpF.Name(), []byte(f), 0600)) {
+		return
+	}
+
+	instance := RunFns{
+		FunctionPaths: []string{tmpF.Name()},
+		Path:          dir,
+		containerFilterProvider: func(s, _ string, node *yaml.RNode) kio.Filter {
+			// parse the filter from the input
+			filter := yaml.YFilter{}
+			b := &bytes.Buffer{}
+			e := yaml.NewEncoder(b)
+			if !assert.NoError(t, e.Encode(node.YNode())) {
+				t.FailNow()
+			}
+			e.Close()
+			d := yaml.NewDecoder(b)
+			if !assert.NoError(t, d.Decode(&filter)) {
+				t.FailNow()
+			}
+
+			return filters.Modifier{
+				Filters: []yaml.YFilter{{Filter: yaml.Lookup("kind")}, filter},
+			}
+		},
+	}
+	err = instance.Execute()
+	if !assert.NoError(t, err) {
+		return
+	}
+	b, err := ioutil.ReadFile(
+		filepath.Join(dir, "java", "java-deployment.resource.yaml"))
+	if !assert.NoError(t, err) {
+		return
+	}
+	assert.Contains(t, string(b), "kind: StatefulSet")
+}
+
+func TestCmd_Execute_Stdout(t *testing.T) {
+	dir, err := ioutil.TempDir("", "kustomize-kyaml-test")
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
+	defer os.RemoveAll(dir)
+
+	_, filename, _, ok := runtime.Caller(0)
+	if !assert.True(t, ok) {
+		t.FailNow()
+	}
+	ds, err := filepath.Abs(filepath.Join(filepath.Dir(filename), "test", "testdata"))
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
+	if !assert.NoError(t, copyutil.CopyDir(ds, dir)) {
+		t.FailNow()
+	}
+	if !assert.NoError(t, os.Chdir(filepath.Dir(dir))) {
+		return
+	}
+
+	// write a test filter
+	f := `apiVersion: v1
+kind: ValueReplacer
+metadata:
+  configFn:
+    container:
+      image: gcr.io/example.com/image:version
+stringMatch: Deployment
+replace: StatefulSet
+`
+	if !assert.NoError(t, ioutil.WriteFile(
+		filepath.Join(dir, "filter.yaml"), []byte(f), 0600)) {
+		return
+	}
+
+	out := &bytes.Buffer{}
+	instance := RunFns{
+		Output: out,
+		Path:   dir,
+		containerFilterProvider: func(s, _ string, node *yaml.RNode) kio.Filter {
+			// parse the filter from the input
+			filter := yaml.YFilter{}
+			b := &bytes.Buffer{}
+			e := yaml.NewEncoder(b)
+			if !assert.NoError(t, e.Encode(node.YNode())) {
+				t.FailNow()
+			}
+			e.Close()
+			d := yaml.NewDecoder(b)
+			if !assert.NoError(t, d.Decode(&filter)) {
+				t.FailNow()
+			}
+
+			return filters.Modifier{
+				Filters: []yaml.YFilter{{Filter: yaml.Lookup("kind")}, filter},
+			}
+		},
+	}
+	if !assert.NoError(t, instance.Execute()) {
+		return
+	}
+	b, err := ioutil.ReadFile(
+		filepath.Join(dir, "java", "java-deployment.resource.yaml"))
+	if !assert.NoError(t, err) {
+		return
+	}
+	assert.NotContains(t, string(b), "kind: StatefulSet")
+	assert.Contains(t, out.String(), "kind: StatefulSet")
+}

kyaml/runfn/test/testdata/java/java-configmap.resource.yaml

@@ -0,0 +1,10 @@
+# Copyright 2019 The Kubernetes Authors.
+# SPDX-License-Identifier: Apache-2.0
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: app-config
+  labels:
+    app.kubernetes.io/component: undefined
+    app.kubernetes.io/instance: undefined
+data: {}

kyaml/runfn/test/testdata/java/java-deployment.resource.yaml

@@ -0,0 +1,36 @@
+# Copyright 2019 The Kubernetes Authors.
+# SPDX-License-Identifier: Apache-2.0
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: app
+  labels:
+    app: java
+spec:
+  selector:
+    matchLabels:
+      app: java
+  template:
+    metadata:
+      labels:
+        app: java
+    spec:
+      restartPolicy: Always
+      containers:
+      - name: app
+        image: gcr.io/project/app:version
+        command:
+        - java
+        - -jar
+        - /app.jar
+        ports:
+        - containerPort: 8080
+        envFrom:
+        - configMapRef:
+            name: app-config
+        env:
+        - name: JAVA_OPTS
+          value: -XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap
+            -Djava.security.egd=file:/dev/./urandom
+        imagePullPolicy: Always
+  minReadySeconds: 5

kyaml/runfn/test/testdata/java/java-service.resource.yaml

@@ -0,0 +1,15 @@
+# Copyright 2019 The Kubernetes Authors.
+# SPDX-License-Identifier: Apache-2.0
+apiVersion: v1
+kind: Service
+metadata:
+  name: app
+  labels:
+    app: java
+spec:
+  selector:
+    app: java
+  ports:
+  - name: "8080"
+    port: 8080
+    targetPort: 8080