Enable functions to defer failures

so multiple validators can be run sequentially without shortcircuiting the pipeline

kyaml/kio/filters/container.go

@@ -151,6 +151,10 @@ type ContainerFilter struct {
 
 	Results *yaml.RNode
 
+	DeferFailure bool
+
+	Exit error
+
 	// SetFlowStyleForConfig sets the style for config to Flow when serializing it
 	SetFlowStyleForConfig bool
 
@@ -160,7 +164,18 @@ type ContainerFilter struct {
 	checkInput func(string)
 }
 
+func (c ContainerFilter) GetExit() error {
+	return c.Exit
+}
+
+type DeferFailureFunction interface {
+	GetExit() error
+}
+
 func (c ContainerFilter) String() string {
+	if c.DeferFailure {
+		return fmt.Sprintf("%s deferFailure: %v", c.Image, c.DeferFailure)
+	}
 	return c.Image
 }
 
@@ -300,18 +315,9 @@ func (c *ContainerFilter) Filter(nodes []*yaml.RNode) ([]*yaml.RNode, error) {
 	}
 	cmd.Stdin = in
 	cmd.Stdout = out
-	if err := cmd.Run(); err != nil {
-		// write the results file on failure
-		results, e := r.Read()
-		if e != nil {
-			return nil, e
-		}
-		if e = c.doResults(r); e != nil {
-			return nil, e
-		}
-		// return the results from the function even on failure
-		return results, err
-	}
+
+	// don't exit immediately if the function fails -- write out the validation
+	c.Exit = cmd.Run()
 
 	output, err := r.Read()
 	if err != nil {
@@ -322,6 +328,10 @@ func (c *ContainerFilter) Filter(nodes []*yaml.RNode) ([]*yaml.RNode, error) {
 		return nil, err
 	}
 
+	if c.Exit != nil && !c.DeferFailure {
+		return append(output, saved...), c.Exit
+	}
+
 	// annotate any generated Resources with a path and index if they don't already have one
 	if err := kioutil.DefaultPathAnnotation(functionDir, output); err != nil {
 		return nil, err
@@ -380,6 +390,7 @@ func (c *ContainerFilter) getArgs() []string {
 
 	// tell functions to write error messages to stderr as well as results
 	os.Setenv("LOG_TO_STDERR", "true")
+	os.Setenv("STRUCTURED_RESULTS", "true")
 
 	// export the local environment vars to the container
 	for _, pair := range os.Environ() {

kyaml/kio/filters/container_test.go

@@ -18,13 +18,14 @@ import (
 
 func TestContainerFilter_Filter(t *testing.T) {
 	var tests = []struct {
-		name              string
-		input             []string
-		expectedOutput    []string
-		expectedError     string
-		expectedResults   string
-		noMakeResultsFile bool
-		instance          ContainerFilter
+		name               string
+		input              []string
+		expectedOutput     []string
+		expectedError      string
+		expectedSavedError string
+		expectedResults    string
+		noMakeResultsFile  bool
+		instance           ContainerFilter
 	}{
 		{
 			name: "add_path_annotation",
@@ -220,6 +221,87 @@ metadata:
 `,
 		},
 
+		{
+			name:               "write_results_defer_failure",
+			expectedSavedError: "exit status 1",
+			instance: ContainerFilter{
+				DeferFailure: true,
+				args: []string{"sh", "-c",
+					`echo '
+apiVersion: config.kubernetes.io/v1alpha1
+kind: ResourceList
+items:
+- apiVersion: apps/v1
+  kind: Deployment
+  metadata:
+    name: deployment-foo
+- apiVersion: v1
+  kind: Service
+  metadata:
+    name: service-foo
+results:
+- apiVersion: config.k8s.io/v1alpha1
+  kind: ObjectError
+  name: "some-validator"
+  items:  
+  - type: error
+    message: "some message"
+    resourceRef:
+      apiVersion: apps/v1
+      kind: Deployment
+      name: foo
+      namespace: bar
+    file:
+      path: deploy.yaml
+      index: 0
+    field:
+      path: "spec.template.spec.containers[3].resources.limits.cpu"
+      currentValue: "200"
+      suggestedValue: "2"
+' && cat not-real-dir
+`,
+				},
+			},
+			expectedOutput: []string{
+				`
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: deployment-foo
+  annotations:
+    config.kubernetes.io/path: 'deployment_deployment-foo.yaml'
+`,
+				`
+apiVersion: v1
+kind: Service
+metadata:
+  name: service-foo
+  annotations:
+    config.kubernetes.io/path: 'service_service-foo.yaml'
+`,
+			},
+			expectedResults: `
+- apiVersion: config.k8s.io/v1alpha1
+  kind: ObjectError
+  name: "some-validator"
+  items:
+  - type: error
+    message: "some message"
+    resourceRef:
+      apiVersion: apps/v1
+      kind: Deployment
+      name: foo
+      namespace: bar
+    file:
+      path: deploy.yaml
+      index: 0
+    field:
+      path: "spec.template.spec.containers[3].resources.limits.cpu"
+      currentValue: "200"
+      suggestedValue: "2"
+`,
+		},
+
 		{
 			name:              "write_results_non_0_exit_missing_file",
 			expectedError:     "open /not/real/file: no such file or directory",
@@ -332,6 +414,13 @@ metadata:
 				return
 			}
 
+			if tt.expectedSavedError != "" {
+				if !assert.EqualError(t, tt.instance.Exit, tt.expectedSavedError) {
+					t.FailNow()
+				}
+				return
+			}
+
 			if !assert.NoError(t, err) {
 				t.FailNow()
 			}

kyaml/kio/filters/functiontypes.go

@@ -19,6 +19,8 @@ type FunctionSpec struct {
 	// Network is the name of the network to use from a container
 	Network string `json:"network,omitempty" yaml:"network,omitempty"`
 
+	DeferFailure bool `json:"deferFailure,omitempty" yaml:"deferFailure,omitempty"`
+
 	// Container is the spec for running a function as a container
 	Container ContainerSpec `json:"container,omitempty" yaml:"container,omitempty"`
 

kyaml/runfn/runfn.go

@@ -164,7 +164,27 @@ func (r RunFns) runFunctions(
 		// the output is nil (reading from Input)
 		outputs = append(outputs, kio.ByteWriter{Writer: r.Output})
 	}
-	return kio.Pipeline{Inputs: []kio.Reader{input}, Filters: fltrs, Outputs: outputs}.Execute()
+	err := kio.Pipeline{
+		Inputs: []kio.Reader{input}, Filters: fltrs, Outputs: outputs}.Execute()
+	if err != nil {
+		return err
+	}
+
+	// check for deferred function errors
+	var errs []string
+	for i := range fltrs {
+		cf, ok := fltrs[i].(filters.DeferFailureFunction)
+		if !ok {
+			continue
+		}
+		if cf.GetExit() != nil {
+			errs = append(errs, cf.GetExit().Error())
+		}
+	}
+	if len(errs) > 0 {
+		return fmt.Errorf(strings.Join(errs, "\n---\n"))
+	}
+	return nil
 }
 
 // getFunctionsFromInput scans the input for functions and runs them
@@ -328,6 +348,7 @@ func (r *RunFns) ffp(spec filters.FunctionSpec, api *yaml.RNode) (kio.Filter, er
 			StorageMounts: r.StorageMounts,
 			GlobalScope:   r.GlobalScope,
 			ResultsFile:   resultsFile,
+			DeferFailure:  spec.DeferFailure,
 		}, nil
 	}
 	if r.EnableStarlark && spec.Starlark.Path != "" {

kyaml/runfn/runfn_test.go

@@ -15,6 +15,7 @@ import (
 
 	"github.com/stretchr/testify/assert"
 	"sigs.k8s.io/kustomize/kyaml/copyutil"
+	"sigs.k8s.io/kustomize/kyaml/errors"
 	"sigs.k8s.io/kustomize/kyaml/kio"
 	"sigs.k8s.io/kustomize/kyaml/kio/filters"
 	"sigs.k8s.io/kustomize/kyaml/yaml"
@@ -234,6 +235,29 @@ metadata:
 			out: []string{"gcr.io/example.com/image:v1.0.0"},
 		},
 
+		// Test
+		//
+		//
+		{name: "defer_failure",
+			in: []f{
+				{
+					path: filepath.Join("foo", "bar.yaml"),
+					value: `
+apiVersion: example.com/v1alpha1
+kind: ExampleFunction
+metadata:
+  annotations:
+    config.kubernetes.io/function: |
+      deferFailure: true
+      container:
+        image: gcr.io/example.com/image:v1.0.0
+    config.kubernetes.io/local-config: "true"
+`,
+				},
+			},
+			out: []string{"gcr.io/example.com/image:v1.0.0 deferFailure: true"},
+		},
+
 		{name: "disable containers",
 			in: []f{
 				{
@@ -674,6 +698,93 @@ func TestCmd_Execute(t *testing.T) {
 	assert.Contains(t, string(b), "kind: StatefulSet")
 }
 
+type TestFilter struct {
+	invoked bool
+	Exit    error
+}
+
+func (f *TestFilter) Filter(input []*yaml.RNode) ([]*yaml.RNode, error) {
+	f.invoked = true
+	return input, nil
+}
+
+func (f *TestFilter) GetExit() error {
+	return f.Exit
+}
+
+func TestCmd_Execute_deferFailure(t *testing.T) {
+	dir := setupTest(t)
+	defer os.RemoveAll(dir)
+
+	// write a test filter to the directory of configuration
+	if !assert.NoError(t, ioutil.WriteFile(
+		filepath.Join(dir, "filter1.yaml"), []byte(`apiVersion: v1
+kind: ValueReplacer
+metadata:
+  annotations:
+    config.kubernetes.io/function: |
+      container:
+        image: 1
+    config.kubernetes.io/local-config: "true"
+stringMatch: Deployment
+replace: StatefulSet
+`), 0600)) {
+		t.FailNow()
+	}
+
+	// write a test filter to the directory of configuration
+	if !assert.NoError(t, ioutil.WriteFile(
+		filepath.Join(dir, "filter2.yaml"), []byte(`apiVersion: v1
+kind: ValueReplacer
+metadata:
+  annotations:
+    config.kubernetes.io/function: |
+      container:
+        image: 2
+    config.kubernetes.io/local-config: "true"
+stringMatch: Deployment
+replace: StatefulSet
+`), 0600)) {
+		t.FailNow()
+	}
+
+	var fltrs []*TestFilter
+	instance := RunFns{
+		Path: dir,
+		functionFilterProvider: func(f filters.FunctionSpec, node *yaml.RNode) (kio.Filter, error) {
+			tf := &TestFilter{
+				Exit: errors.Errorf("message: %s", f.Container.Image),
+			}
+			fltrs = append(fltrs, tf)
+			return tf, nil
+		},
+	}
+	instance.init()
+
+	err := instance.Execute()
+
+	// make sure all filters were run
+	if !assert.Equal(t, 2, len(fltrs)) {
+		t.FailNow()
+	}
+	for i := range fltrs {
+		if !assert.True(t, fltrs[i].invoked) {
+			t.FailNow()
+		}
+	}
+
+	if !assert.EqualError(t, err, "message: 1\n---\nmessage: 2") {
+		t.FailNow()
+	}
+	b, err := ioutil.ReadFile(
+		filepath.Join(dir, "java", "java-deployment.resource.yaml"))
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
+	// files weren't changed because there was an error
+	assert.Contains(t, string(b), "kind: Deployment")
+}
+
 // TestCmd_Execute_setOutput tests the execution of a filter reading and writing to a dir
 func TestCmd_Execute_setFunctionPaths(t *testing.T) {
 	dir := setupTest(t)