Merge pull request #2185 from pwittrock/master

cmd/config: support for stdin/stdout in source/sink

cmd/config/docs/commands/sink.md

@@ -6,10 +6,10 @@
 
 [Alpha] Implement a Sink by writing input to a local directory.
 
-    kustomize config sink DIR
+    kustomize config sink [DIR]
 
   DIR:
-    Path to local directory.
+    Path to local directory.  If unspecified, sink will write to stdout as if it were a single file.
 
 `sink` writes its input to a directory
 

cmd/config/docs/commands/source.md

@@ -6,10 +6,11 @@
 
 [Alpha] Implement a Source by reading a local directory.
 
-    kustomize config source DIR
+    kustomize config source DIR...
 
   DIR:
-    Path to local directory.
+    One or more paths to local directories.  Contents from directories will be concatenated.
+    If no directories are provided, source will read from stdin as if it were a single file.
 
 `source` emits configuration to act as input to a function
 

cmd/config/internal/commands/cmdcreatesetter.go

@@ -38,7 +38,6 @@ func NewCreateSetterRunner(parent string) *CreateSetterRunner {
 		"create a partial setter for only part of the field value.")
 	fixDocs(parent, set)
 	set.MarkFlagRequired("type")
-	set.MarkFlagRequired("field")
 	r.Command = set
 	return r
 }

cmd/config/internal/commands/fmt.go

@@ -31,6 +31,8 @@ formatting substitution verbs {'%n': 'metadata.name', '%s': 'metadata.namespace'
 		`if true, keep index and filename annotations set on Resources.`)
 	c.Flags().BoolVar(&r.Override, "override", false,
 		`if true, override existing filepath annotations.`)
+	c.Flags().BoolVar(&r.UseSchema, "use-schema", false,
+		`if true, uses openapi resource schema to format resources.`)
 	r.Command = c
 	return r
 }
@@ -46,6 +48,7 @@ type FmtRunner struct {
 	SetFilenames    bool
 	KeepAnnotations bool
 	Override        bool
+	UseSchema       bool
 }
 
 func (r *FmtRunner) preRunE(c *cobra.Command, args []string) error {
@@ -56,7 +59,9 @@ func (r *FmtRunner) preRunE(c *cobra.Command, args []string) error {
 }
 
 func (r *FmtRunner) runE(c *cobra.Command, args []string) error {
-	f := []kio.Filter{filters.FormatFilter{}}
+	f := []kio.Filter{filters.FormatFilter{
+		UseSchema: r.UseSchema,
+	}}
 
 	// format with file names
 	if r.SetFilenames {

cmd/config/internal/commands/merge3.go

@@ -25,6 +25,8 @@ func GetMerge3Runner(name string) *Merge3Runner {
 		"Path to updated package")
 	c.Flags().StringVar(&r.toDir, "to", "",
 		"Path to destination package")
+	c.Flags().BoolVar(&r.path, "path-merge-key", false,
+		"Use the path as part of the merge key when merging resources")
 
 	r.Command = c
 	return r
@@ -40,6 +42,7 @@ type Merge3Runner struct {
 	ancestor string
 	fromDir  string
 	toDir    string
+	path     bool
 }
 
 func (r *Merge3Runner) runE(c *cobra.Command, args []string) error {
@@ -47,6 +50,7 @@ func (r *Merge3Runner) runE(c *cobra.Command, args []string) error {
 		OriginalPath: r.ancestor,
 		UpdatedPath:  r.fromDir,
 		DestPath:     r.toDir,
+		MergeOnPath:  r.path,
 	}.Merge()
 	if err != nil {
 		return err

cmd/config/internal/commands/sink.go

@@ -7,6 +7,7 @@ import (
 	"github.com/spf13/cobra"
 	"sigs.k8s.io/kustomize/cmd/config/internal/generateddocs/commands"
 	"sigs.k8s.io/kustomize/kyaml/kio"
+	"sigs.k8s.io/kustomize/kyaml/kio/kioutil"
 )
 
 // GetSinkRunner returns a command for Sink.
@@ -18,7 +19,7 @@ func GetSinkRunner(name string) *SinkRunner {
 		Long:    commands.SinkLong,
 		Example: commands.SinkExamples,
 		RunE:    r.runE,
-		Args:    cobra.ExactArgs(1),
+		Args:    cobra.MaximumNArgs(1),
 	}
 	fixDocs(name, c)
 	r.Command = c
@@ -35,14 +36,18 @@ type SinkRunner struct {
 }
 
 func (r *SinkRunner) runE(c *cobra.Command, args []string) error {
+	var outputs []kio.Writer
+	if len(args) == 1 {
+		outputs = []kio.Writer{&kio.LocalPackageWriter{PackagePath: args[0]}}
+	} else {
+		outputs = []kio.Writer{&kio.ByteWriter{
+			Writer:           c.OutOrStdout(),
+			ClearAnnotations: []string{kioutil.PathAnnotation}},
+		}
+	}
+
 	err := kio.Pipeline{
-		Inputs: []kio.Reader{&kio.ByteReader{Reader: c.InOrStdin()}},
-		Outputs: []kio.Writer{
-			&kio.LocalPackageWriter{
-				PackagePath:      args[0],
-				ClearAnnotations: []string{"config.kubernetes.io/path"},
-			},
-		},
-	}.Execute()
+		Inputs:  []kio.Reader{&kio.ByteReader{Reader: c.InOrStdin()}},
+		Outputs: outputs}.Execute()
 	return handleError(c, err)
 }

cmd/config/internal/commands/sink_test.go

@@ -21,8 +21,6 @@ func TestSinkCommand(t *testing.T) {
 	}
 	defer os.RemoveAll(d)
 
-	// fmt the files
-	b := &bytes.Buffer{}
 	r := commands.GetSinkRunner("")
 	r.Command.SetIn(bytes.NewBufferString(`apiVersion: config.kubernetes.io/v1alpha1
 kind: ResourceList
@@ -75,7 +73,6 @@ items:
     replicas: 3
 `))
 	r.Command.SetArgs([]string{d})
-	r.Command.SetOut(b)
 	if !assert.NoError(t, r.Command.Execute()) {
 		t.FailNow()
 	}
@@ -138,3 +135,117 @@ spec:
 		t.FailNow()
 	}
 }
+
+func TestSinkCommand_Stdout(t *testing.T) {
+	d, err := ioutil.TempDir("", "kustomize-source-test")
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
+	defer os.RemoveAll(d)
+
+	// fmt the files
+	out := &bytes.Buffer{}
+	r := commands.GetSinkRunner("")
+	r.Command.SetIn(bytes.NewBufferString(`apiVersion: config.kubernetes.io/v1alpha1
+kind: ResourceList
+items:
+- kind: Deployment
+  metadata:
+    labels:
+      app: nginx2
+    name: foo
+    annotations:
+      app: nginx2
+      config.kubernetes.io/index: '0'
+      config.kubernetes.io/path: 'f1.yaml'
+  spec:
+    replicas: 1
+- kind: Service
+  metadata:
+    name: foo
+    annotations:
+      app: nginx
+      config.kubernetes.io/index: '1'
+      config.kubernetes.io/path: 'f1.yaml'
+  spec:
+    selector:
+      app: nginx
+- apiVersion: v1
+  kind: Abstraction
+  metadata:
+    name: foo
+    annotations:
+      config.kubernetes.io/function: |
+        container:
+          image: gcr.io/example/reconciler:v1
+      config.kubernetes.io/local-config: "true"
+      config.kubernetes.io/index: '0'
+      config.kubernetes.io/path: 'f2.yaml'
+  spec:
+    replicas: 3
+- apiVersion: apps/v1
+  kind: Deployment
+  metadata:
+    labels:
+      app: nginx
+    name: bar
+    annotations:
+      app: nginx
+      config.kubernetes.io/index: '1'
+      config.kubernetes.io/path: 'f2.yaml'
+  spec:
+    replicas: 3
+`))
+
+	r.Command.SetOut(out)
+	r.Command.SetArgs([]string{})
+	if !assert.NoError(t, r.Command.Execute()) {
+		t.FailNow()
+	}
+
+	expected := `kind: Deployment
+metadata:
+  labels:
+    app: nginx2
+  name: foo
+  annotations:
+    app: nginx2
+spec:
+  replicas: 1
+---
+kind: Service
+metadata:
+  name: foo
+  annotations:
+    app: nginx
+spec:
+  selector:
+    app: nginx
+---
+apiVersion: v1
+kind: Abstraction
+metadata:
+  name: foo
+  annotations:
+    config.kubernetes.io/function: |
+      container:
+        image: gcr.io/example/reconciler:v1
+    config.kubernetes.io/local-config: "true"
+spec:
+  replicas: 3
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: nginx
+  name: bar
+  annotations:
+    app: nginx
+spec:
+  replicas: 3
+`
+	if !assert.Equal(t, expected, out.String()) {
+		t.FailNow()
+	}
+}

cmd/config/internal/commands/source.go

@@ -21,7 +21,6 @@ func GetSourceRunner(name string) *SourceRunner {
 		Long:    commands.SourceLong,
 		Example: commands.SourceExamples,
 		RunE:    r.runE,
-		Args:    cobra.ExactArgs(1),
 	}
 	fixDocs(name, c)
 	c.Flags().StringVar(&r.WrapKind, "wrap-kind", kio.ResourceListKind,
@@ -70,8 +69,14 @@ func (r *SourceRunner) runE(c *cobra.Command, args []string) error {
 		FunctionConfig:        functionConfig,
 	})
 
-	err := kio.Pipeline{
-		Inputs:  []kio.Reader{kio.LocalPackageReader{PackagePath: args[0]}},
-		Outputs: outputs}.Execute()
+	var inputs []kio.Reader
+	for _, a := range args {
+		inputs = append(inputs, kio.LocalPackageReader{PackagePath: a})
+	}
+	if len(inputs) == 0 {
+		inputs = []kio.Reader{&kio.ByteReader{Reader: c.InOrStdin()}}
+	}
+
+	err := kio.Pipeline{Inputs: inputs, Outputs: outputs}.Execute()
 	return handleError(c, err)
 }

cmd/config/internal/commands/source_test.go

@@ -134,3 +134,67 @@ items:
 		return
 	}
 }
+
+func TestSourceCommand_Stdin(t *testing.T) {
+	d, err := ioutil.TempDir("", "kustomize-source-test")
+	if !assert.NoError(t, err) {
+		return
+	}
+	defer os.RemoveAll(d)
+
+	in := bytes.NewBufferString(`
+kind: Deployment
+metadata:
+  labels:
+    app: nginx2
+  name: foo
+  annotations:
+    app: nginx2
+spec:
+  replicas: 1
+---
+kind: Service
+metadata:
+  name: foo
+  annotations:
+    app: nginx
+spec:
+  selector:
+    app: nginx
+`)
+
+	out := &bytes.Buffer{}
+	r := commands.GetSourceRunner("")
+	r.Command.SetArgs([]string{})
+	r.Command.SetIn(in)
+	r.Command.SetOut(out)
+	if !assert.NoError(t, r.Command.Execute()) {
+		return
+	}
+
+	if !assert.Equal(t, `apiVersion: config.kubernetes.io/v1alpha1
+kind: ResourceList
+items:
+- kind: Deployment
+  metadata:
+    labels:
+      app: nginx2
+    name: foo
+    annotations:
+      app: nginx2
+      config.kubernetes.io/index: '0'
+  spec:
+    replicas: 1
+- kind: Service
+  metadata:
+    name: foo
+    annotations:
+      app: nginx
+      config.kubernetes.io/index: '1'
+  spec:
+    selector:
+      app: nginx
+`, out.String()) {
+		return
+	}
+}

cmd/config/internal/generateddocs/commands/docs.go

@@ -350,10 +350,10 @@ var SinkShort = `[Alpha] Implement a Sink by writing input to a local directory.
 var SinkLong = `
 [Alpha] Implement a Sink by writing input to a local directory.
 
-    kustomize config sink DIR
+    kustomize config sink [DIR]
 
   DIR:
-    Path to local directory.
+    Path to local directory.  If unspecified, sink will write to stdout as if it were a single file.
 
 ` + "`" + `sink` + "`" + ` writes its input to a directory
 `
@@ -364,10 +364,11 @@ var SourceShort = `[Alpha] Implement a Source by reading a local directory.`
 var SourceLong = `
 [Alpha] Implement a Source by reading a local directory.
 
-    kustomize config source DIR
+    kustomize config source DIR...
 
   DIR:
-    Path to local directory.
+    One or more paths to local directories.  Contents from directories will be concatenated.
+    If no directories are provided, source will read from stdin as if it were a single file.
 
 ` + "`" + `source` + "`" + ` emits configuration to act as input to a function
 `

functions/examples/injection-tshirt-sizes/image/main.go

@@ -48,9 +48,9 @@ var cpuSizes = map[string]string{
 
 // memorySizes is the mapping from tshirt-size to memory reservation quantity
 var memorySizes = map[string]string{
-	"small":  "50MiB",
-	"medium": "1GiB",
-	"large":  "32GiB",
+	"small":  "50M",
+	"medium": "1G",
+	"large":  "32G",
 }
 
 // inject sets the cpu and memory reservations on all containers for Resources annotated

kyaml/kio/byteio_writer.go

@@ -113,7 +113,6 @@ func (w ByteWriter) Write(nodes []*yaml.RNode) error {
 	}
 	doc := &yaml.Node{
 		Kind:    yaml.DocumentNode,
-		Style:   yaml.FoldedStyle,
 		Content: []*yaml.Node{list}}
 	for i := range nodes {
 		items.Content = append(items.Content, nodes[i].YNode())

kyaml/kio/filters/fmtr.go

@@ -56,7 +56,8 @@ func FormatFileOrDirectory(path string) error {
 }
 
 type FormatFilter struct {
-	Process func(n *yaml.Node) error
+	Process   func(n *yaml.Node) error
+	UseSchema bool
 }
 
 var _ kio.Filter = FormatFilter{}
@@ -78,7 +79,12 @@ func (f FormatFilter) Filter(slice []*yaml.RNode) ([]*yaml.RNode, error) {
 			continue
 		}
 		kind, apiVersion := kindNode.YNode().Value, apiVersionNode.YNode().Value
-		s := openapi.SchemaForResourceType(yaml.TypeMeta{APIVersion: apiVersion, Kind: kind})
+		var s *openapi.ResourceSchema
+		if f.UseSchema {
+			s = openapi.SchemaForResourceType(yaml.TypeMeta{APIVersion: apiVersion, Kind: kind})
+		} else {
+			s = nil
+		}
 		err = (&formatter{apiVersion: apiVersion, kind: kind, process: f.Process}).
 			fmtNode(slice[i].YNode(), "", s)
 		if err != nil {
@@ -151,7 +157,6 @@ func (f *formatter) fmtNode(n *yaml.Node, path string, schema *openapi.ResourceS
 				s = schema.Elements()
 			}
 		}
-
 		// format the node using the schema
 		err := f.fmtNode(n.Content[i], p, s)
 		if err != nil {

kyaml/kio/filters/fmtr_test.go

@@ -74,6 +74,73 @@ spec:
           containerPort: 80
 `
 
+	buff := &bytes.Buffer{}
+	err := kio.Pipeline{
+		Inputs: []kio.Reader{&kio.ByteReader{Reader: strings.NewReader(y)}},
+		Filters: []kio.Filter{FormatFilter{
+			UseSchema: true,
+		}},
+		Outputs: []kio.Writer{kio.ByteWriter{Writer: buff}},
+	}.Execute()
+	assert.NoError(t, err)
+	assert.Equal(t, expected, buff.String())
+}
+
+func TestFormat_UnsortedInput_No_Schema(t *testing.T) {
+	y := `
+apiVersion: apps/v1
+spec:
+  template:
+    spec:
+      containers:
+      - name: nginx
+        image: nginx:1.0.0
+        args:
+        - on
+        - 1
+        - hello
+        ports:
+        - name: http
+          targetPort: 80
+          containerPort: 80
+kind: Deployment
+metadata:
+  name: foo
+  labels:
+    foo: on
+    foo2: hello1
+  annotations:
+    bar: 1
+    bar2: hello2
+`
+
+	// keep the style on values that parse as non-string types
+	expected := `apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: foo
+  labels:
+    foo: on
+    foo2: hello1
+  annotations:
+    bar: 1
+    bar2: hello2
+spec:
+  template:
+    spec:
+      containers:
+      - name: nginx
+        image: nginx:1.0.0
+        args:
+        - on
+        - 1
+        - hello
+        ports:
+        - name: http
+          targetPort: 80
+          containerPort: 80
+`
+
 	buff := &bytes.Buffer{}
 	err := kio.Pipeline{
 		Inputs:  []kio.Reader{&kio.ByteReader{Reader: strings.NewReader(y)}},
@@ -120,16 +187,18 @@ spec:
 	buff := &bytes.Buffer{}
 	err := kio.Pipeline{
 		Inputs: []kio.Reader{&kio.ByteReader{Reader: strings.NewReader(y)}},
-		Filters: []kio.Filter{FormatFilter{Process: func(n *yaml.Node) error {
-			if yaml.IsYaml1_1NonString(n) {
-				// don't change these styles, they are important for backwards compatibility
-				// e.g. "on" must remain quoted, on must remain unquoted
+		Filters: []kio.Filter{FormatFilter{
+			UseSchema: true,
+			Process: func(n *yaml.Node) error {
+				if yaml.IsYaml1_1NonString(n) {
+					// don't change these styles, they are important for backwards compatibility
+					// e.g. "on" must remain quoted, on must remain unquoted
+					return nil
+				}
+				// style does not have semantic meaning
+				n.Style = 0
 				return nil
-			}
-			// style does not have semantic meaning
-			n.Style = 0
-			return nil
-		}}},
+			}}},
 		Outputs: []kio.Writer{kio.ByteWriter{Writer: buff}},
 	}.Execute()
 	assert.NoError(t, err)
@@ -186,16 +255,18 @@ spec:
 	buff = &bytes.Buffer{}
 	err = kio.Pipeline{
 		Inputs: []kio.Reader{&kio.ByteReader{Reader: strings.NewReader(y)}},
-		Filters: []kio.Filter{FormatFilter{Process: func(n *yaml.Node) error {
-			if yaml.IsYaml1_1NonString(n) {
-				// don't change these styles, they are important for backwards compatibility
-				// e.g. "on" must remain quoted, on must remain unquoted
+		Filters: []kio.Filter{FormatFilter{
+			UseSchema: true,
+			Process: func(n *yaml.Node) error {
+				if yaml.IsYaml1_1NonString(n) {
+					// don't change these styles, they are important for backwards compatibility
+					// e.g. "on" must remain quoted, on must remain unquoted
+					return nil
+				}
+				// style does not have semantic meaning
+				n.Style = yaml.SingleQuotedStyle
 				return nil
-			}
-			// style does not have semantic meaning
-			n.Style = yaml.SingleQuotedStyle
-			return nil
-		}}},
+			}}},
 		Outputs: []kio.Writer{kio.ByteWriter{Writer: buff}},
 	}.Execute()
 	assert.NoError(t, err)

kyaml/kio/filters/merge3.go

@@ -7,6 +7,7 @@ import (
 	"fmt"
 
 	"sigs.k8s.io/kustomize/kyaml/kio"
+	"sigs.k8s.io/kustomize/kyaml/kio/kioutil"
 	"sigs.k8s.io/kustomize/kyaml/yaml"
 	"sigs.k8s.io/kustomize/kyaml/yaml/merge3"
 )
@@ -24,6 +25,11 @@ type Merge3 struct {
 	UpdatedPath    string
 	DestPath       string
 	MatchFilesGlob []string
+
+	// MergeOnPath will use the relative filepath as part of the merge key.
+	// This may be necessary if the directory contains multiple copies of
+	// the same resource, or resources patches.
+	MergeOnPath bool
 }
 
 func (m Merge3) Merge() error {
@@ -61,7 +67,7 @@ func (m Merge3) Merge() error {
 // Filter combines Resources with the same GVK + N + NS into tuples, and then merges them
 func (m Merge3) Filter(nodes []*yaml.RNode) ([]*yaml.RNode, error) {
 	// index the nodes by their identity
-	tl := tuples{}
+	tl := tuples{mergeOnPath: m.MergeOnPath}
 	for i := range nodes {
 		if err := tl.add(nodes[i]); err != nil {
 			return nil, err
@@ -102,6 +108,37 @@ func (m Merge3) Filter(nodes []*yaml.RNode) ([]*yaml.RNode, error) {
 // tuples combines nodes with the same GVK + N + NS
 type tuples struct {
 	list []*tuple
+
+	// mergeOnPath if set to true will use the resource filepath
+	// as part of the merge key
+	mergeOnPath bool
+}
+
+// isSameResource returns true if meta1 and meta2 are for the same logic resource
+func (ts *tuples) isSameResource(meta1, meta2 yaml.ResourceMeta) bool {
+	if meta1.Name != meta2.Name {
+		return false
+	}
+	if meta1.Namespace != meta2.Namespace {
+		return false
+	}
+	if meta1.APIVersion != meta2.APIVersion {
+		return false
+	}
+	if meta1.Kind != meta2.Kind {
+		return false
+	}
+	if ts.mergeOnPath {
+		// directories may contain multiple copies of a resource with the same
+		// name, namespace, apiVersion and kind -- e.g. kustomize patches, or
+		// multiple environments
+		// mergeOnPath configures the merge logic to use the path as part of the
+		// resource key
+		if meta1.Annotations[kioutil.PathAnnotation] != meta2.Annotations[kioutil.PathAnnotation] {
+			return false
+		}
+	}
+	return true
 }
 
 // add adds a node to the list, combining it with an existing matching Resource if found
@@ -112,8 +149,7 @@ func (ts *tuples) add(node *yaml.RNode) error {
 	}
 	for i := range ts.list {
 		t := ts.list[i]
-		if t.meta.Name == nodeMeta.Name && t.meta.Namespace == nodeMeta.Namespace &&
-			t.meta.APIVersion == nodeMeta.APIVersion && t.meta.Kind == nodeMeta.Kind {
+		if ts.isSameResource(t.meta, nodeMeta) {
 			return t.add(node)
 		}
 	}

kyaml/kio/filters/merge3_test.go

@@ -54,3 +54,78 @@ func TestMerge3_Merge(t *testing.T) {
 		t.FailNow()
 	}
 }
+
+// TestMerge3_Merge_path tests that if the same resource is specified multiple times
+// with MergeOnPath, that the resources will be merged by the filepath name.
+func TestMerge3_Merge_path(t *testing.T) {
+	_, datadir, _, ok := runtime.Caller(0)
+	if !assert.True(t, ok) {
+		t.FailNow()
+	}
+	datadir = filepath.Join(filepath.Dir(datadir), "testdata2")
+
+	// setup the local directory
+	dir, err := ioutil.TempDir("", "kyaml-test")
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
+	defer os.RemoveAll(dir)
+
+	if !assert.NoError(t, copyutil.CopyDir(
+		filepath.Join(datadir, "dataset1-localupdates"),
+		filepath.Join(dir, "dataset1"))) {
+		t.FailNow()
+	}
+
+	err = filters.Merge3{
+		OriginalPath: filepath.Join(datadir, "dataset1"),
+		UpdatedPath:  filepath.Join(datadir, "dataset1-remoteupdates"),
+		DestPath:     filepath.Join(dir, "dataset1"),
+		MergeOnPath:  true,
+	}.Merge()
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
+
+	diffs, err := copyutil.Diff(
+		filepath.Join(dir, "dataset1"),
+		filepath.Join(datadir, "dataset1-expected"))
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
+	if !assert.Empty(t, diffs.List()) {
+		t.FailNow()
+	}
+}
+
+// TestMerge3_Merge_fail tests that if the same resource is defined multiple times
+// that merge will fail
+func TestMerge3_Merge_fail(t *testing.T) {
+	_, datadir, _, ok := runtime.Caller(0)
+	if !assert.True(t, ok) {
+		t.FailNow()
+	}
+	datadir = filepath.Join(filepath.Dir(datadir), "testdata2")
+
+	// setup the local directory
+	dir, err := ioutil.TempDir("", "kyaml-test")
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
+	defer os.RemoveAll(dir)
+
+	if !assert.NoError(t, copyutil.CopyDir(
+		filepath.Join(datadir, "dataset1-localupdates"),
+		filepath.Join(dir, "dataset1"))) {
+		t.FailNow()
+	}
+
+	err = filters.Merge3{
+		OriginalPath: filepath.Join(datadir, "dataset1"),
+		UpdatedPath:  filepath.Join(datadir, "dataset1-remoteupdates"),
+		DestPath:     filepath.Join(dir, "dataset1"),
+	}.Merge()
+	if !assert.Error(t, err) {
+		t.FailNow()
+	}
+}

kyaml/kio/filters/testdata2/dataset1-expected/java/java-deployment-1.resource.yaml

@@ -0,0 +1,38 @@
+# Copyright 2019 The Kubernetes Authors.
+# SPDX-License-Identifier: Apache-2.0
+# 
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: app
+  labels:
+    app: java
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: java2
+  template:
+    metadata:
+      labels:
+        app: java2
+    spec:
+      restartPolicy: Always
+      containers:
+      - name: app
+        image: gcr.io/project/app:version
+        command:
+        - java
+        - -jar
+        - /app.jar
+        ports:
+        - containerPort: 8080
+        envFrom:
+        - configMapRef:
+            name: app-config
+        env:
+        - name: JAVA_OPTS
+          value: -XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap
+            -Djava.security.egd=file:/dev/./urandom
+        imagePullPolicy: Always
+  minReadySeconds: 15

kyaml/kio/filters/testdata2/dataset1-expected/java/java-deployment-2.resource.yaml

@@ -0,0 +1,38 @@
+# Copyright 2019 The Kubernetes Authors.
+# SPDX-License-Identifier: Apache-2.0
+# 
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: app
+  labels:
+    app: java3
+spec:
+  replicas: 4
+  selector:
+    matchLabels:
+      app: java
+  template:
+    metadata:
+      labels:
+        app: java
+    spec:
+      restartPolicy: Always
+      containers:
+      - name: app
+        image: gcr.io/project/app:version
+        command:
+        - java
+        - -jar
+        - /app.jar
+        ports:
+        - containerPort: 8080
+        envFrom:
+        - configMapRef:
+            name: app-config
+        env:
+        - name: JAVA_OPTS
+          value: -XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap
+            -Djava.security.egd=file:/dev/./urandom
+        imagePullPolicy: Always
+  minReadySeconds: 25

kyaml/kio/filters/testdata2/dataset1-localupdates/java/java-deployment-1.resource.yaml

@@ -0,0 +1,38 @@
+# Copyright 2019 The Kubernetes Authors.
+# SPDX-License-Identifier: Apache-2.0
+# 
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: app
+  labels:
+    app: java
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: java
+  template:
+    metadata:
+      labels:
+        app: java
+    spec:
+      restartPolicy: Always
+      containers:
+      - name: app
+        image: gcr.io/project/app:version
+        command:
+        - java
+        - -jar
+        - /app.jar
+        ports:
+        - containerPort: 8080
+        envFrom:
+        - configMapRef:
+            name: app-config
+        env:
+        - name: JAVA_OPTS
+          value: -XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap
+            -Djava.security.egd=file:/dev/./urandom
+        imagePullPolicy: Always
+  minReadySeconds: 15

kyaml/kio/filters/testdata2/dataset1-localupdates/java/java-deployment-2.resource.yaml

@@ -0,0 +1,38 @@
+# Copyright 2019 The Kubernetes Authors.
+# SPDX-License-Identifier: Apache-2.0
+# 
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: app
+  labels:
+    app: java
+spec:
+  replicas: 4
+  selector:
+    matchLabels:
+      app: java
+  template:
+    metadata:
+      labels:
+        app: java
+    spec:
+      restartPolicy: Always
+      containers:
+      - name: app
+        image: gcr.io/project/app:version
+        command:
+        - java
+        - -jar
+        - /app.jar
+        ports:
+        - containerPort: 8080
+        envFrom:
+        - configMapRef:
+            name: app-config
+        env:
+        - name: JAVA_OPTS
+          value: -XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap
+            -Djava.security.egd=file:/dev/./urandom
+        imagePullPolicy: Always
+  minReadySeconds: 25

kyaml/kio/filters/testdata2/dataset1-remoteupdates/java/java-deployment-1.resource.yaml

@@ -0,0 +1,38 @@
+# Copyright 2019 The Kubernetes Authors.
+# SPDX-License-Identifier: Apache-2.0
+# 
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: app
+  labels:
+    app: java
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: java2
+  template:
+    metadata:
+      labels:
+        app: java2
+    spec:
+      restartPolicy: Always
+      containers:
+      - name: app
+        image: gcr.io/project/app:version
+        command:
+        - java
+        - -jar
+        - /app.jar
+        ports:
+        - containerPort: 8080
+        envFrom:
+        - configMapRef:
+            name: app-config
+        env:
+        - name: JAVA_OPTS
+          value: -XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap
+            -Djava.security.egd=file:/dev/./urandom
+        imagePullPolicy: Always
+  minReadySeconds: 5

kyaml/kio/filters/testdata2/dataset1-remoteupdates/java/java-deployment-2.resource.yaml

@@ -0,0 +1,38 @@
+# Copyright 2019 The Kubernetes Authors.
+# SPDX-License-Identifier: Apache-2.0
+# 
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: app
+  labels:
+    app: java3
+spec:
+  replicas: 4
+  selector:
+    matchLabels:
+      app: java
+  template:
+    metadata:
+      labels:
+        app: java
+    spec:
+      restartPolicy: Always
+      containers:
+      - name: app
+        image: gcr.io/project/app:version
+        command:
+        - java
+        - -jar
+        - /app.jar
+        ports:
+        - containerPort: 8080
+        envFrom:
+        - configMapRef:
+            name: app-config
+        env:
+        - name: JAVA_OPTS
+          value: -XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap
+            -Djava.security.egd=file:/dev/./urandom
+        imagePullPolicy: Always
+  minReadySeconds: 10

kyaml/kio/filters/testdata2/dataset1/java/java-deployment-1.resource.yaml

@@ -0,0 +1,38 @@
+# Copyright 2019 The Kubernetes Authors.
+# SPDX-License-Identifier: Apache-2.0
+# 
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: app
+  labels:
+    app: java
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: java
+  template:
+    metadata:
+      labels:
+        app: java
+    spec:
+      restartPolicy: Always
+      containers:
+      - name: app
+        image: gcr.io/project/app:version
+        command:
+        - java
+        - -jar
+        - /app.jar
+        ports:
+        - containerPort: 8080
+        envFrom:
+        - configMapRef:
+            name: app-config
+        env:
+        - name: JAVA_OPTS
+          value: -XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap
+            -Djava.security.egd=file:/dev/./urandom
+        imagePullPolicy: Always
+  minReadySeconds: 5

kyaml/kio/filters/testdata2/dataset1/java/java-deployment-2.resource.yaml

@@ -0,0 +1,38 @@
+# Copyright 2019 The Kubernetes Authors.
+# SPDX-License-Identifier: Apache-2.0
+# 
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: app
+  labels:
+    app: java
+spec:
+  replicas: 4
+  selector:
+    matchLabels:
+      app: java
+  template:
+    metadata:
+      labels:
+        app: java
+    spec:
+      restartPolicy: Always
+      containers:
+      - name: app
+        image: gcr.io/project/app:version
+        command:
+        - java
+        - -jar
+        - /app.jar
+        ports:
+        - containerPort: 8080
+        envFrom:
+        - configMapRef:
+            name: app-config
+        env:
+        - name: JAVA_OPTS
+          value: -XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap
+            -Djava.security.egd=file:/dev/./urandom
+        imagePullPolicy: Always
+  minReadySeconds: 10

kyaml/openapi/openapi.go

@@ -4,15 +4,26 @@
 package openapi
 
 import (
+	"encoding/json"
+	"fmt"
 	"sync"
 
 	"github.com/go-openapi/spec"
+	"sigs.k8s.io/kustomize/kyaml/errors"
 	"sigs.k8s.io/kustomize/kyaml/yaml"
 )
 
-var setup sync.Once
-var schema spec.Schema
-var schemaByResourceType map[yaml.TypeMeta]*spec.Schema
+// globalSchema contains global state information about the openapi
+var globalSchema openapiData
+
+// openapiData contains the parsed openapi state.  this is in a struct rather than
+// a list of vars so that it can be reset from tests.
+type openapiData struct {
+	setup                sync.Once
+	schema               spec.Schema
+	schemaByResourceType map[yaml.TypeMeta]*spec.Schema
+	noUseBuiltInSchema   bool
+}
 
 // ResourceSchema wraps the OpenAPI Schema.
 type ResourceSchema struct {
@@ -26,13 +37,95 @@ type ResourceSchema struct {
 // as metadata, replicas and spec.template.spec
 func SchemaForResourceType(t yaml.TypeMeta) *ResourceSchema {
 	initSchema()
-	rs, found := schemaByResourceType[t]
+	rs, found := globalSchema.schemaByResourceType[t]
 	if !found {
 		return nil
 	}
 	return &ResourceSchema{Schema: rs}
 }
 
+// AddSchema parses s, and adds definitions from s to the global schema.
+func AddSchema(s []byte) (*spec.Schema, error) {
+	return parse(s)
+}
+
+// AddDefinitions adds the definitions to the global schema.
+func AddDefinitions(definitions spec.Definitions) {
+	// initialize values if they have not yet been set
+	if globalSchema.schemaByResourceType == nil {
+		globalSchema.schemaByResourceType = map[yaml.TypeMeta]*spec.Schema{}
+	}
+	if globalSchema.schema.Definitions == nil {
+		globalSchema.schema.Definitions = spec.Definitions{}
+	}
+
+	// index the schema definitions so we can lookup them up for Resources
+	for k := range definitions {
+		// index by GVK, if no GVK is found then it is the schema for a subfield
+		// of a Resource
+		d := definitions[k]
+
+		// copy definitions to the schema
+		globalSchema.schema.Definitions[k] = d
+		gvk, found := d.VendorExtensible.Extensions[kubernetesGVKExtensionKey]
+		if !found {
+			continue
+		}
+		// cast the extension to a []map[string]string
+		exts, ok := gvk.([]interface{})
+		if !ok || len(exts) != 1 {
+			continue
+		}
+		m, ok := exts[0].(map[string]interface{})
+		if !ok {
+			continue
+		}
+
+		// build the index key and save it
+		g := m[groupKey].(string)
+		apiVersion := m[versionKey].(string)
+		if g != "" {
+			apiVersion = g + "/" + apiVersion
+		}
+		globalSchema.schemaByResourceType[yaml.TypeMeta{Kind: m[kindKey].(string), APIVersion: apiVersion}] = &d
+	}
+}
+
+// Resolve resolves the reference against the global schema
+func Resolve(ref *spec.Ref) (*spec.Schema, error) {
+	return resolve(Schema(), ref)
+}
+
+// Schema returns the global schema
+func Schema() *spec.Schema {
+	return rootSchema()
+}
+
+// GetSchema parses s into a ResourceSchema, resolving References within the
+// global schema.
+func GetSchema(s string) (*ResourceSchema, error) {
+	var sc spec.Schema
+	if err := sc.UnmarshalJSON([]byte(s)); err != nil {
+		return nil, errors.Wrap(err)
+	}
+	if sc.Ref.String() != "" {
+		r, err := Resolve(&sc.Ref)
+		if err != nil {
+			return nil, errors.Wrap(err)
+		}
+		sc = *r
+	}
+
+	return &ResourceSchema{Schema: &sc}, nil
+}
+
+// SuppressBuiltInSchemaUse can be called to prevent using the built-in Kubernetes
+// schema as part of the global schema.
+// Must be called before the schema is used.
+func SuppressBuiltInSchemaUse() {
+	globalSchema.noUseBuiltInSchema = false
+}
+
 // Elements returns the Schema for the elements of an array.
 func (r *ResourceSchema) Elements() *ResourceSchema {
 	// load the schema from swagger.json
@@ -44,7 +137,7 @@ func (r *ResourceSchema) Elements() *ResourceSchema {
 	}
 	s := *r.Schema.Items.Schema
 	for s.Ref.String() != "" {
-		sc, e := spec.ResolveRef(rootSchema(), &s.Ref)
+		sc, e := Resolve(&s.Ref)
 		if e != nil {
 			return nil
 		}
@@ -96,7 +189,7 @@ func (r *ResourceSchema) Field(field string) *ResourceSchema {
 
 	// resolve the reference to the Schema if the Schema has one
 	for s.Ref.String() != "" {
-		sc, e := spec.ResolveRef(rootSchema(), &s.Ref)
+		sc, e := Resolve(&s.Ref)
 		if e != nil {
 			return nil
 		}
@@ -148,53 +241,57 @@ const (
 
 // initSchema parses the json schema
 func initSchema() {
-	setup.Do(func() {
-		// initialize the map
-		schemaByResourceType = map[yaml.TypeMeta]*spec.Schema{}
+	globalSchema.setup.Do(func() {
+		if globalSchema.noUseBuiltInSchema {
+			// don't parse the built in schema
+			return
+		}
 
 		// parse the swagger, this should never fail
-		parse(MustAsset(openAPIAssetName))
-
-		// TODO(pwittrock): add support for parsing additional schemas from
-		// environment variables, files or other sources
+		if _, err := parse(MustAsset(openAPIAssetName)); err != nil {
+			// this should never happen
+			panic(err)
+		}
 	})
 }
 
 // parse parses and indexes a single json schema
-func parse(b []byte) {
-	if err := schema.UnmarshalJSON(b); err != nil {
-		panic(err)
-	}
-	// index the schema definitions so we can lookup them up for Resources
-	for k := range schema.Definitions {
-		// index by GVK, if no GVK is found then it is the schema for a subfield
-		// of a Resource
-		d := schema.Definitions[k]
-		gvk, found := d.VendorExtensible.Extensions[kubernetesGVKExtensionKey]
-		if !found {
-			continue
-		}
-		// cast the extension to a []map[string]string
-		exts, ok := gvk.([]interface{})
-		if !ok || len(exts) != 1 {
-			continue
-		}
-		m, ok := exts[0].(map[string]interface{})
-		if !ok {
-			continue
-		}
+func parse(b []byte) (*spec.Schema, error) {
+	var sc spec.Schema
 
-		// build the index key and save it
-		g := m[groupKey].(string)
-		apiVersion := m[versionKey].(string)
-		if g != "" {
-			apiVersion = g + "/" + apiVersion
+	if err := sc.UnmarshalJSON(b); err != nil {
+		return nil, errors.Wrap(err)
+	}
+	AddDefinitions(sc.Definitions)
+	return &sc, nil
+}
+
+func resolve(root interface{}, ref *spec.Ref) (*spec.Schema, error) {
+	res, _, err := ref.GetPointer().Get(root)
+	if err != nil {
+		return nil, errors.Wrap(err)
+	}
+	switch sch := res.(type) {
+	case spec.Schema:
+		return &sch, nil
+	case *spec.Schema:
+		return sch, nil
+	case map[string]interface{}:
+		b, err := json.Marshal(sch)
+		if err != nil {
+			return nil, err
 		}
-		schemaByResourceType[yaml.TypeMeta{Kind: m[kindKey].(string), APIVersion: apiVersion}] = &d
+		newSch := new(spec.Schema)
+		if err = json.Unmarshal(b, newSch); err != nil {
+			return nil, err
+		}
+		return newSch, nil
+	default:
+		return nil, errors.Wrap(fmt.Errorf("unknown type for the resolved reference"))
 	}
 }
 
 func rootSchema() *spec.Schema {
 	initSchema()
-	return &schema
+	return &globalSchema.schema
 }

kyaml/openapi/openapi_test.go

@@ -1,18 +1,73 @@
 // Copyright 2019 The Kubernetes Authors.
 // SPDX-License-Identifier: Apache-2.0
 
-package openapi_test
+package openapi
 
 import (
+	"fmt"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
-	"sigs.k8s.io/kustomize/kyaml/openapi"
 	"sigs.k8s.io/kustomize/kyaml/yaml"
 )
 
+func TestAddSchema(t *testing.T) {
+	// reset package vars
+	globalSchema = openapiData{}
+
+	_, err := AddSchema(additionalSchema)
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
+	s, err := GetSchema(`{"$ref": "#/definitions/io.k8s.config.setters.replicas"}`)
+	if !assert.Greater(t, len(globalSchema.schema.Definitions), 200) {
+		t.FailNow()
+	}
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
+	assert.Equal(t, `map[x-kustomize:map[setBy:Jane setter:map[name:replicas value:5]]]`,
+		fmt.Sprintf("%v", s.Schema.Extensions))
+}
+
+func TestNoUseBuiltInSchema_AddSchema(t *testing.T) {
+	// reset package vars
+	globalSchema = openapiData{}
+
+	SuppressBuiltInSchemaUse()
+	_, err := AddSchema(additionalSchema)
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
+	s, err := GetSchema(`{"$ref": "#/definitions/io.k8s.config.setters.replicas"}`)
+	if !assert.Greater(t, len(globalSchema.schema.Definitions), 1) {
+		t.FailNow()
+	}
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
+	assert.Equal(t, `map[x-kustomize:map[setBy:Jane setter:map[name:replicas value:5]]]`,
+		fmt.Sprintf("%v", s.Schema.Extensions))
+}
+
+var additionalSchema = []byte(`
+{
+  "definitions": {
+    "io.k8s.config.setters.replicas": {
+      "description": "replicas description.",
+      "type": "integer",
+      "x-kustomize": {"setBy":"Jane","setter": {"name":"replicas","value":"5"}}
+    }
+  },
+  "invalid": "field"
+}
+`)
+
 func TestSchemaForResourceType(t *testing.T) {
-	s := openapi.SchemaForResourceType(
+	// reset package vars
+	globalSchema = openapiData{}
+
+	s := SchemaForResourceType(
 		yaml.TypeMeta{APIVersion: "apps/v1", Kind: "Deployment"})
 	if !assert.NotNil(t, s) {
 		t.FailNow()

kyaml/setters/addyaml.go

@@ -55,8 +55,9 @@ func (m *customFieldSetter) Filter(object *yaml.RNode) (*yaml.RNode, error) {
 			return node.PipeE(m)
 		})
 	case yaml.ScalarNode:
-		// only create the setter for fields with the given name
-		if m.currentFieldName != m.Field {
+		// if filed is empty, create the setter for all fields with given value
+		// else only create the setter for given field and value combination, with given name
+		if m.Field != "" && m.currentFieldName != m.Field {
 			return object, nil
 		}
 		if err := m.create(object); err != nil {