TemplateProcessor can add custom resource schemas to openapi

kyaml/fn/framework/example_test.go

@@ -9,6 +9,7 @@ import (
 	"log"
 	"path/filepath"
 
+	"github.com/markbates/pkger"
 	"sigs.k8s.io/kustomize/kyaml/errors"
 	"sigs.k8s.io/kustomize/kyaml/fn/framework"
 	"sigs.k8s.io/kustomize/kyaml/fn/framework/command"
@@ -221,7 +222,7 @@ func ExampleTemplateProcessor_generate_files() {
 		// Templates
 		ResourceTemplates: []framework.ResourceTemplate{{
 			Templates: framework.TemplatesFromFile(
-				filepath.Join("testdata", "example", "templatefiles", "deployment.template"),
+				pkger.Include("/fn/framework/testdata/example/templatefiles/deployment.template"),
 			),
 		}},
 	}

kyaml/fn/framework/framework.go

@@ -111,7 +111,7 @@ func Execute(p ResourceListProcessor, rlSource *kio.ByteReadWriter) error {
 	rl := ResourceList{}
 	var err error
 	if rl.Items, err = rlSource.Read(); err != nil {
-		return errors.Wrap(err)
+		return errors.WrapPrefixf(err, "failed to read ResourceList input")
 	}
 	rl.FunctionConfig = rlSource.FunctionConfig
 

kyaml/fn/framework/processors.go

@@ -13,10 +13,11 @@ import (
 	"text/template"
 
 	"github.com/markbates/pkger"
-
+	"k8s.io/kube-openapi/pkg/validation/spec"
 	"sigs.k8s.io/kustomize/kyaml/errors"
 	"sigs.k8s.io/kustomize/kyaml/kio"
 	"sigs.k8s.io/kustomize/kyaml/kio/filters"
+	"sigs.k8s.io/kustomize/kyaml/openapi"
 	"sigs.k8s.io/kustomize/kyaml/yaml"
 )
 
@@ -204,12 +205,30 @@ type TemplateProcessor struct {
 	// PostProcessFilters provides a hook to manipulate the ResourceList's items after template
 	// filters are applied.
 	PostProcessFilters []kio.Filter
+
+	// AdditionalSchemas is a function that returns a list of schema definitions to add to openapi.
+	// This enables correct merging of custom resource fields.
+	AdditionalSchemas SchemaDefinitionFunc
 }
 
+// SchemaDefinitionFunc is a function that provides a list of schema definitions.
+// TemplateProcessor uses this to defer loading of schemas to the point where they are used.
+type SchemaDefinitionFunc func() ([]*spec.Definitions, error)
+
 // Filter implements the kio.Filter interface, enabling you to use TemplateProcessor
 // as part of a higher-level ResourceListProcessor like VersionedAPIProcessor.
 // It sets up all the features of TemplateProcessors as a pipeline of filters and executes them.
 func (tp TemplateProcessor) Filter(items []*yaml.RNode) ([]*yaml.RNode, error) {
+	if tp.AdditionalSchemas != nil {
+		defs, err := tp.AdditionalSchemas()
+		if err != nil {
+			return nil, errors.WrapPrefixf(err, "parsing AdditionalSchemas")
+		}
+		for i := range defs {
+			openapi.AddDefinitions(*defs[i])
+		}
+	}
+
 	buf := &kio.PackageBuffer{Nodes: items}
 	pipeline := kio.Pipeline{
 		Inputs: []kio.Reader{buf},
@@ -355,8 +374,7 @@ func TemplatesFromFile(files ...string) TemplatesFunc {
 	return func() ([]*template.Template, error) {
 		var templates []*template.Template
 		for i := range files {
-			n := filepath.Base(files[i])
+			t, err := parseTemplate(files[i])
-			t, err := template.New(n).ParseFiles(files[i])
 			if err != nil {
 				return nil, err
 			}
@@ -366,6 +384,24 @@ func TemplatesFromFile(files ...string) TemplatesFunc {
 	}
 }
 
+func parseTemplate(filename string) (*template.Template, error) {
+	f, err := pkger.Open(filename)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+	bs, err := ioutil.ReadAll(f)
+	if err != nil {
+		return nil, err
+	}
+
+	t, err := template.New(filepath.Base(filename)).Parse(string(bs))
+	if err != nil {
+		return nil, err
+	}
+	return t, nil
+}
+
 // TemplatesFromDir returns a TemplatesFunc that will generate templates from the provided
 // directories. Only files suffixed with .template.yaml will be included.
 // This is a helper to facilitate providing ResourceTemplates, PatchTemplates and
@@ -382,18 +418,7 @@ func TemplatesFromDir(dirs ...pkger.Dir) TemplatesFunc {
 				if !strings.HasSuffix(info.Name(), ".template.yaml") {
 					return nil
 				}
-				name := path.Join(dir, info.Name())
+				t, err := parseTemplate(path.Join(dir, info.Name()))
-				f, err := pkger.Open(name)
-				if err != nil {
-					return err
-				}
-				defer f.Close()
-
-				b, err := ioutil.ReadAll(f)
-				if err != nil {
-					return err
-				}
-				t, err := template.New(info.Name()).Parse(string(b))
 				if err != nil {
 					return err
 				}
@@ -408,3 +433,71 @@ func TemplatesFromDir(dirs ...pkger.Dir) TemplatesFunc {
 		return pt, nil
 	}
 }
+
+// SchemaDefinitionsFromFile returns a SchemaDefinitionFunc that will load schemas from the provided files.
+// This is a helper to facilitate providing custom resource schemas to a TemplateProcessor.
+func SchemaDefinitionsFromFile(files ...string) SchemaDefinitionFunc {
+	return func() ([]*spec.Definitions, error) {
+		var defs []*spec.Definitions
+		for _, filename := range files {
+			def, err := readSchemaJSON(filename)
+			if err != nil {
+				return nil, err
+			}
+			defs = append(defs, &def)
+		}
+		return defs, nil
+	}
+}
+
+// SchemaDefinitionsFromDir returns a SchemaDefinitionFunc that will load schemas from the provided directories.
+// This is a helper to facilitate providing custom resource schemas to a TemplateProcessor.
+func SchemaDefinitionsFromDir(dirs ...pkger.Dir) SchemaDefinitionFunc {
+	return func() ([]*spec.Definitions, error) {
+		var defs []*spec.Definitions
+		for i := range dirs {
+			dir := string(dirs[i])
+			err := pkger.Walk(dir, func(p string, info os.FileInfo, err error) error {
+				if err != nil {
+					return err
+				}
+				if !strings.HasSuffix(info.Name(), ".json") {
+					return nil
+				}
+				def, err := readSchemaJSON(path.Join(dir, info.Name()))
+				if err != nil {
+					return err
+				}
+				defs = append(defs, &def)
+				return nil
+			})
+			if err != nil {
+				return nil, err
+			}
+		}
+		return defs, nil
+	}
+}
+
+func readSchemaJSON(filename string) (spec.Definitions, error) {
+	f, err := pkger.Open(filename)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+	b, err := ioutil.ReadAll(f)
+	if err != nil {
+		return nil, err
+	}
+
+	var schema spec.Schema
+	err = schema.UnmarshalJSON(b)
+	if err != nil {
+		return nil, err
+	}
+
+	if schema.Definitions != nil {
+		return schema.Definitions, nil
+	}
+	return nil, errors.Errorf("schema did not contain any definitions")
+}

kyaml/fn/framework/processors_test.go

@@ -11,7 +11,9 @@ import (
 	"github.com/markbates/pkger"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	"k8s.io/kube-openapi/pkg/validation/spec"
 	"sigs.k8s.io/kustomize/kyaml/errors"
+	"sigs.k8s.io/kustomize/kyaml/openapi"
 	"sigs.k8s.io/kustomize/kyaml/yaml"
 
 	"sigs.k8s.io/kustomize/kyaml/fn/framework"
@@ -27,7 +29,7 @@ func TestTemplateProcessor_ResourceTemplates(t *testing.T) {
 		TemplateData: &API{},
 		ResourceTemplates: []framework.ResourceTemplate{{
 			Templates: framework.TemplatesFromDir(pkger.Dir(
-				"/fn/framework/testdata/template-processor/templates")),
+				"/fn/framework/testdata/template-processor/templates/basic")),
 		}},
 	}
 
@@ -81,7 +83,7 @@ func TestTemplateProcessor_PatchTemplates(t *testing.T) {
 			// Patch from dir with no selector templating
 			&framework.ResourcePatchTemplate{
 				Templates: framework.TemplatesFromDir(pkger.Dir(
-					"/fn/framework/testdata/template-processor/patches")),
+					"/fn/framework/testdata/template-processor/patches/basic")),
 				Selector: &framework.Selector{Names: []string{"foo"}},
 			},
 			// Patch from string with selector templating
@@ -525,3 +527,76 @@ spec:
 		})
 	}
 }
+
+func TestTemplateProcessor_AdditionalSchemas(t *testing.T) {
+	p := framework.TemplateProcessor{
+		AdditionalSchemas: func() ([]*spec.Definitions, error) {
+			// This adds the same thing twice, just to exercise both the ...FromDir and the ...FromFile helpers
+			c1, err := framework.SchemaDefinitionsFromDir("/fn/framework/testdata/template-processor/schemas")()
+			if err != nil {
+				return nil, errors.WrapPrefixf(err, "schema from dir")
+			}
+			c2, err := framework.SchemaDefinitionsFromFile("/fn/framework/testdata/template-processor/schemas/foo.json")()
+			if err != nil {
+				return nil, errors.WrapPrefixf(err, "schema from file")
+			}
+			return append(c1, c2...), nil
+		},
+		ResourceTemplates: []framework.ResourceTemplate{{
+			Templates: framework.TemplatesFromFile("/fn/framework/testdata/template-processor/templates/custom-resource/foo.yaml"),
+		}},
+		PatchTemplates: []framework.PatchTemplate{
+			&framework.ResourcePatchTemplate{
+				Templates: framework.TemplatesFromFile("/fn/framework/testdata/template-processor/patches/custom-resource/patch.template.yaml")},
+		},
+	}
+	out := new(bytes.Buffer)
+
+	rw := &kio.ByteReadWriter{Reader: bytes.NewBufferString(`
+apiVersion: config.kubernetes.io/v1alpha1
+kind: ResourceList
+items:
+- apiVersion: example.com/v1
+  kind: Foo
+  metadata:
+    name: source
+  spec:
+    targets:
+    - app: C
+      size: medium
+`),
+		Writer: out}
+	defer openapi.ResetOpenAPI()
+	require.NoError(t, framework.Execute(p, rw))
+	require.Equal(t, strings.TrimSpace(`
+apiVersion: config.kubernetes.io/v1alpha1
+kind: ResourceList
+items:
+- apiVersion: example.com/v1
+  kind: Foo
+  metadata:
+    name: source
+  spec:
+    targets:
+    - app: C
+      size: large
+      type: Ruby
+    - app: B
+      size: small
+- apiVersion: example.com/v1
+  kind: Foo
+  metadata:
+    name: example
+  spec:
+    targets:
+    - app: A
+      type: Go
+      size: small
+    - app: B
+      type: Go
+      size: small
+    - app: C
+      type: Ruby
+      size: large
+`), strings.TrimSpace(out.String()))
+}

kyaml/fn/framework/testdata/template-processor/patches/custom-resource/patch.template.yaml

@@ -0,0 +1,10 @@
+# Copyright 2021 The Kubernetes Authors.
+# SPDX-License-Identifier: Apache-2.0
+
+spec:
+  targets:
+  - app: B
+    size: small
+  - app: C
+    type: Ruby
+    size: large

kyaml/fn/framework/testdata/template-processor/schemas/foo.json

@@ -0,0 +1,58 @@
+{
+  "definitions": {
+    "com.example.v1.Foo": {
+      "type": "object",
+      "properties": {
+        "apiVersion": {
+          "description": "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources",
+          "type": "string"
+        },
+        "kind": {
+          "description": "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds",
+          "type": "string"
+        },
+        "metadata": {
+          "description": "Standard object's metadata. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata",
+          "$ref": "#/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.ObjectMeta"
+        },
+        "spec": {
+          "type": "object",
+          "required": [
+            "targets"
+          ],
+          "properties": {
+            "targets": {
+              "type": "array",
+              "x-kubernetes-patch-merge-key": "app",
+              "x-kubernetes-patch-strategy": "merge",
+              "items": {
+                "type": "object",
+                "required": [
+                  "app"
+                ],
+                "properties": {
+                  "app": {
+                    "type": "string"
+                  },
+                  "size": {
+                    "type": "string"
+                  },
+                  "type": {
+                    "type": "string"
+                  }
+                }
+              }
+            }
+          }
+        }
+      },
+      "x-kubernetes-group-version-kind": [
+        {
+          "group": "example.com",
+          "kind": "Foo",
+          "version": "v1"
+        }
+      ]
+    }
+  }
+}

kyaml/fn/framework/testdata/template-processor/templates/custom-resource/foo.yaml

@@ -0,0 +1,14 @@
+# Copyright 2021 The Kubernetes Authors.
+# SPDX-License-Identifier: Apache-2.0
+
+apiVersion: example.com/v1
+kind: Foo
+metadata:
+  name: example
+spec:
+  targets:
+  - app: A
+    type: Go
+    size: small
+  - app: B
+    type: Go