Merge pull request #3257 from monopole/binaryData

Secrets and ConfigMaps with binary data in kyaml

api/internal/generators/configmap.go

@@ -0,0 +1,51 @@
+// Copyright 2020 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package generators
+
+import (
+	"sigs.k8s.io/kustomize/api/filters/filtersutil"
+	"sigs.k8s.io/kustomize/api/ifc"
+	"sigs.k8s.io/kustomize/api/types"
+	"sigs.k8s.io/kustomize/kyaml/yaml"
+)
+
+// MakeConfigMap makes a configmap.
+//
+// ConfigMap: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.19/#configmap-v1-core
+//
+// ConfigMaps and Secrets are similar.
+//
+// Both objects have a `data` field, which contains a map from keys to
+// values that must be UTF-8 valid strings.  Such data might be simple text,
+// or whoever made the data may have done so by performing a base64 encoding
+// on binary data. Regardless, k8s has no means to know this, so it treats
+// the data field as a string.
+//
+// The ConfigMap has an additional field `binaryData`, also a map, but its
+// values are _intended_ to be interpreted as a base64 encoding of []byte,
+// by whatever makes use of the ConfigMap.
+//
+// In a ConfigMap, any key used in `data` cannot also be used in `binaryData`
+// and vice-versa.  A key must be unique across both maps.
+func MakeConfigMap(
+	ldr ifc.KvLoader, args *types.ConfigMapArgs) (rn *yaml.RNode, err error) {
+	rn, err = makeBaseNode("ConfigMap", args.Name, args.Namespace)
+	if err != nil {
+		return nil, err
+	}
+	m, err := makeValidatedDataMap(ldr, args.Name, args.KvPairSources)
+	if err != nil {
+		return nil, err
+	}
+	for _, k := range filtersutil.SortedMapKeys(m) {
+		fldName, vrN := makeConfigMapValueRNode(m[k])
+		if _, err = rn.Pipe(
+			yaml.LookupCreate(yaml.MappingNode, fldName),
+			yaml.SetField(k, vrN)); err != nil {
+			return nil, err
+		}
+	}
+	copyLabelsAndAnnotations(rn, args.Options)
+	return rn, err
+}

api/internal/generators/configmap_test.go

@@ -0,0 +1,223 @@
+// Copyright 2020 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package generators_test
+
+import (
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"sigs.k8s.io/kustomize/api/filesys"
+	. "sigs.k8s.io/kustomize/api/internal/generators"
+	"sigs.k8s.io/kustomize/api/kv"
+	"sigs.k8s.io/kustomize/api/loader"
+	valtest_test "sigs.k8s.io/kustomize/api/testutils/valtest"
+	"sigs.k8s.io/kustomize/api/types"
+)
+
+var binaryHello = []byte{
+	0xff, // non-utf8
+	0x68, // h
+	0x65, // e
+	0x6c, // l
+	0x6c, // l
+	0x6f, // o
+}
+
+func manyHellos(count int) (result []byte) {
+	for i := 0; i < count; i++ {
+		result = append(result, binaryHello...)
+	}
+	return
+}
+
+func TestMakeConfigMap(t *testing.T) {
+	type expected struct {
+		out    string
+		errMsg string
+	}
+
+	testCases := map[string]struct {
+		args types.ConfigMapArgs
+		exp  expected
+	}{
+		"construct config map from env": {
+			args: types.ConfigMapArgs{
+				GeneratorArgs: types.GeneratorArgs{
+					Name: "envConfigMap",
+					KvPairSources: types.KvPairSources{
+						EnvSources: []string{
+							filepath.Join("configmap", "app.env"),
+						},
+					},
+				},
+			},
+			exp: expected{
+				out: `apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: envConfigMap
+data:
+  DB_PASSWORD: qwerty
+  DB_USERNAME: admin
+`,
+			},
+		},
+		"construct config map from text file": {
+			args: types.ConfigMapArgs{
+				GeneratorArgs: types.GeneratorArgs{
+					Name: "fileConfigMap1",
+					KvPairSources: types.KvPairSources{
+						FileSources: []string{
+							filepath.Join("configmap", "app-init.ini"),
+						},
+					},
+				},
+			},
+			exp: expected{
+				out: `apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: fileConfigMap1
+data:
+  app-init.ini: |
+    FOO=bar
+    BAR=baz
+`,
+			},
+		},
+		"construct config map from text and binary file": {
+			args: types.ConfigMapArgs{
+				GeneratorArgs: types.GeneratorArgs{
+					Name: "fileConfigMap2",
+					KvPairSources: types.KvPairSources{
+						FileSources: []string{
+							filepath.Join("configmap", "app-init.ini"),
+							filepath.Join("configmap", "app.bin"),
+						},
+					},
+				},
+			},
+			exp: expected{
+				out: `apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: fileConfigMap2
+data:
+  app-init.ini: |
+    FOO=bar
+    BAR=baz
+binaryData:
+  app.bin: |
+    /2hlbGxv/2hlbGxv/2hlbGxv/2hlbGxv/2hlbGxv/2hlbGxv/2hlbGxv/2hlbGxv/2hlbG
+    xv/2hlbGxv/2hlbGxv/2hlbGxv/2hlbGxv/2hlbGxv/2hlbGxv/2hlbGxv/2hlbGxv/2hl
+    bGxv/2hlbGxv/2hlbGxv/2hlbGxv/2hlbGxv/2hlbGxv/2hlbGxv/2hlbGxv/2hlbGxv/2
+    hlbGxv/2hlbGxv/2hlbGxv/2hlbGxv
+`,
+			},
+		},
+		"construct config map from literal": {
+			args: types.ConfigMapArgs{
+				GeneratorArgs: types.GeneratorArgs{
+					Name: "literalConfigMap1",
+					KvPairSources: types.KvPairSources{
+						LiteralSources: []string{"a=x", "b=y", "c=\"Hello World\"", "d='true'"},
+					},
+					Options: &types.GeneratorOptions{
+						Labels: map[string]string{
+							"foo": "bar",
+						},
+					},
+				},
+			},
+			exp: expected{
+				out: `apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: literalConfigMap1
+  labels:
+    foo: 'bar'
+data:
+  a: x
+  b: y
+  c: Hello World
+  d: "true"
+`,
+			},
+		},
+		"construct config map from literal with GeneratorOptions in ConfigMapArgs": {
+			args: types.ConfigMapArgs{
+				GeneratorArgs: types.GeneratorArgs{
+					Name: "literalConfigMap2",
+					KvPairSources: types.KvPairSources{
+						LiteralSources: []string{"a=x", "b=y", "c=\"Hello World\"", "d='true'"},
+					},
+					Options: &types.GeneratorOptions{
+						Labels: map[string]string{
+							"veggie": "celery",
+							"dog":    "beagle",
+							"cat":    "annoying",
+						},
+						Annotations: map[string]string{
+							"river": "Missouri",
+							"city":  "Iowa City",
+						},
+					},
+				},
+			},
+			exp: expected{
+				out: `apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: literalConfigMap2
+  labels:
+    cat: 'annoying'
+    dog: 'beagle'
+    veggie: 'celery'
+  annotations:
+    city: 'Iowa City'
+    river: 'Missouri'
+data:
+  a: x
+  b: y
+  c: Hello World
+  d: "true"
+`,
+			},
+		},
+	}
+	fSys := filesys.MakeFsInMemory()
+	fSys.WriteFile(
+		filesys.RootedPath("configmap", "app.env"),
+		[]byte("DB_USERNAME=admin\nDB_PASSWORD=qwerty\n"))
+	fSys.WriteFile(
+		filesys.RootedPath("configmap", "app-init.ini"),
+		[]byte("FOO=bar\nBAR=baz\n"))
+	fSys.WriteFile(
+		filesys.RootedPath("configmap", "app.bin"),
+		manyHellos(30))
+	kvLdr := kv.NewLoader(
+		loader.NewFileLoaderAtRoot(fSys),
+		valtest_test.MakeFakeValidator())
+
+	for n := range testCases {
+		tc := testCases[n]
+		t.Run(n, func(t *testing.T) {
+			rn, err := MakeConfigMap(kvLdr, &tc.args)
+			if err != nil {
+				if !assert.EqualError(t, err, tc.exp.errMsg) {
+					t.FailNow()
+				}
+				return
+			}
+			if tc.exp.errMsg != "" {
+				t.Fatalf("%s: should return error '%s'", n, tc.exp.errMsg)
+			}
+			output := rn.MustString()
+			if !assert.Equal(t, tc.exp.out, output) {
+				t.FailNow()
+			}
+		})
+	}
+}

api/internal/generators/secret.go

@@ -0,0 +1,60 @@
+// Copyright 2020 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package generators
+
+import (
+	"sigs.k8s.io/kustomize/api/filters/filtersutil"
+	"sigs.k8s.io/kustomize/api/ifc"
+	"sigs.k8s.io/kustomize/api/types"
+	"sigs.k8s.io/kustomize/kyaml/yaml"
+)
+
+// MakeSecret makes a kubernetes Secret.
+//
+// Secret: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.19/#secret-v1-core
+//
+// ConfigMaps and Secrets are similar.
+//
+// Like a ConfigMap, a Secret has a `data` field, but unlike a ConfigMap it has
+// no `binaryData` field.
+//
+// All of a Secret's data is assumed to be opaque in nature, and assumed to be
+// base64 encoded from its original representation, regardless of whether the
+// original data was UTF-8 text or binary.
+//
+// This encoding provides no secrecy. It's just a neutral, common means to
+// represent opaque text and binary data.  Beneath the base64 encoding
+// is presumably further encoding under control of the Secret's consumer.
+//
+// A Secret has string field `type` which holds an identifier, used by the
+// client, to choose the algorithm to interpret the `data` field.  Kubernetes
+// cannot make use of this data; it's up to a controller or some pod's service
+// to interpret the value, using `type` as a clue as to how to do this.
+func MakeSecret(
+	ldr ifc.KvLoader, args *types.SecretArgs) (rn *yaml.RNode, err error) {
+	rn, err = makeBaseNode("Secret", args.Name, args.Namespace)
+	if err != nil {
+		return nil, err
+	}
+	if _, err := rn.Pipe(
+		yaml.FieldSetter{
+			Name:  "type",
+			Value: yaml.NewStringRNode("Opaque")}); err != nil {
+		return nil, err
+	}
+	m, err := makeValidatedDataMap(ldr, args.Name, args.KvPairSources)
+	if err != nil {
+		return nil, err
+	}
+	for _, k := range filtersutil.SortedMapKeys(m) {
+		vrN := makeSecretValueRNode(m[k])
+		if _, err = rn.Pipe(
+			yaml.LookupCreate(yaml.MappingNode, yaml.DataField),
+			yaml.SetField(k, vrN)); err != nil {
+			return nil, err
+		}
+	}
+	copyLabelsAndAnnotations(rn, args.Options)
+	return rn, err
+}

api/internal/generators/secret_test.go

@@ -0,0 +1,203 @@
+// Copyright 2020 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package generators_test
+
+import (
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"sigs.k8s.io/kustomize/api/filesys"
+	. "sigs.k8s.io/kustomize/api/internal/generators"
+	"sigs.k8s.io/kustomize/api/kv"
+	"sigs.k8s.io/kustomize/api/loader"
+	valtest_test "sigs.k8s.io/kustomize/api/testutils/valtest"
+	"sigs.k8s.io/kustomize/api/types"
+)
+
+func TestMakeSecret(t *testing.T) {
+	type expected struct {
+		out    string
+		errMsg string
+	}
+
+	testCases := map[string]struct {
+		args types.SecretArgs
+		exp  expected
+	}{
+		"construct secret from env": {
+			args: types.SecretArgs{
+				GeneratorArgs: types.GeneratorArgs{
+					Name: "envSecret",
+					KvPairSources: types.KvPairSources{
+						EnvSources: []string{
+							filepath.Join("secret", "app.env"),
+						},
+					},
+				},
+			},
+			exp: expected{
+				out: `apiVersion: v1
+kind: Secret
+metadata:
+  name: envSecret
+type: Opaque
+data:
+  DB_PASSWORD: cXdlcnR5
+  DB_USERNAME: YWRtaW4=
+`,
+			},
+		},
+		"construct secret from text file": {
+			args: types.SecretArgs{
+				GeneratorArgs: types.GeneratorArgs{
+					Name: "fileSecret1",
+					KvPairSources: types.KvPairSources{
+						FileSources: []string{
+							filepath.Join("secret", "app-init.ini"),
+						},
+					},
+				},
+			},
+			exp: expected{
+				out: `apiVersion: v1
+kind: Secret
+metadata:
+  name: fileSecret1
+type: Opaque
+data:
+  app-init.ini: Rk9PPWJhcgpCQVI9YmF6Cg==
+`,
+			},
+		},
+		"construct secret from text and binary file": {
+			args: types.SecretArgs{
+				GeneratorArgs: types.GeneratorArgs{
+					Name: "fileSecret2",
+					KvPairSources: types.KvPairSources{
+						FileSources: []string{
+							filepath.Join("secret", "app-init.ini"),
+							filepath.Join("secret", "app.bin"),
+						},
+					},
+				},
+			},
+			exp: expected{
+				out: `apiVersion: v1
+kind: Secret
+metadata:
+  name: fileSecret2
+type: Opaque
+data:
+  app-init.ini: Rk9PPWJhcgpCQVI9YmF6Cg==
+  app.bin: //0=
+`,
+			},
+		},
+		"construct secret from literal": {
+			args: types.SecretArgs{
+				GeneratorArgs: types.GeneratorArgs{
+					Name: "literalSecret1",
+					KvPairSources: types.KvPairSources{
+						LiteralSources: []string{"a=x", "b=y", "c=\"Hello World\"", "d='true'"},
+					},
+					Options: &types.GeneratorOptions{
+						Labels: map[string]string{
+							"foo": "bar",
+						},
+					},
+				},
+			},
+			exp: expected{
+				out: `apiVersion: v1
+kind: Secret
+metadata:
+  name: literalSecret1
+  labels:
+    foo: 'bar'
+type: Opaque
+data:
+  a: eA==
+  b: eQ==
+  c: SGVsbG8gV29ybGQ=
+  d: dHJ1ZQ==
+`,
+			},
+		},
+		"construct secret from literal with GeneratorOptions in SecretArgs": {
+			args: types.SecretArgs{
+				GeneratorArgs: types.GeneratorArgs{
+					Name: "literalSecret2",
+					KvPairSources: types.KvPairSources{
+						LiteralSources: []string{"a=x", "b=y", "c=\"Hello World\"", "d='true'"},
+					},
+					Options: &types.GeneratorOptions{
+						Labels: map[string]string{
+							"veggie": "celery",
+							"dog":    "beagle",
+							"cat":    "annoying",
+						},
+						Annotations: map[string]string{
+							"river": "Missouri",
+							"city":  "Iowa City",
+						},
+					},
+				},
+			},
+			exp: expected{
+				out: `apiVersion: v1
+kind: Secret
+metadata:
+  name: literalSecret2
+  labels:
+    cat: 'annoying'
+    dog: 'beagle'
+    veggie: 'celery'
+  annotations:
+    city: 'Iowa City'
+    river: 'Missouri'
+type: Opaque
+data:
+  a: eA==
+  b: eQ==
+  c: SGVsbG8gV29ybGQ=
+  d: dHJ1ZQ==
+`,
+			},
+		},
+	}
+	fSys := filesys.MakeFsInMemory()
+	fSys.WriteFile(
+		filesys.RootedPath("secret", "app.env"),
+		[]byte("DB_USERNAME=admin\nDB_PASSWORD=qwerty\n"))
+	fSys.WriteFile(
+		filesys.RootedPath("secret", "app-init.ini"),
+		[]byte("FOO=bar\nBAR=baz\n"))
+	fSys.WriteFile(
+		filesys.RootedPath("secret", "app.bin"),
+		[]byte{0xff, 0xfd})
+	kvLdr := kv.NewLoader(
+		loader.NewFileLoaderAtRoot(fSys),
+		valtest_test.MakeFakeValidator())
+
+	for n := range testCases {
+		tc := testCases[n]
+		t.Run(n, func(t *testing.T) {
+			rn, err := MakeSecret(kvLdr, &tc.args)
+			if err != nil {
+				if !assert.EqualError(t, err, tc.exp.errMsg) {
+					t.FailNow()
+				}
+				return
+			}
+			if tc.exp.errMsg != "" {
+				t.Fatalf("%s: should return error '%s'", n, tc.exp.errMsg)
+			}
+			output := rn.MustString()
+			if !assert.Equal(t, tc.exp.out, output) {
+				t.FailNow()
+			}
+		})
+	}
+}

api/internal/generators/utils.go

@@ -0,0 +1,139 @@
+// Copyright 2020 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package generators
+
+import (
+	"encoding/base64"
+	"fmt"
+	"strings"
+	"unicode/utf8"
+
+	"github.com/go-errors/errors"
+	"sigs.k8s.io/kustomize/api/filters/filtersutil"
+	"sigs.k8s.io/kustomize/api/ifc"
+	"sigs.k8s.io/kustomize/api/types"
+	"sigs.k8s.io/kustomize/kyaml/yaml"
+)
+
+func makeBaseNode(kind, name, namespace string) (*yaml.RNode, error) {
+	rn, err := yaml.Parse(fmt.Sprintf(`
+apiVersion: v1
+kind: %s
+`, kind))
+	if err != nil {
+		return nil, err
+	}
+	if name == "" {
+		return nil, errors.Errorf("a configmap must have a name")
+	}
+	if _, err := rn.Pipe(yaml.SetK8sName(name)); err != nil {
+		return nil, err
+	}
+	if namespace != "" {
+		if _, err := rn.Pipe(yaml.SetK8sNamespace(namespace)); err != nil {
+			return nil, err
+		}
+	}
+	return rn, nil
+}
+
+func makeValidatedDataMap(
+	ldr ifc.KvLoader, name string, sources types.KvPairSources) (map[string]string, error) {
+	pairs, err := ldr.Load(sources)
+	if err != nil {
+		return nil, errors.WrapPrefix(err, "loading KV pairs", 0)
+	}
+	knownKeys := make(map[string]string)
+	for _, p := range pairs {
+		// legal key: alphanumeric characters, '-', '_' or '.'
+		if err := ldr.Validator().ErrIfInvalidKey(p.Key); err != nil {
+			return nil, err
+		}
+		if _, ok := knownKeys[p.Key]; ok {
+			return nil, errors.Errorf(
+				"configmap %s illegally repeats the key `%s`", name, p.Key)
+		}
+		knownKeys[p.Key] = p.Value
+	}
+	return knownKeys, nil
+}
+
+// copyLabelsAndAnnotations copies labels and annotations from
+// GeneratorOptions into the given object.
+func copyLabelsAndAnnotations(
+	rn *yaml.RNode, opts *types.GeneratorOptions) error {
+	if opts == nil {
+		return nil
+	}
+	for _, k := range filtersutil.SortedMapKeys(opts.Labels) {
+		v := opts.Labels[k]
+		if _, err := rn.Pipe(yaml.SetLabel(k, v)); err != nil {
+			return err
+		}
+	}
+	for _, k := range filtersutil.SortedMapKeys(opts.Annotations) {
+		v := opts.Annotations[k]
+		if _, err := rn.Pipe(yaml.SetAnnotation(k, v)); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// In a secret, all data is base64 encoded, regardless of its conformance
+// or lack thereof to UTF-8.
+func makeSecretValueRNode(s string) *yaml.RNode {
+	yN := &yaml.Node{Kind: yaml.ScalarNode}
+	// Purposely don't use YAML tags to identify the data as being plain text or
+	// binary.  It kubernetes Secrets the values in the `data` map are expected
+	// to be base64 encoded, and in ConfigMaps that same can be said for the
+	// values in the `binaryData` field.
+	yN.Tag = yaml.NodeTagString
+	yN.Value = encodeBase64(s)
+	if strings.Contains(yN.Value, "\n") {
+		yN.Style = yaml.LiteralStyle
+	}
+	return yaml.NewRNode(yN)
+}
+
+func makeConfigMapValueRNode(s string) (field string, rN *yaml.RNode) {
+	yN := &yaml.Node{Kind: yaml.ScalarNode}
+	yN.Tag = yaml.NodeTagString
+	if utf8.ValidString(s) {
+		field = yaml.DataField
+		yN.Value = s
+	} else {
+		field = yaml.BinaryDataField
+		yN.Value = encodeBase64(s)
+	}
+	if strings.Contains(yN.Value, "\n") {
+		yN.Style = yaml.LiteralStyle
+	}
+	return field, yaml.NewRNode(yN)
+}
+
+// encodeBase64 encodes s as base64 that is broken up into multiple lines
+// as appropriate for the resulting length.
+func encodeBase64(s string) string {
+	const lineLen = 70
+	encLen := base64.StdEncoding.EncodedLen(len(s))
+	lines := encLen/lineLen + 1
+	buf := make([]byte, encLen*2+lines)
+	in := buf[0:encLen]
+	out := buf[encLen:]
+	base64.StdEncoding.Encode(in, []byte(s))
+	k := 0
+	for i := 0; i < len(in); i += lineLen {
+		j := i + lineLen
+		if j > len(in) {
+			j = len(in)
+		}
+		k += copy(out[k:], in[i:j])
+		if lines > 1 {
+			out[k] = '\n'
+			k++
+		}
+	}
+	return string(out[:k])
+}

api/internal/wrappy/factory.go

@@ -6,12 +6,13 @@ package wrappy
 import (
 	"bytes"
 	"fmt"
-	"sort"
 
-	"github.com/go-errors/errors"
+	"sigs.k8s.io/kustomize/api/hasher"
 	"sigs.k8s.io/kustomize/api/ifc"
+	"sigs.k8s.io/kustomize/api/internal/generators"
 	"sigs.k8s.io/kustomize/api/konfig"
 	"sigs.k8s.io/kustomize/api/types"
+	"sigs.k8s.io/kustomize/kyaml/filtersutil"
 	"sigs.k8s.io/kustomize/kyaml/kio"
 	"sigs.k8s.io/kustomize/kyaml/yaml"
 )
@@ -73,112 +74,38 @@ func (k *WNodeFactory) FromMap(m map[string]interface{}) ifc.Kunstructured {
 	return rn
 }
 
-func (k *WNodeFactory) Hasher() ifc.KunstructuredHasher {
-	panic("TODO(#WNodeFactory): implement Hasher")
+// kustHash computes a hash of an unstructured object.
+type kustHash struct{}
+
+// Hash returns a hash of the given object
+func (h *kustHash) Hash(m ifc.Kunstructured) (string, error) {
+	node, err := filtersutil.GetRNode(m)
+	if err != nil {
+		return "", err
+	}
+	return hasher.HashRNode(node)
 }
 
+func (k *WNodeFactory) Hasher() ifc.KunstructuredHasher {
+	return &kustHash{}
+}
+
+// MakeConfigMap makes a wrapped configmap.
 func (k *WNodeFactory) MakeConfigMap(
 	ldr ifc.KvLoader, args *types.ConfigMapArgs) (ifc.Kunstructured, error) {
-	rn, err := k.makeConfigMap(ldr, args)
+	rn, err := generators.MakeConfigMap(ldr, args)
 	if err != nil {
 		return nil, err
 	}
 	return FromRNode(rn), nil
 }
 
-func (k *WNodeFactory) makeConfigMap(
-	ldr ifc.KvLoader, args *types.ConfigMapArgs) (*yaml.RNode, error) {
-	rn, err := yaml.Parse(`
-apiVersion: v1
-kind: ConfigMap
-`)
-	if err != nil {
-		return nil, err
-	}
-	err = applyGeneratorArgs(rn, ldr, args.GeneratorArgs)
-	return rn, err
-}
-
+// MakeSecret makes a wrapped secret.
 func (k *WNodeFactory) MakeSecret(
 	ldr ifc.KvLoader, args *types.SecretArgs) (ifc.Kunstructured, error) {
-	rn, err := k.makeSecret(ldr, args)
+	rn, err := generators.MakeSecret(ldr, args)
 	if err != nil {
 		return nil, err
 	}
 	return FromRNode(rn), nil
 }
-
-func (k *WNodeFactory) makeSecret(
-	ldr ifc.KvLoader, args *types.SecretArgs) (*yaml.RNode, error) {
-	rn, err := yaml.Parse(`
-apiVersion: v1
-kind: Secret
-`)
-	if err != nil {
-		return nil, err
-	}
-	err = applyGeneratorArgs(rn, ldr, args.GeneratorArgs)
-	if 1+1 == 2 {
-		err = fmt.Errorf("TODO(WNodeFactory): finish implementation of makeSecret")
-	}
-	return rn, err
-}
-
-func applyGeneratorArgs(
-	rn *yaml.RNode, ldr ifc.KvLoader, args types.GeneratorArgs) error {
-	if _, err := rn.Pipe(yaml.SetK8sName(args.Name)); err != nil {
-		return err
-	}
-	if args.Namespace != "" {
-		if _, err := rn.Pipe(yaml.SetK8sNamespace(args.Namespace)); err != nil {
-			return err
-		}
-	}
-	all, err := ldr.Load(args.KvPairSources)
-	if err != nil {
-		return errors.WrapPrefix(err, "loading KV pairs", 0)
-	}
-	for _, p := range all {
-		if err := ldr.Validator().ErrIfInvalidKey(p.Key); err != nil {
-			return err
-		}
-		if _, err := rn.Pipe(yaml.SetK8sData(p.Key, p.Value)); err != nil {
-			return errors.WrapPrefix(err, "configMap generate error", 0)
-		}
-	}
-	copyLabelsAndAnnotations(rn, args.Options)
-	return nil
-}
-
-// copyLabelsAndAnnotations copies labels and annotations from
-// GeneratorOptions into the given object.
-func copyLabelsAndAnnotations(
-	rn *yaml.RNode, opts *types.GeneratorOptions) error {
-	if opts == nil {
-		return nil
-	}
-	for _, k := range sortedKeys(opts.Labels) {
-		v := opts.Labels[k]
-		if _, err := rn.Pipe(yaml.SetLabel(k, v)); err != nil {
-			return err
-		}
-	}
-	for _, k := range sortedKeys(opts.Annotations) {
-		v := opts.Annotations[k]
-		if _, err := rn.Pipe(yaml.SetAnnotation(k, v)); err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
-func sortedKeys(m map[string]string) []string {
-	keys := make([]string, len(m))
-	i := 0
-	for k := range m {
-		keys[i] = k
-		i++
-	}
-	sort.Strings(keys)
-	return keys
-}

api/internal/wrappy/factory_test.go

@@ -5,392 +5,36 @@ package wrappy
 
 import (
 	"fmt"
-	"path/filepath"
 	"reflect"
 	"testing"
-
-	"github.com/stretchr/testify/assert"
-	"sigs.k8s.io/kustomize/api/filesys"
-	"sigs.k8s.io/kustomize/api/kv"
-	"sigs.k8s.io/kustomize/api/loader"
-	valtest_test "sigs.k8s.io/kustomize/api/testutils/valtest"
-	"sigs.k8s.io/kustomize/api/types"
 )
 
-func TestMakeConfigMap(t *testing.T) {
+func TestHasher(t *testing.T) {
+	input := `
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: foo
+data:
+  one: ""
+binaryData:
+  two: ""
+`
+	expect := "698h7c7t9m"
+
 	factory := &WNodeFactory{}
-	type expected struct {
-		out    string
-		errMsg string
+	k, err := factory.SliceFromBytes([]byte(input))
+	if err != nil {
+		t.Fatal(err)
 	}
 
-	testCases := map[string]struct {
-		args types.ConfigMapArgs
-		exp  expected
-	}{
-		"construct config map from env": {
-			args: types.ConfigMapArgs{
-				GeneratorArgs: types.GeneratorArgs{
-					Name: "envConfigMap",
-					KvPairSources: types.KvPairSources{
-						EnvSources: []string{
-							filepath.Join("configmap", "app.env"),
-						},
-					},
-				},
-			},
-			exp: expected{
-				out: `apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: envConfigMap
-data:
-  DB_USERNAME: admin
-  DB_PASSWORD: qwerty
-`,
-			},
-		},
-		"construct config map from text file": {
-			args: types.ConfigMapArgs{
-				GeneratorArgs: types.GeneratorArgs{
-					Name: "fileConfigMap1",
-					KvPairSources: types.KvPairSources{
-						FileSources: []string{
-							filepath.Join("configmap", "app-init.ini"),
-						},
-					},
-				},
-			},
-			exp: expected{
-				out: `apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: fileConfigMap1
-data:
-  app-init.ini: |
-    FOO=bar
-    BAR=baz
-`,
-			},
-		},
-		"construct config map from text and binary file": {
-			args: types.ConfigMapArgs{
-				GeneratorArgs: types.GeneratorArgs{
-					Name: "fileConfigMap2",
-					KvPairSources: types.KvPairSources{
-						FileSources: []string{
-							filepath.Join("configmap", "app-init.ini"),
-							filepath.Join("configmap", "app.bin"),
-						},
-					},
-				},
-			},
-			exp: expected{
-				errMsg: "configMap generate error: key 'app.bin' appears " +
-					"to have non-utf8 data; binaryData field not yet supported",
-				out: `apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: fileConfigMap2
-data:
-  app-init.ini: |
-    FOO=bar
-    BAR=baz
-`,
-			},
-		},
-		"construct config map from literal": {
-			args: types.ConfigMapArgs{
-				GeneratorArgs: types.GeneratorArgs{
-					Name: "literalConfigMap1",
-					KvPairSources: types.KvPairSources{
-						LiteralSources: []string{"a=x", "b=y", "c=\"Hello World\"", "d='true'"},
-					},
-					Options: &types.GeneratorOptions{
-						Labels: map[string]string{
-							"foo": "bar",
-						},
-					},
-				},
-			},
-			exp: expected{
-				out: `apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: literalConfigMap1
-  labels:
-    foo: 'bar'
-data:
-  a: x
-  b: y
-  c: Hello World
-  d: "true"
-`,
-			},
-		},
-		"construct config map from literal with GeneratorOptions in ConfigMapArgs": {
-			args: types.ConfigMapArgs{
-				GeneratorArgs: types.GeneratorArgs{
-					Name: "literalConfigMap2",
-					KvPairSources: types.KvPairSources{
-						LiteralSources: []string{"a=x", "b=y", "c=\"Hello World\"", "d='true'"},
-					},
-					Options: &types.GeneratorOptions{
-						Labels: map[string]string{
-							"veggie": "celery",
-							"dog":    "beagle",
-							"cat":    "annoying",
-						},
-						Annotations: map[string]string{
-							"river": "Missouri",
-							"city":  "Iowa City",
-						},
-					},
-				},
-			},
-			exp: expected{
-				out: `apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: literalConfigMap2
-  labels:
-    cat: 'annoying'
-    dog: 'beagle'
-    veggie: 'celery'
-  annotations:
-    city: 'Iowa City'
-    river: 'Missouri'
-data:
-  a: x
-  b: y
-  c: Hello World
-  d: "true"
-`,
-			},
-		},
+	hasher := factory.Hasher()
+	result, err := hasher.Hash(k[0])
+	if err != nil {
+		t.Fatal(err)
 	}
-	fSys := filesys.MakeFsInMemory()
-	fSys.WriteFile(
-		filesys.RootedPath("configmap", "app.env"),
-		[]byte("DB_USERNAME=admin\nDB_PASSWORD=qwerty\n"))
-	fSys.WriteFile(
-		filesys.RootedPath("configmap", "app-init.ini"),
-		[]byte("FOO=bar\nBAR=baz\n"))
-	fSys.WriteFile(
-		filesys.RootedPath("configmap", "app.bin"),
-		[]byte{0xff, 0xfd})
-	kvLdr := kv.NewLoader(
-		loader.NewFileLoaderAtRoot(fSys),
-		valtest_test.MakeFakeValidator())
-
-	for n := range testCases {
-		tc := testCases[n]
-		t.Run(n, func(t *testing.T) {
-			rn, err := factory.makeConfigMap(kvLdr, &tc.args)
-			if err != nil {
-				if !assert.EqualError(t, err, tc.exp.errMsg) {
-					t.FailNow()
-				}
-				return
-			}
-			if tc.exp.errMsg != "" {
-				t.Fatalf("%s: should return error '%s'", n, tc.exp.errMsg)
-			}
-			output := rn.MustString()
-			if !assert.Equal(t, tc.exp.out, output) {
-				t.FailNow()
-			}
-		})
-	}
-}
-
-func TestMakeSecret(t *testing.T) {
-	factory := &WNodeFactory{}
-	type expected struct {
-		out    string
-		errMsg string
-	}
-
-	testCases := map[string]struct {
-		args types.SecretArgs
-		exp  expected
-	}{
-		"construct secret from env": {
-			args: types.SecretArgs{
-				GeneratorArgs: types.GeneratorArgs{
-					Name: "envSecret",
-					KvPairSources: types.KvPairSources{
-						EnvSources: []string{
-							filepath.Join("secret", "app.env"),
-						},
-					},
-				},
-			},
-			exp: expected{
-				errMsg: "TODO(WNodeFactory): finish implementation of makeSecret",
-				out: `apiVersion: v1
-kind: Secret
-metadata:
-  name: envSecret
-data:
-  DB_USERNAME: admin
-  DB_PASSWORD: qwerty
-`,
-			},
-		},
-		"construct secret from text file": {
-			args: types.SecretArgs{
-				GeneratorArgs: types.GeneratorArgs{
-					Name: "fileSecret1",
-					KvPairSources: types.KvPairSources{
-						FileSources: []string{
-							filepath.Join("secret", "app-init.ini"),
-						},
-					},
-				},
-			},
-			exp: expected{
-				errMsg: "TODO(WNodeFactory): finish implementation of makeSecret",
-				out: `apiVersion: v1
-kind: Secret
-metadata:
-  name: fileSecret1
-data:
-  app-init.ini: |
-    FOO=bar
-    BAR=baz
-`,
-			},
-		},
-		"construct secret from text and binary file": {
-			args: types.SecretArgs{
-				GeneratorArgs: types.GeneratorArgs{
-					Name: "fileSecret2",
-					KvPairSources: types.KvPairSources{
-						FileSources: []string{
-							filepath.Join("secret", "app-init.ini"),
-							filepath.Join("secret", "app.bin"),
-						},
-					},
-				},
-			},
-			exp: expected{
-				errMsg: "TODO(WNodeFactory): finish implementation of makeSecret",
-				out: `apiVersion: v1
-kind: Secret
-metadata:
-  name: fileSecret2
-data:
-  app-init.ini: |
-    FOO=bar
-    BAR=baz
-`,
-			},
-		},
-		"construct secret from literal": {
-			args: types.SecretArgs{
-				GeneratorArgs: types.GeneratorArgs{
-					Name: "literalSecret1",
-					KvPairSources: types.KvPairSources{
-						LiteralSources: []string{"a=x", "b=y", "c=\"Hello World\"", "d='true'"},
-					},
-					Options: &types.GeneratorOptions{
-						Labels: map[string]string{
-							"foo": "bar",
-						},
-					},
-				},
-			},
-			exp: expected{
-				errMsg: "TODO(WNodeFactory): finish implementation of makeSecret",
-				out: `apiVersion: v1
-kind: Secret
-metadata:
-  name: literalSecret1
-  labels:
-    foo: 'bar'
-data:
-  a: x
-  b: y
-  c: Hello World
-  d: "true"
-`,
-			},
-		},
-		"construct secret from literal with GeneratorOptions in SecretArgs": {
-			args: types.SecretArgs{
-				GeneratorArgs: types.GeneratorArgs{
-					Name: "literalSecret2",
-					KvPairSources: types.KvPairSources{
-						LiteralSources: []string{"a=x", "b=y", "c=\"Hello World\"", "d='true'"},
-					},
-					Options: &types.GeneratorOptions{
-						Labels: map[string]string{
-							"veggie": "celery",
-							"dog":    "beagle",
-							"cat":    "annoying",
-						},
-						Annotations: map[string]string{
-							"river": "Missouri",
-							"city":  "Iowa City",
-						},
-					},
-				},
-			},
-			exp: expected{
-				errMsg: "TODO(WNodeFactory): finish implementation of makeSecret",
-				out: `apiVersion: v1
-kind: Secret
-metadata:
-  name: literalSecret2
-  labels:
-    cat: 'annoying'
-    dog: 'beagle'
-    veggie: 'celery'
-  annotations:
-    city: 'Iowa City'
-    river: 'Missouri'
-data:
-  a: x
-  b: y
-  c: Hello World
-  d: "true"
-`,
-			},
-		},
-	}
-	fSys := filesys.MakeFsInMemory()
-	fSys.WriteFile(
-		filesys.RootedPath("secret", "app.env"),
-		[]byte("DB_USERNAME=admin\nDB_PASSWORD=qwerty\n"))
-	fSys.WriteFile(
-		filesys.RootedPath("secret", "app-init.ini"),
-		[]byte("FOO=bar\nBAR=baz\n"))
-	fSys.WriteFile(
-		filesys.RootedPath("secret", "app.bin"),
-		[]byte{0xff, 0xfd})
-	kvLdr := kv.NewLoader(
-		loader.NewFileLoaderAtRoot(fSys),
-		valtest_test.MakeFakeValidator())
-
-	for n := range testCases {
-		tc := testCases[n]
-		t.Run(n, func(t *testing.T) {
-			rn, err := factory.makeSecret(kvLdr, &tc.args)
-			if err != nil {
-				if !assert.EqualError(t, err, tc.exp.errMsg) {
-					t.FailNow()
-				}
-				return
-			}
-			if tc.exp.errMsg != "" {
-				t.Fatalf("%s: should return error '%s'", n, tc.exp.errMsg)
-			}
-			output := rn.MustString()
-			if !assert.Equal(t, tc.exp.out, output) {
-				t.FailNow()
-			}
-		})
+	if result != expect {
+		t.Fatalf("expect %s but got %s", expect, result)
 	}
 }
 

kyaml/yaml/const.go

@@ -23,6 +23,7 @@ const (
 	KindField        = "kind"
 	MetadataField    = "metadata"
 	DataField        = "data"
+	BinaryDataField  = "binaryData"
 	NameField        = "name"
 	NamespaceField   = "namespace"
 	LabelsField      = "labels"

kyaml/yaml/kfns.go

@@ -4,10 +4,6 @@
 package yaml
 
 import (
-	"fmt"
-	"strings"
-	"unicode/utf8"
-
 	"gopkg.in/yaml.v3"
 	"sigs.k8s.io/kustomize/kyaml/errors"
 )
@@ -44,46 +40,6 @@ func ClearEmptyAnnotations(rn *RNode) error {
 	return nil
 }
 
-// k8sDataSetter place key value pairs in either a 'data' or 'binaryData' field.
-// Useful for creating ConfigMaps and Secrets.
-type k8sDataSetter struct {
-	Key             string `yaml:"key,omitempty"`
-	Value           string `yaml:"value,omitempty"`
-	ProtectExisting bool   `yaml:"protectExisting,omitempty"`
-}
-
-func (s k8sDataSetter) Filter(rn *RNode) (*RNode, error) {
-	if !utf8.Valid([]byte(s.Value)) {
-		// Core k8s ConfigMaps store k,v pairs with 'v' passing the above utf8
-		// test in a mapping field called "data" as a string. Pairs with a 'v'
-		// failing this test go into a field called binaryData as a []byte.
-		// TODO: support this distinction in kyaml with NodeTagBytes?
-		return nil, errors.Errorf(
-			"key '%s' appears to have non-utf8 data; "+
-				"binaryData field not yet supported", s.Key)
-	}
-	keyNode, err := rn.Pipe(Lookup(DataField, s.Key))
-	if err != nil {
-		return nil, err
-	}
-	if keyNode != nil && s.ProtectExisting {
-		return nil, fmt.Errorf(
-			"protecting existing %s='%s' against attempt to add new value '%s'",
-			s.Key, strings.TrimSpace(keyNode.MustString()), s.Value)
-	}
-	v := NewScalarRNode(s.Value)
-	v.YNode().Tag = NodeTagString
-	// TODO: use schema to determine node style and tag.
-	// FormatNonStringStyle(v.YNode(), *k8sSch)
-	_, err = rn.Pipe(
-		LookupCreate(yaml.MappingNode, DataField), SetField(s.Key, v))
-	return rn, err
-}
-
-func SetK8sData(key, value string) k8sDataSetter {
-	return k8sDataSetter{Key: key, Value: value, ProtectExisting: true}
-}
-
 // k8sMetaSetter sets a name at metadata.{key}.
 // Creates metadata if does not exist.
 type k8sMetaSetter struct {
@@ -92,11 +48,9 @@ type k8sMetaSetter struct {
 }
 
 func (s k8sMetaSetter) Filter(rn *RNode) (*RNode, error) {
-	v := NewScalarRNode(s.Value)
-	v.YNode().Tag = NodeTagString
 	_, err := rn.Pipe(
 		PathGetter{Path: []string{MetadataField}, Create: yaml.MappingNode},
-		FieldSetter{Name: s.Key, Value: v})
+		FieldSetter{Name: s.Key, Value: NewStringRNode(s.Value)})
 	return rn, err
 }
 
@@ -117,20 +71,13 @@ type AnnotationSetter struct {
 }
 
 func (s AnnotationSetter) Filter(rn *RNode) (*RNode, error) {
+	v := NewStringRNode(s.Value)
 	// some tools get confused about the type if annotations are not quoted
-	v := NewScalarRNode(s.Value)
-	v.YNode().Tag = NodeTagString
 	v.YNode().Style = yaml.SingleQuotedStyle
-
 	if err := ClearEmptyAnnotations(rn); err != nil {
 		return nil, err
 	}
-
-	return rn.Pipe(
-		PathGetter{
-			Path:   []string{MetadataField, AnnotationsField},
-			Create: yaml.MappingNode},
-		FieldSetter{Name: s.Key, Value: v})
+	return addMetadataNode(rn, AnnotationsField, s.Key, v)
 }
 
 func SetAnnotation(key, value string) AnnotationSetter {
@@ -172,14 +119,17 @@ type LabelSetter struct {
 }
 
 func (s LabelSetter) Filter(rn *RNode) (*RNode, error) {
+	v := NewStringRNode(s.Value)
 	// some tools get confused about the type if labels are not quoted
-	v := NewScalarRNode(s.Value)
-	v.YNode().Tag = NodeTagString
 	v.YNode().Style = yaml.SingleQuotedStyle
+	return addMetadataNode(rn, LabelsField, s.Key, v)
+}
+
+func addMetadataNode(rn *RNode, field, key string, v *RNode) (*RNode, error) {
 	return rn.Pipe(
 		PathGetter{
-			Path: []string{MetadataField, LabelsField}, Create: yaml.MappingNode},
-		FieldSetter{Name: s.Key, Value: v})
+			Path: []string{MetadataField, field}, Create: yaml.MappingNode},
+		FieldSetter{Name: key, Value: v})
 }
 
 func SetLabel(key, value string) LabelSetter {

kyaml/yaml/kfns_test.go

@@ -9,56 +9,6 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
-var input = `apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: the-map
-data:
-  altGreeting: "Good Morning!"
-  enableRisky: "false"
-`
-
-func TestSetK8sData(t *testing.T) {
-	rn := MustParse(`apiVersion: v1
-kind: ConfigMap
-data:
-  altGreeting: "Good Morning!"
-`)
-	_, err := rn.Pipe(
-		SetK8sData("foo", "bar"),
-		SetK8sData("fruit", "apple"),
-		SetK8sData("veggie", "celery"))
-	assert.NoError(t, err)
-	output := rn.MustString()
-
-	expected := `apiVersion: v1
-kind: ConfigMap
-data:
-  altGreeting: "Good Morning!"
-  foo: bar
-  fruit: apple
-  veggie: celery
-`
-	if !assert.Equal(t, expected, output) {
-		t.FailNow()
-	}
-}
-
-func TestSetK8sDataForbidOverwrite(t *testing.T) {
-	rn := MustParse(`apiVersion: v1
-kind: ConfigMap
-data:
-  altGreeting: "Good Morning!"
-`)
-	_, err := rn.Pipe(
-		SetK8sData("foo", "bar"),
-		SetK8sData("altGreeting", "hey"),
-		SetK8sData("veggie", "celery"))
-	assert.EqualError(
-		t, err, "protecting existing altGreeting='\"Good Morning!\"' "+
-			"against attempt to add new value 'hey'")
-}
-
 func TestSetMeta(t *testing.T) {
 	rn := MustParse(`apiVersion: v1
 kind: ConfigMap
@@ -85,7 +35,14 @@ metadata:
 }
 
 func TestSetLabel1(t *testing.T) {
-	rn := MustParse(input)
+	rn := MustParse(`apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: the-map
+data:
+  altGreeting: "Good Morning!"
+  enableRisky: "false"
+`)
 	_, err := rn.Pipe(SetLabel("foo", "bar"))
 	if err != nil {
 		t.Fatalf("unexpected error %v", err)
@@ -133,7 +90,14 @@ metadata:
 }
 
 func TestAnnotation(t *testing.T) {
-	rn := MustParse(input)
+	rn := MustParse(`apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: the-map
+data:
+  altGreeting: "Good Morning!"
+  enableRisky: "false"
+`)
 	_, err := rn.Pipe(SetAnnotation("foo", "bar"))
 	if err != nil {
 		t.Fatalf("unexpected error %v", err)

kyaml/yaml/rnode.go

@@ -100,6 +100,15 @@ func NewScalarRNode(value string) *RNode {
 		}}
 }
 
+// NewStringRNode returns a new Scalar *RNode containing the provided string.
+// If the string is non-utf8, it will be base64 encoded, and the tag
+// will indicate binary data.
+func NewStringRNode(value string) *RNode {
+	n := yaml.Node{Kind: yaml.ScalarNode}
+	n.SetString(value)
+	return NewRNode(&n)
+}
+
 // NewListRNode returns a new List *RNode containing the provided scalar values.
 func NewListRNode(values ...string) *RNode {
 	seq := &RNode{value: &yaml.Node{Kind: yaml.SequenceNode}}

kyaml/yaml/rnode_test.go

@@ -90,6 +90,31 @@ func TestRNodeHasNilEntryInList(t *testing.T) {
 	}
 }
 
+func TestRNodeNewStringRNodeText(t *testing.T) {
+	rn := NewStringRNode("cat")
+	if !assert.Equal(t, `cat
+`,
+		rn.MustString()) {
+		t.FailNow()
+	}
+}
+
+func TestRNodeNewStringRNodeBinary(t *testing.T) {
+	rn := NewStringRNode(string([]byte{
+		0xff, // non-utf8
+		0x68, // h
+		0x65, // e
+		0x6c, // l
+		0x6c, // l
+		0x6f, // o
+	}))
+	if !assert.Equal(t, `!!binary /2hlbGxv
+`,
+		rn.MustString()) {
+		t.FailNow()
+	}
+}
+
 func TestRNodeGetValidatedMetadata(t *testing.T) {
 	testConfigMap := map[string]interface{}{
 		"apiVersion": "v1",