diff --git a/docs/kustomization.yaml b/docs/kustomization.yaml index 277122978..b717cb401 100644 --- a/docs/kustomization.yaml +++ b/docs/kustomization.yaml @@ -105,9 +105,9 @@ secretGenerator: - tls.key=secret/tls.key type: "kubernetes.io/tls" - name: env_file_secret - # env is a path to a file to read lines of key=val - # you can only specify one env file per secret. - env: env.txt + # paths to files with k=v pairs, one pair per line. + envs: + - env.txt type: Opaque # generatorOptions modify behavior of all ConfigMap and Secret generators diff --git a/k8sdeps/configmapandsecret/basefactory.go b/k8sdeps/configmapandsecret/basefactory.go index 8e0d767f1..5706a47c9 100644 --- a/k8sdeps/configmapandsecret/basefactory.go +++ b/k8sdeps/configmapandsecret/basefactory.go @@ -42,15 +42,14 @@ func (bf baseFactory) loadKvPairs( if err != nil { return nil, errors.Wrap(err, fmt.Sprintf( "plugins: %s", - args.EnvSource)) + args.KVSources)) } all = append(all, pairs...) - - pairs, err = bf.keyValuesFromEnvFile(args.EnvSource) + pairs, err = bf.keyValuesFromEnvFiles(args.EnvSources) if err != nil { return nil, errors.Wrap(err, fmt.Sprintf( - "env source file: %s", - args.EnvSource)) + "env source files: %v", + args.EnvSources)) } all = append(all, pairs...) @@ -136,13 +135,18 @@ func (bf baseFactory) keyValuesFromFileSources(sources []string) ([]kv.Pair, err return kvs, nil } -func (bf baseFactory) keyValuesFromEnvFile(path string) ([]kv.Pair, error) { - if path == "" { - return nil, nil +func (bf baseFactory) keyValuesFromEnvFiles(paths []string) ([]kv.Pair, error) { + var kvs []kv.Pair + for _, path := range paths { + content, err := bf.ldr.Load(path) + if err != nil { + return nil, err + } + more, err := kv.KeyValuesFromLines(content) + if err != nil { + return nil, err + } + kvs = append(kvs, more...) } - content, err := bf.ldr.Load(path) - if err != nil { - return nil, err - } - return kv.KeyValuesFromLines(content) + return kvs, nil } diff --git a/k8sdeps/configmapandsecret/configmapfactory_test.go b/k8sdeps/configmapandsecret/configmapfactory_test.go index 08d41b1fb..b13fb4588 100644 --- a/k8sdeps/configmapandsecret/configmapfactory_test.go +++ b/k8sdeps/configmapandsecret/configmapfactory_test.go @@ -99,7 +99,7 @@ func TestConstructConfigMap(t *testing.T) { GeneratorArgs: types.GeneratorArgs{ Name: "envConfigMap", DataSources: types.DataSources{ - EnvSource: "configmap/app.env", + EnvSources: []string{"configmap/app.env"}, }, }, }, diff --git a/k8sdeps/configmapandsecret/secretfactory_test.go b/k8sdeps/configmapandsecret/secretfactory_test.go index 281aaff40..703de4257 100644 --- a/k8sdeps/configmapandsecret/secretfactory_test.go +++ b/k8sdeps/configmapandsecret/secretfactory_test.go @@ -97,7 +97,7 @@ func TestConstructSecret(t *testing.T) { GeneratorArgs: types.GeneratorArgs{ Name: "envSecret", DataSources: types.DataSources{ - EnvSource: "secret/app.env", + EnvSources: []string{"secret/app.env"}, }, }, }, diff --git a/k8sdeps/kv/plugin/registry.go b/k8sdeps/kv/plugin/registry.go index 0c0274314..3a5cf6e56 100644 --- a/k8sdeps/kv/plugin/registry.go +++ b/k8sdeps/kv/plugin/registry.go @@ -32,7 +32,6 @@ type Registry struct { } const ( - PluginSymbol = "KustomizePlugin" PluginRoot = "plugin" pluginTypeGo = types.PluginType("go") pluginTypeBuiltIn = types.PluginType("builtin") diff --git a/pkg/commands/kustfile/kustomizationfile.go b/pkg/commands/kustfile/kustomizationfile.go index b06e37024..f6287eb4c 100644 --- a/pkg/commands/kustfile/kustomizationfile.go +++ b/pkg/commands/kustfile/kustomizationfile.go @@ -159,16 +159,13 @@ func (mf *kustomizationFile) Read() (*types.Kustomization, error) { if err != nil { return nil, err } - data = types.DealWithDeprecatedFields(data) + data = types.FixKustomizationPreUnmarshalling(data) var k types.Kustomization err = yaml.Unmarshal(data, &k) if err != nil { return nil, err } - msgs := k.DealWithMissingFields() - if len(msgs) > 0 { - log.Printf(strings.Join(msgs, "\n")) - } + k.FixKustomizationPostUnmarshalling() err = mf.parseCommentedFields(data) if err != nil { return nil, err diff --git a/pkg/commands/kustfile/kustomizationfile_test.go b/pkg/commands/kustfile/kustomizationfile_test.go index 7a1ab8b97..88a7f652c 100644 --- a/pkg/commands/kustfile/kustomizationfile_test.go +++ b/pkg/commands/kustfile/kustomizationfile_test.go @@ -81,7 +81,7 @@ func TestWriteAndRead(t *testing.T) { if err != nil { t.Fatalf("Couldn't read kustomization file: %v\n", err) } - kustomization.DealWithMissingFields() + kustomization.FixKustomizationPostUnmarshalling() if !reflect.DeepEqual(kustomization, content) { t.Fatal("Read kustomization is different from written kustomization") } diff --git a/pkg/plugins/config.go b/pkg/plugins/config.go new file mode 100644 index 000000000..8081a4d07 --- /dev/null +++ b/pkg/plugins/config.go @@ -0,0 +1,19 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package plugins + +const pluginSymbol = "KustomizePlugin" diff --git a/pkg/plugins/loader.go b/pkg/plugins/loader.go index 592e0524d..40068a7d1 100644 --- a/pkg/plugins/loader.go +++ b/pkg/plugins/loader.go @@ -22,7 +22,6 @@ import ( "plugin" "github.com/pkg/errors" - kplugin "sigs.k8s.io/kustomize/k8sdeps/kv/plugin" "sigs.k8s.io/kustomize/pkg/ifc" "sigs.k8s.io/kustomize/pkg/resid" "sigs.k8s.io/kustomize/pkg/resmap" @@ -122,11 +121,11 @@ func (l *Loader) loadGoPlugin(id resid.ResId) (c Configurable, err error) { if err != nil { return nil, errors.Wrapf(err, "plugin %s fails to load", name) } - symbol, err := p.Lookup(kplugin.PluginSymbol) + symbol, err := p.Lookup(pluginSymbol) if err != nil { return nil, errors.Wrapf( err, "plugin %s doesn't have symbol %s", - name, kplugin.PluginSymbol) + name, pluginSymbol) } c, ok = symbol.(Configurable) if !ok { diff --git a/pkg/resmap/factory_test.go b/pkg/resmap/factory_test.go index 5054df2b1..dc74412f8 100644 --- a/pkg/resmap/factory_test.go +++ b/pkg/resmap/factory_test.go @@ -153,7 +153,7 @@ func TestNewFromConfigMaps(t *testing.T) { GeneratorArgs: types.GeneratorArgs{ Name: "envConfigMap", DataSources: types.DataSources{ - EnvSource: "app.env", + EnvSources: []string{"app.env"}, }, }, }, diff --git a/pkg/target/baseandoverlaymedium_test.go b/pkg/target/baseandoverlaymedium_test.go index 6c409db28..ade613b22 100644 --- a/pkg/target/baseandoverlaymedium_test.go +++ b/pkg/target/baseandoverlaymedium_test.go @@ -158,21 +158,33 @@ patchesStrategicMerge: - deployment/deployment.yaml configMapGenerator: - name: app-env - env: configmap/app.env + env: configmap/db.env + envs: + - configmap/units.ini + - configmap/food.ini - name: app-config files: - - configmap/app-init.ini + - nonsense=configmap/dummy.txt images: - name: nginx newTag: 1.8.0`) - th.writeF("/app/overlay/configmap/app.env", ` + th.writeF("/app/overlay/configmap/db.env", ` DB_USERNAME=admin DB_PASSWORD=somepw `) - th.writeF("/app/overlay/configmap/app-init.ini", ` -FOO=bar -BAR=baz + th.writeF("/app/overlay/configmap/units.ini", ` +LENGTH=kilometer +ENERGY=electronvolt +`) + th.writeF("/app/overlay/configmap/food.ini", ` +FRUIT=banana +LEGUME=chickpea +`) + th.writeF("/app/overlay/configmap/dummy.txt", + `Lorem ipsum dolor sit amet, consectetur +adipiscing elit, sed do eiusmod tempor +incididunt ut labore et dolore magna aliqua. `) th.writeF("/app/overlay/deployment/deployment.yaml", ` apiVersion: extensions/v1beta1 @@ -217,10 +229,8 @@ spec: th.assertActualEqualsExpected(m, ` apiVersion: v1 data: - app-init.ini: |2 - - FOO=bar - BAR=baz + nonsense: "Lorem ipsum dolor sit amet, consectetur\nadipiscing elit, sed do eiusmod + tempor\nincididunt ut labore et dolore magna aliqua. \n" kind: ConfigMap metadata: annotations: @@ -229,12 +239,16 @@ metadata: app: mungebot org: kubernetes repo: test-infra - name: test-infra-app-config-fd62mfc87h + name: test-infra-app-config-f462h769f9 --- apiVersion: v1 data: DB_PASSWORD: somepw DB_USERNAME: admin + ENERGY: electronvolt + FRUIT: banana + LEGUME: chickpea + LENGTH: kilometer kind: ConfigMap metadata: annotations: @@ -243,7 +257,7 @@ metadata: app: mungebot org: kubernetes repo: test-infra - name: test-infra-app-env-bh449c299k + name: test-infra-app-env-ffmd9b969m --- apiVersion: v1 kind: Service @@ -303,7 +317,7 @@ spec: valueFrom: configMapKeyRef: key: somekey - name: test-infra-app-env-bh449c299k + name: test-infra-app-env-ffmd9b969m - name: foo value: bar image: nginx:1.8.0 @@ -314,7 +328,7 @@ spec: - configMapRef: name: someConfigMap - configMapRef: - name: test-infra-app-env-bh449c299k + name: test-infra-app-env-ffmd9b969m image: busybox name: busybox volumeMounts: @@ -322,7 +336,7 @@ spec: name: app-env volumes: - configMap: - name: test-infra-app-env-bh449c299k + name: test-infra-app-env-ffmd9b969m name: app-env `) } diff --git a/pkg/target/generatorplugin_test.go b/pkg/target/generatorplugin_test.go index 42810e910..657b06c60 100644 --- a/pkg/target/generatorplugin_test.go +++ b/pkg/target/generatorplugin_test.go @@ -118,13 +118,14 @@ generators: th.assertActualEqualsExpected(m, ` apiVersion: v1 data: + DB_PASSWORD: aWxvdmV5b3U= FRUIT: YXBwbGU= ROUTER_PASSWORD: YWRtaW4= VEGETABLE: Y2Fycm90 longsecret.txt: CkxvcmVtIGlwc3VtIGRvbG9yIHNpdCBhbWV0LApjb25zZWN0ZXR1ciBhZGlwaXNjaW5nIGVsaXQsCnNlZCBkbyBlaXVzbW9kIHRlbXBvciBpbmNpZGlkdW50CnV0IGxhYm9yZSBldCBkb2xvcmUgbWFnbmEgYWxpcXVhLgo= kind: Secret metadata: - name: -2kt2h55789 + name: -ktm999dkcc type: Opaque `) } diff --git a/pkg/target/kusttarget.go b/pkg/target/kusttarget.go index b7a5b6641..1f7e7eb67 100644 --- a/pkg/target/kusttarget.go +++ b/pkg/target/kusttarget.go @@ -57,12 +57,13 @@ func NewKustTarget( if err != nil { return nil, err } - content = types.DealWithDeprecatedFields(content) + content = types.FixKustomizationPreUnmarshalling(content) var k types.Kustomization err = unmarshal(content, &k) if err != nil { return nil, err } + k.FixKustomizationPostUnmarshalling() errs := k.EnforceFields() if len(errs) > 0 { return nil, fmt.Errorf( diff --git a/pkg/types/kustomization.go b/pkg/types/kustomization.go index a3d1b02fb..63b45e6f1 100644 --- a/pkg/types/kustomization.go +++ b/pkg/types/kustomization.go @@ -142,18 +142,32 @@ type Kustomization struct { Inventory *Inventory `json:"inventory,omitempty" yaml:"inventory:omitempty"` } -// DealWithMissingFields fills the missing fields -func (k *Kustomization) DealWithMissingFields() []string { - var msgs []string +// FixKustomizationPostUnmarshalling fixes things +// like empty fields that should not be empty, or +// moving content of deprecated fields to newer +// fields. +func (k *Kustomization) FixKustomizationPostUnmarshalling() { if k.APIVersion == "" { k.APIVersion = KustomizationVersion - msgs = append(msgs, "Fixed the missing field by adding apiVersion: "+KustomizationVersion) } if k.Kind == "" { k.Kind = KustomizationKind - msgs = append(msgs, "Fixed the missing field by adding kind: "+KustomizationKind) } - return msgs + // The EnvSource field is deprecated in favor of the list. + for i, g := range k.ConfigMapGenerator { + if g.EnvSource != "" { + k.ConfigMapGenerator[i].EnvSources = + append(g.EnvSources, g.EnvSource) + k.ConfigMapGenerator[i].EnvSource = "" + } + } + for i, g := range k.SecretGenerator { + if g.EnvSource != "" { + k.SecretGenerator[i].EnvSources = + append(g.EnvSources, g.EnvSource) + k.SecretGenerator[i].EnvSource = "" + } + } } func (k *Kustomization) EnforceFields() []string { @@ -167,9 +181,10 @@ func (k *Kustomization) EnforceFields() []string { return errs } -// DealWithDeprecatedFields should be called immediately after -// loading from storage. -func DealWithDeprecatedFields(data []byte) []byte { +// FixKustomizationPreUnmarshalling modies the raw data +// before marshalling - e.g. changes old field names to +// new field names. +func FixKustomizationPreUnmarshalling(data []byte) []byte { deprecateFieldsMap := map[string]string{ "patches:": "patchesStrategicMerge:", "imageTags:": "images:", @@ -246,24 +261,31 @@ type SecretArgs struct { // DataSources contains some generic sources for configmaps. type DataSources struct { - // LiteralSources is a list of literal sources. - // Each literal source should be a key and literal value, - // e.g. `somekey=somevalue` - // It will be similar to kubectl create configmap|secret --from-literal + // LiteralSources is a list of literal + // pair sources. Each literal source should + // be a key and literal value, e.g. `key=value` LiteralSources []string `json:"literals,omitempty" yaml:"literals,omitempty"` - // FileSources is a list of file sources. - // Each file source can be specified using its file path, in which case file - // basename will be used as configmap key, or optionally with a key and file - // path, in which case the given key will be used. - // Specifying a directory will iterate each named file in the directory - // whose basename is a valid configmap key. - // It will be similar to kubectl create configmap|secret --from-file + // FileSources is a list of file "sources" to + // use in creating a list of key, value pairs. + // A source takes the form: [{key}=]{path} + // If the "key=" part is missing, the key is the + // path's basename. If they "key=" part is present, + // it becomes the key (replacing the basename). + // In either case, the value is the file contents. + // Specifying a directory will iterate each named + // file in the directory whose basename is a + // valid configmap key. FileSources []string `json:"files,omitempty" yaml:"files,omitempty"` - // EnvSource format should be a path to a file to read lines of key=val - // pairs to create a configmap. - // i.e. a Docker .env file or a .ini file. + // EnvSources is a list of file paths. + // The contents of each file should be one + // key=value pair per line, e.g. a Docker + // or npm ".env" file or a ".ini" file + // (wikipedia.org/wiki/INI_file) + EnvSources []string `json:"envs,omitempty" yaml:"envs,omitempty"` + + // Deprecated. Use EnvSources instead. EnvSource string `json:"env,omitempty" yaml:"env,omitempty"` } diff --git a/plugin/builtin/SecretGenerator.go b/plugin/builtin/SecretGenerator.go index 500e24b9d..bf6028efa 100644 --- a/plugin/builtin/SecretGenerator.go +++ b/plugin/builtin/SecretGenerator.go @@ -3,8 +3,6 @@ package main import ( - "fmt" - "sigs.k8s.io/kustomize/pkg/ifc" "sigs.k8s.io/kustomize/pkg/resmap" "sigs.k8s.io/kustomize/pkg/types" @@ -23,26 +21,16 @@ func (p *plugin) Config( ldr ifc.Loader, rf *resmap.Factory, k ifc.Kunstructured) error { p.ldr = ldr p.rf = rf - var err error - // TODO: Should validate this. + // TODO: validate behavior values. p.args.Behavior, err = k.GetFieldValue("behavior") if err != nil { return err } - - envFiles, err := k.GetStringSlice("envFiles") + p.args.EnvSources, err = k.GetStringSlice("envFiles") if err != nil { return err } - if len(envFiles) > 2 { - // TODO: refactor to allow this - return fmt.Errorf("cannot yet accept more than one envFile") - } - if len(envFiles) > 0 { - p.args.EnvSource = envFiles[0] - } - p.args.FileSources, err = k.GetStringSlice("valueFiles") if err != nil { return err @@ -51,7 +39,6 @@ func (p *plugin) Config( if err != nil { return err } - return nil }