Merge pull request #133 from Liujingfang1/loader

Add glob support

pkg/commands/testdata/testcase-base-only/in/kustomization.yaml

@@ -6,5 +6,4 @@ commonLabels:
 commonAnnotations:
   note: This is a test annotation
 resources:
-  - deployment.yaml
-  - service.yaml
+  - resources/*.yaml

pkg/commands/testdata/testcase-multiple-patches-noconflict/in/overlay/kustomization.yaml

@@ -2,8 +2,7 @@ namePrefix: staging-
 commonLabels:
   env: staging
 patches:
-  - deployment-patch1.yaml
-  - deployment-patch2.yaml
+  - patches/deployment-patch*.yaml
 bases:
   - ../package/
 configMapGenerator:

pkg/crds/crds_test.go

@@ -18,6 +18,7 @@ package crds
 
 import (
 	"reflect"
+	"sort"
 	"testing"
 
 	"github.com/kubernetes-sigs/kustomize/pkg/internal/loadertest"
@@ -150,30 +151,36 @@ func makeLoader(t *testing.T) loader.Loader {
 }
 
 func TestRegisterCRD(t *testing.T) {
+	refpathconfigs := []transformers.ReferencePathConfig{
+		transformers.NewReferencePathConfig(
+			schema.GroupVersionKind{Kind: "Bee", Version: "v1beta1"},
+			[]transformers.PathConfig{
+				{
+					CreateIfNotPresent: false,
+					GroupVersionKind:   &schema.GroupVersionKind{Kind: "MyKind"},
+					Path:               []string{"spec", "beeRef", "name"},
+				},
+			},
+		),
+		transformers.NewReferencePathConfig(
+			schema.GroupVersionKind{Kind: "Secret", Version: "v1"},
+			[]transformers.PathConfig{
+				{
+					CreateIfNotPresent: false,
+					GroupVersionKind:   &schema.GroupVersionKind{Kind: "MyKind"},
+					Path:               []string{"spec", "secretRef", "name"},
+				},
+			},
+		),
+	}
+
+	sort.Slice(refpathconfigs, func(i, j int) bool {
+		return refpathconfigs[i].GVK() < refpathconfigs[j].GVK()
+	})
+
 	expected := []pathConfigs{
 		{
-			namereferencePathConfigs: []transformers.ReferencePathConfig{
-				transformers.NewReferencePathConfig(
-					schema.GroupVersionKind{Kind: "Bee", Version: "v1beta1"},
-					[]transformers.PathConfig{
-						{
-							CreateIfNotPresent: false,
-							GroupVersionKind:   &schema.GroupVersionKind{Kind: "MyKind"},
-							Path:               []string{"spec", "beeRef", "name"},
-						},
-					},
-				),
-				transformers.NewReferencePathConfig(
-					schema.GroupVersionKind{Kind: "Secret", Version: "v1"},
-					[]transformers.PathConfig{
-						{
-							CreateIfNotPresent: false,
-							GroupVersionKind:   &schema.GroupVersionKind{Kind: "MyKind"},
-							Path:               []string{"spec", "secretRef", "name"},
-						},
-					},
-				),
-			},
+			namereferencePathConfigs: refpathconfigs,
 		},
 	}
 
@@ -181,6 +188,10 @@ func TestRegisterCRD(t *testing.T) {
 
 	pathconfig, _ := registerCRD(loader, "/testpath/crd.json")
 
+	sort.Slice(pathconfig[0].namereferencePathConfigs, func(i, j int) bool {
+		return pathconfig[0].namereferencePathConfigs[i].GVK() < pathconfig[0].namereferencePathConfigs[j].GVK()
+	})
+
 	if !reflect.DeepEqual(pathconfig, expected) {
 		t.Fatalf("expected\n %v\n but got\n %v\n", expected, pathconfig)
 	}

pkg/fs/fakefs.go

@@ -71,6 +71,18 @@ func (fs *FakeFS) ReadFile(name string) ([]byte, error) {
 	return nil, fmt.Errorf("cannot read file %q", name)
 }
 
+// ReadFiles looks through all files in the fake filesystem
+// and find the matching files and then read content from all of them
+func (fs *FakeFS) ReadFiles(name string) (map[string][]byte, error) {
+	result := map[string][]byte{}
+	for p, f := range fs.m {
+		if fs.pathMatch(p, name) {
+			result[p] = f.content
+		}
+	}
+	return result, nil
+}
+
 // WriteFile always succeeds and does nothing.
 func (fs *FakeFS) WriteFile(name string, c []byte) error {
 	ff := &FakeFile{}
@@ -78,3 +90,10 @@ func (fs *FakeFS) WriteFile(name string, c []byte) error {
 	fs.m[name] = ff
 	return nil
 }
+
+func (fs *FakeFS) pathMatch(path, pattern string) bool {
+	if path == pattern {
+		return true
+	}
+	return false
+}

pkg/fs/fs.go

@@ -29,6 +29,7 @@ type FileSystem interface {
 	Open(name string) (File, error)
 	Stat(name string) (os.FileInfo, error)
 	ReadFile(name string) ([]byte, error)
+	ReadFiles(name string) (map[string][]byte, error)
 	WriteFile(name string, data []byte) error
 }
 

pkg/fs/realfs.go

@@ -19,6 +19,7 @@ package fs
 import (
 	"io/ioutil"
 	"os"
+	"path/filepath"
 )
 
 var _ FileSystem = realFS{}
@@ -46,6 +47,26 @@ func (realFS) Stat(name string) (os.FileInfo, error) { return os.Stat(name) }
 // ReadFile delegates to ioutil.ReadFile.
 func (realFS) ReadFile(name string) ([]byte, error) { return ioutil.ReadFile(name) }
 
+// ReadFiles use glob to find the matching files and then read content from all of them
+func (realFS) ReadFiles(name string) (map[string][]byte, error) {
+	files, err := filepath.Glob(name)
+	if err != nil || len(files) == 0 {
+		return nil, err
+	}
+
+	output := map[string][]byte{}
+	for _, file := range files {
+		bytes, err := ioutil.ReadFile(file)
+		if err != nil {
+			return nil, err
+		}
+		if bytes != nil {
+			output[file] = bytes
+		}
+	}
+	return output, nil
+}
+
 // WriteFile delegates to ioutil.WriteFile with read/write permissions.
 func (realFS) WriteFile(name string, c []byte) error {
 	return ioutil.WriteFile(name, c, 0666)

pkg/fs/realfs_test.go

@@ -0,0 +1,58 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package fs
+
+import (
+	"os"
+	"path"
+	"reflect"
+	"testing"
+)
+
+func TestReadFilesRealFS(t *testing.T) {
+	x := MakeRealFS()
+	testDir := "kustomize_testing_dir"
+	err := x.Mkdir(testDir, 0777)
+	defer os.RemoveAll(testDir)
+
+	if err != nil {
+		t.Fatalf("unexpected error %s", err)
+	}
+
+	err = x.WriteFile(path.Join(testDir, "foo"), []byte(`foo`))
+	if err != nil {
+		t.Fatalf("unexpected error %s", err)
+	}
+	err = x.WriteFile(path.Join(testDir, "bar"), []byte(`bar`))
+	if err != nil {
+		t.Fatalf("unexpected error %s", err)
+	}
+
+	expected := map[string][]byte{
+		path.Join(testDir, "foo"): []byte(`foo`),
+		path.Join(testDir, "bar"): []byte(`bar`),
+	}
+
+	content, err := x.ReadFiles("kustomize_testing_dir/*")
+	if !reflect.DeepEqual(content, expected) {
+		t.Fatalf("actual: %+v doesn't match expected: %+v", content, expected)
+
+	}
+	if err != nil {
+		t.Fatalf("unexpected error %s", err)
+	}
+}

pkg/internal/loadertest/fakeloader.go

@@ -67,3 +67,8 @@ func (f FakeLoader) New(newRoot string) (loader.Loader, error) {
 func (f FakeLoader) Load(location string) ([]byte, error) {
 	return f.delegate.Load(location)
 }
+
+// GlobLoad performs load from a given location.
+func (f FakeLoader) GlobLoad(location string) (map[string][]byte, error) {
+	return f.delegate.GlobLoad(location)
+}

pkg/loader/fileloader.go

@@ -79,3 +79,13 @@ func (l *fileLoader) Load(fullFilePath string) ([]byte, error) {
 	}
 	return l.fs.ReadFile(fullFilePath)
 }
+
+// GlobLoad returns the map from path to bytes from reading a glob path.
+// Implements the Loader interface.
+func (l *fileLoader) GlobLoad(fullFilePath string) (map[string][]byte, error) {
+	// Validate path to load from is a full file path.
+	if !filepath.IsAbs(fullFilePath) {
+		return nil, fmt.Errorf("Attempting to load file without full file path: %s\n", fullFilePath)
+	}
+	return l.fs.ReadFiles(fullFilePath)
+}

pkg/loader/loader.go

@@ -28,6 +28,8 @@ type Loader interface {
 	New(newRoot string) (Loader, error)
 	// Load returns the bytes read from the location or an error.
 	Load(location string) ([]byte, error)
+	// GlobLoad returns the bytes read from a glob path or an error.
+	GlobLoad(location string) (map[string][]byte, error)
 }
 
 // Private implmentation of Loader interface.
@@ -44,6 +46,8 @@ type SchemeLoader interface {
 	FullLocation(root string, path string) (string, error)
 	// Load bytes at scheme-specific location or an error.
 	Load(location string) ([]byte, error)
+	// GlobLoad returns the bytes read from a glob path or an error.
+	GlobLoad(location string) (map[string][]byte, error)
 }
 
 const emptyRoot = ""
@@ -91,6 +95,21 @@ func (l *loaderImpl) Load(location string) ([]byte, error) {
 	return scheme.Load(fullLocation)
 }
 
+// GlobLoad returns a map from path to bytes read from scheme-specific location or an error.
+// "location" can be an absolute path, or if relative, full location is
+// calculated from the Root().
+func (l *loaderImpl) GlobLoad(location string) (map[string][]byte, error) {
+	scheme, err := l.getSchemeLoader(location)
+	if err != nil {
+		return nil, err
+	}
+	fullLocation, err := scheme.FullLocation(l.root, location)
+	if err != nil {
+		return nil, err
+	}
+	return scheme.GlobLoad(fullLocation)
+}
+
 // Helper function to parse scheme from location parameter.
 func (l *loaderImpl) getSchemeLoader(location string) (SchemeLoader, error) {
 	for _, scheme := range l.schemes {

pkg/resmap/resmap.go

@@ -113,16 +113,18 @@ func NewResourceSliceFromPatches(
 	loader loader.Loader, paths []string) ([]*resource.Resource, error) {
 	result := []*resource.Resource{}
 	for _, path := range paths {
-		content, err := loader.Load(path)
+		contents, err := loader.GlobLoad(path)
 		if err != nil {
 			return nil, err
 		}
+		for p, content := range contents {
+			res, err := newResourceSliceFromBytes(content)
+			if err != nil {
+				return nil, internal.ErrorHandler(err, p)
+			}
+			result = append(result, res...)
 
-		res, err := newResourceSliceFromBytes(content)
-		if err != nil {
-			return nil, internal.ErrorHandler(err, path)
 		}
-		result = append(result, res...)
 	}
 	return result, nil
 }
@@ -131,15 +133,17 @@ func NewResourceSliceFromPatches(
 func NewResMapFromFiles(loader loader.Loader, paths []string) (ResMap, error) {
 	result := []ResMap{}
 	for _, path := range paths {
-		content, err := loader.Load(path)
+		contents, err := loader.GlobLoad(path)
 		if err != nil {
 			return nil, errors.Wrap(err, "Load from path "+path+" failed")
 		}
-		res, err := newResMapFromBytes(content)
-		if err != nil {
-			return nil, internal.ErrorHandler(err, path)
+		for p, content := range contents {
+			res, err := newResMapFromBytes(content)
+			if err != nil {
+				return nil, internal.ErrorHandler(err, p)
+			}
+			result = append(result, res)
 		}
-		result = append(result, res)
 	}
 	return MergeWithoutOverride(result...)
 }

pkg/transformers/pathconfig.go

@@ -61,3 +61,8 @@ func NewReferencePathConfig(gvk schema.GroupVersionKind, pathconfigs []PathConfi
 		pathConfigs:   pathconfigs,
 	}
 }
+
+// GVK returns the Group version kind of a Reference PathConfig
+func (r ReferencePathConfig) GVK() string {
+	return r.referencedGVK.String()
+}