From 6392e6629fe04b38e62e79914cfcc0c3fd794bdc Mon Sep 17 00:00:00 2001 From: Jingfang Liu Date: Thu, 21 Jun 2018 15:58:37 -0700 Subject: [PATCH] Add glob support --- pkg/fs/fakefs.go | 19 +++++++++ pkg/fs/fs.go | 1 + pkg/fs/realfs.go | 21 ++++++++++ pkg/fs/realfs_test.go | 58 +++++++++++++++++++++++++++ pkg/internal/loadertest/fakeloader.go | 5 +++ pkg/loader/fileloader.go | 10 +++++ pkg/loader/loader.go | 19 +++++++++ pkg/resmap/resmap.go | 24 ++++++----- 8 files changed, 147 insertions(+), 10 deletions(-) create mode 100644 pkg/fs/realfs_test.go diff --git a/pkg/fs/fakefs.go b/pkg/fs/fakefs.go index a3ca4ee01..572522edf 100644 --- a/pkg/fs/fakefs.go +++ b/pkg/fs/fakefs.go @@ -71,6 +71,18 @@ func (fs *FakeFS) ReadFile(name string) ([]byte, error) { return nil, fmt.Errorf("cannot read file %q", name) } +// ReadFiles looks through all files in the fake filesystem +// and find the matching files and then read content from all of them +func (fs *FakeFS) ReadFiles(name string) (map[string][]byte, error) { + result := map[string][]byte{} + for p, f := range fs.m { + if fs.pathMatch(p, name) { + result[p] = f.content + } + } + return result, nil +} + // WriteFile always succeeds and does nothing. func (fs *FakeFS) WriteFile(name string, c []byte) error { ff := &FakeFile{} @@ -78,3 +90,10 @@ func (fs *FakeFS) WriteFile(name string, c []byte) error { fs.m[name] = ff return nil } + +func (fs *FakeFS) pathMatch(path, pattern string) bool { + if path == pattern { + return true + } + return false +} diff --git a/pkg/fs/fs.go b/pkg/fs/fs.go index 788ea68ff..acc5e7d38 100644 --- a/pkg/fs/fs.go +++ b/pkg/fs/fs.go @@ -29,6 +29,7 @@ type FileSystem interface { Open(name string) (File, error) Stat(name string) (os.FileInfo, error) ReadFile(name string) ([]byte, error) + ReadFiles(name string) (map[string][]byte, error) WriteFile(name string, data []byte) error } diff --git a/pkg/fs/realfs.go b/pkg/fs/realfs.go index f2c3237b1..47fca063a 100644 --- a/pkg/fs/realfs.go +++ b/pkg/fs/realfs.go @@ -19,6 +19,7 @@ package fs import ( "io/ioutil" "os" + "path/filepath" ) var _ FileSystem = realFS{} @@ -46,6 +47,26 @@ func (realFS) Stat(name string) (os.FileInfo, error) { return os.Stat(name) } // ReadFile delegates to ioutil.ReadFile. func (realFS) ReadFile(name string) ([]byte, error) { return ioutil.ReadFile(name) } +// ReadFiles use glob to find the matching files and then read content from all of them +func (realFS) ReadFiles(name string) (map[string][]byte, error) { + files, err := filepath.Glob(name) + if err != nil || len(files) == 0 { + return nil, err + } + + output := map[string][]byte{} + for _, file := range files { + bytes, err := ioutil.ReadFile(file) + if err != nil { + return nil, err + } + if bytes != nil { + output[file] = bytes + } + } + return output, nil +} + // WriteFile delegates to ioutil.WriteFile with read/write permissions. func (realFS) WriteFile(name string, c []byte) error { return ioutil.WriteFile(name, c, 0666) diff --git a/pkg/fs/realfs_test.go b/pkg/fs/realfs_test.go new file mode 100644 index 000000000..b2c5e3e86 --- /dev/null +++ b/pkg/fs/realfs_test.go @@ -0,0 +1,58 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package fs + +import ( + "os" + "path" + "reflect" + "testing" +) + +func TestReadFilesRealFS(t *testing.T) { + x := MakeRealFS() + testDir := "kustomize_testing_dir" + err := x.Mkdir(testDir, 0777) + defer os.RemoveAll(testDir) + + if err != nil { + t.Fatalf("unexpected error %s", err) + } + + err = x.WriteFile(path.Join(testDir, "foo"), []byte(`foo`)) + if err != nil { + t.Fatalf("unexpected error %s", err) + } + err = x.WriteFile(path.Join(testDir, "bar"), []byte(`bar`)) + if err != nil { + t.Fatalf("unexpected error %s", err) + } + + expected := map[string][]byte{ + path.Join(testDir, "foo"): []byte(`foo`), + path.Join(testDir, "bar"): []byte(`bar`), + } + + content, err := x.ReadFiles("kustomize_testing_dir/*") + if !reflect.DeepEqual(content, expected) { + t.Fatalf("actual: %+v doesn't match expected: %+v", content, expected) + + } + if err != nil { + t.Fatalf("unexpected error %s", err) + } +} diff --git a/pkg/internal/loadertest/fakeloader.go b/pkg/internal/loadertest/fakeloader.go index 811c0a015..b4b016c83 100644 --- a/pkg/internal/loadertest/fakeloader.go +++ b/pkg/internal/loadertest/fakeloader.go @@ -67,3 +67,8 @@ func (f FakeLoader) New(newRoot string) (loader.Loader, error) { func (f FakeLoader) Load(location string) ([]byte, error) { return f.delegate.Load(location) } + +// GlobLoad performs load from a given location. +func (f FakeLoader) GlobLoad(location string) (map[string][]byte, error) { + return f.delegate.GlobLoad(location) +} diff --git a/pkg/loader/fileloader.go b/pkg/loader/fileloader.go index 4122e47f4..fbbaab76c 100644 --- a/pkg/loader/fileloader.go +++ b/pkg/loader/fileloader.go @@ -79,3 +79,13 @@ func (l *fileLoader) Load(fullFilePath string) ([]byte, error) { } return l.fs.ReadFile(fullFilePath) } + +// GlobLoad returns the map from path to bytes from reading a glob path. +// Implements the Loader interface. +func (l *fileLoader) GlobLoad(fullFilePath string) (map[string][]byte, error) { + // Validate path to load from is a full file path. + if !filepath.IsAbs(fullFilePath) { + return nil, fmt.Errorf("Attempting to load file without full file path: %s\n", fullFilePath) + } + return l.fs.ReadFiles(fullFilePath) +} diff --git a/pkg/loader/loader.go b/pkg/loader/loader.go index f4b87f5cc..38f9f6f33 100644 --- a/pkg/loader/loader.go +++ b/pkg/loader/loader.go @@ -28,6 +28,8 @@ type Loader interface { New(newRoot string) (Loader, error) // Load returns the bytes read from the location or an error. Load(location string) ([]byte, error) + // GlobLoad returns the bytes read from a glob path or an error. + GlobLoad(location string) (map[string][]byte, error) } // Private implmentation of Loader interface. @@ -44,6 +46,8 @@ type SchemeLoader interface { FullLocation(root string, path string) (string, error) // Load bytes at scheme-specific location or an error. Load(location string) ([]byte, error) + // GlobLoad returns the bytes read from a glob path or an error. + GlobLoad(location string) (map[string][]byte, error) } const emptyRoot = "" @@ -91,6 +95,21 @@ func (l *loaderImpl) Load(location string) ([]byte, error) { return scheme.Load(fullLocation) } +// GlobLoad returns a map from path to bytes read from scheme-specific location or an error. +// "location" can be an absolute path, or if relative, full location is +// calculated from the Root(). +func (l *loaderImpl) GlobLoad(location string) (map[string][]byte, error) { + scheme, err := l.getSchemeLoader(location) + if err != nil { + return nil, err + } + fullLocation, err := scheme.FullLocation(l.root, location) + if err != nil { + return nil, err + } + return scheme.GlobLoad(fullLocation) +} + // Helper function to parse scheme from location parameter. func (l *loaderImpl) getSchemeLoader(location string) (SchemeLoader, error) { for _, scheme := range l.schemes { diff --git a/pkg/resmap/resmap.go b/pkg/resmap/resmap.go index 9af59fc94..d54d29f5a 100644 --- a/pkg/resmap/resmap.go +++ b/pkg/resmap/resmap.go @@ -113,16 +113,18 @@ func NewResourceSliceFromPatches( loader loader.Loader, paths []string) ([]*resource.Resource, error) { result := []*resource.Resource{} for _, path := range paths { - content, err := loader.Load(path) + contents, err := loader.GlobLoad(path) if err != nil { return nil, err } + for p, content := range contents { + res, err := newResourceSliceFromBytes(content) + if err != nil { + return nil, internal.ErrorHandler(err, p) + } + result = append(result, res...) - res, err := newResourceSliceFromBytes(content) - if err != nil { - return nil, internal.ErrorHandler(err, path) } - result = append(result, res...) } return result, nil } @@ -131,15 +133,17 @@ func NewResourceSliceFromPatches( func NewResMapFromFiles(loader loader.Loader, paths []string) (ResMap, error) { result := []ResMap{} for _, path := range paths { - content, err := loader.Load(path) + contents, err := loader.GlobLoad(path) if err != nil { return nil, errors.Wrap(err, "Load from path "+path+" failed") } - res, err := newResMapFromBytes(content) - if err != nil { - return nil, internal.ErrorHandler(err, path) + for p, content := range contents { + res, err := newResMapFromBytes(content) + if err != nil { + return nil, internal.ErrorHandler(err, p) + } + result = append(result, res) } - result = append(result, res) } return MergeWithoutOverride(result...) }