From f3b34c44b5902ab937a9362ac35fe0a1eb1adffa Mon Sep 17 00:00:00 2001 From: Yigit Demirbas Date: Mon, 18 Sep 2023 20:22:59 +0200 Subject: [PATCH 1/7] Add skip-validation flag to edit add resource cmd --- kustomize/commands/create/create.go | 2 +- kustomize/commands/edit/add/addcomponent.go | 2 +- kustomize/commands/edit/add/addresource.go | 6 ++- kustomize/commands/internal/util/util.go | 37 +++++++++++-------- kustomize/commands/internal/util/util_test.go | 15 ++++++-- 5 files changed, 41 insertions(+), 21 deletions(-) diff --git a/kustomize/commands/create/create.go b/kustomize/commands/create/create.go index 4b42b740b..0f6fa8b14 100644 --- a/kustomize/commands/create/create.go +++ b/kustomize/commands/create/create.go @@ -99,7 +99,7 @@ func runCreate(opts createFlags, fSys filesys.FileSystem, rf *resource.Factory) var resources []string var err error if opts.resources != "" { - resources, err = util.GlobPatternsWithLoader(fSys, loader.NewFileLoaderAtCwd(fSys), strings.Split(opts.resources, ",")) + resources, err = util.GlobPatternsWithLoader(fSys, loader.NewFileLoaderAtCwd(fSys), strings.Split(opts.resources, ","), false) if err != nil { return err } diff --git a/kustomize/commands/edit/add/addcomponent.go b/kustomize/commands/edit/add/addcomponent.go index 946b38ae9..9746d9300 100644 --- a/kustomize/commands/edit/add/addcomponent.go +++ b/kustomize/commands/edit/add/addcomponent.go @@ -49,7 +49,7 @@ func (o *addComponentOptions) Validate(args []string) error { // RunAddComponent runs addComponent command (do real work). func (o *addComponentOptions) RunAddComponent(fSys filesys.FileSystem) error { - components, err := util.GlobPatternsWithLoader(fSys, loader.NewFileLoaderAtCwd(fSys), o.componentFilePaths) + components, err := util.GlobPatternsWithLoader(fSys, loader.NewFileLoaderAtCwd(fSys), o.componentFilePaths, false) if err != nil { return err } diff --git a/kustomize/commands/edit/add/addresource.go b/kustomize/commands/edit/add/addresource.go index 98e1adb28..0a1829d91 100644 --- a/kustomize/commands/edit/add/addresource.go +++ b/kustomize/commands/edit/add/addresource.go @@ -16,6 +16,7 @@ import ( type addResourceOptions struct { resourceFilePaths []string + skipValidation bool } // newCmdAddResource adds the name of a file containing a resource to the kustomization file. @@ -35,6 +36,9 @@ func newCmdAddResource(fSys filesys.FileSystem) *cobra.Command { return o.RunAddResource(fSys) }, } + cmd.Flags().BoolVar(&o.skipValidation, "skip-validation", false, + "skip validation for resources", + ) return cmd } @@ -49,7 +53,7 @@ func (o *addResourceOptions) Validate(args []string) error { // RunAddResource runs addResource command (do real work). func (o *addResourceOptions) RunAddResource(fSys filesys.FileSystem) error { - resources, err := util.GlobPatternsWithLoader(fSys, loader.NewFileLoaderAtCwd(fSys), o.resourceFilePaths) + resources, err := util.GlobPatternsWithLoader(fSys, loader.NewFileLoaderAtCwd(fSys), o.resourceFilePaths, o.skipValidation) if err != nil { return err } diff --git a/kustomize/commands/internal/util/util.go b/kustomize/commands/internal/util/util.go index 94004e997..01c21473d 100644 --- a/kustomize/commands/internal/util/util.go +++ b/kustomize/commands/internal/util/util.go @@ -30,28 +30,35 @@ func GlobPatterns(fSys filesys.FileSystem, patterns []string) ([]string, error) return result, nil } -// GlobPatterns accepts a slice of glob strings and returns the set of matching file paths. If files are not found, will try load from remote. -// It returns an error if there are no matching files or it can't load from remote. -func GlobPatternsWithLoader(fSys filesys.FileSystem, ldr ifc.Loader, patterns []string) ([]string, error) { +// GlobPatterns accepts a slice of glob strings and returns the set of matching file paths. +// If validation is skipped, then it will return the patterns as provided. +// Otherwise, It will try to load the files from the filesystem. +// If files are not found in the filesystem, it will try to load from remote. +// It returns an error if validation is not skipped and there are no matching files or it can't load from remote. +func GlobPatternsWithLoader(fSys filesys.FileSystem, ldr ifc.Loader, patterns []string, skipValidation bool) ([]string, error) { var result []string for _, pattern := range patterns { - files, err := fSys.Glob(pattern) - if err != nil { - return nil, err - } - if len(files) == 0 { - loader, err := ldr.New(pattern) + if skipValidation { + result = append(result, pattern) + } else { + files, err := fSys.Glob(pattern) if err != nil { - return nil, fmt.Errorf("%s has no match: %w", pattern, err) + return nil, err + } + if len(files) != 0 { + result = append(result, files...) } else { - result = append(result, pattern) - if loader != nil { - loader.Cleanup() + loader, err := ldr.New(pattern) + if err != nil { + return nil, fmt.Errorf("%s has no match: %w", pattern, err) + } else { + result = append(result, pattern) + if loader != nil { + loader.Cleanup() + } } } - continue } - result = append(result, files...) } return result, nil } diff --git a/kustomize/commands/internal/util/util_test.go b/kustomize/commands/internal/util/util_test.go index 631e1145f..5e6a57e4c 100644 --- a/kustomize/commands/internal/util/util_test.go +++ b/kustomize/commands/internal/util/util_test.go @@ -71,7 +71,7 @@ func TestGlobPatternsWithLoaderRemoteFile(t *testing.T) { } // test load remote file - resources, err := GlobPatternsWithLoader(fSys, ldr, []string{httpPath}) + resources, err := GlobPatternsWithLoader(fSys, ldr, []string{httpPath}, false) if err != nil { t.Fatalf("unexpected load error: %v", err) } @@ -80,7 +80,7 @@ func TestGlobPatternsWithLoaderRemoteFile(t *testing.T) { } // test load local and remote file - resources, err = GlobPatternsWithLoader(fSys, ldr, []string{httpPath, "/test.yml"}) + resources, err = GlobPatternsWithLoader(fSys, ldr, []string{httpPath, "/test.yml"}, false) if err != nil { t.Fatalf("unexpected load error: %v", err) } @@ -90,7 +90,7 @@ func TestGlobPatternsWithLoaderRemoteFile(t *testing.T) { // test load invalid file invalidURL := "http://invalid" - resources, err = GlobPatternsWithLoader(fSys, ldr, []string{invalidURL}) + resources, err = GlobPatternsWithLoader(fSys, ldr, []string{invalidURL}, false) if err == nil { t.Fatalf("expected error but did not receive one") } else if err.Error() != invalidURL+" has no match: "+invalidURL+" not exist" { @@ -99,6 +99,15 @@ func TestGlobPatternsWithLoaderRemoteFile(t *testing.T) { if len(resources) > 0 { t.Fatalf("incorrect resources: %v", resources) } + + // test load unreachable remote file with skipped verification + resources, err = GlobPatternsWithLoader(fSys, ldr, []string{invalidURL}, true) + if err != nil { + t.Fatalf("unexpected load error: %v", err) + } + if len(resources) != 1 || resources[0] != invalidURL { + t.Fatalf("incorrect resources: %v", resources) + } } type fakeLoader struct { From df0cd3c4a3f752f672d91291b0857ab41d9d3b01 Mon Sep 17 00:00:00 2001 From: Yigit Demirbas Date: Mon, 18 Sep 2023 20:54:56 +0200 Subject: [PATCH 2/7] modified to fix linter issues --- kustomize/commands/internal/util/util.go | 39 +++++++++++++----------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/kustomize/commands/internal/util/util.go b/kustomize/commands/internal/util/util.go index 01c21473d..b2f7f8716 100644 --- a/kustomize/commands/internal/util/util.go +++ b/kustomize/commands/internal/util/util.go @@ -40,23 +40,28 @@ func GlobPatternsWithLoader(fSys filesys.FileSystem, ldr ifc.Loader, patterns [] for _, pattern := range patterns { if skipValidation { result = append(result, pattern) - } else { - files, err := fSys.Glob(pattern) - if err != nil { - return nil, err - } - if len(files) != 0 { - result = append(result, files...) - } else { - loader, err := ldr.New(pattern) - if err != nil { - return nil, fmt.Errorf("%s has no match: %w", pattern, err) - } else { - result = append(result, pattern) - if loader != nil { - loader.Cleanup() - } - } + continue + } + + files, err := fSys.Glob(pattern) + if err != nil { + return nil, fmt.Errorf("error on checking the filesystem: %w", err) + } + + if len(files) != 0 { + result = append(result, files...) + continue + } + + loader, err := ldr.New(pattern) + if err != nil { + return nil, fmt.Errorf("%s has no match: %w", pattern, err) + } + + result = append(result, pattern) + if loader != nil { + if err = loader.Cleanup(); err != nil { + return nil, fmt.Errorf("error on cleaning up loader: %w", err) } } } From 039b05fb166f781ab5cb3e959bc1efea59fc5592 Mon Sep 17 00:00:00 2001 From: Yigit Demirbas Date: Wed, 27 Sep 2023 12:40:05 +0200 Subject: [PATCH 3/7] refactor tests to use require instead --- kustomize/commands/internal/util/util.go | 6 +- kustomize/commands/internal/util/util_test.go | 64 ++++++------------- 2 files changed, 23 insertions(+), 47 deletions(-) diff --git a/kustomize/commands/internal/util/util.go b/kustomize/commands/internal/util/util.go index b2f7f8716..d13b6a529 100644 --- a/kustomize/commands/internal/util/util.go +++ b/kustomize/commands/internal/util/util.go @@ -45,7 +45,7 @@ func GlobPatternsWithLoader(fSys filesys.FileSystem, ldr ifc.Loader, patterns [] files, err := fSys.Glob(pattern) if err != nil { - return nil, fmt.Errorf("error on checking the filesystem: %w", err) + return nil, fmt.Errorf("error checking the filesystem: %w", err) } if len(files) != 0 { @@ -55,13 +55,13 @@ func GlobPatternsWithLoader(fSys filesys.FileSystem, ldr ifc.Loader, patterns [] loader, err := ldr.New(pattern) if err != nil { - return nil, fmt.Errorf("%s has no match: %w", pattern, err) + return nil, fmt.Errorf("pattern %s has no match: %w", pattern, err) } result = append(result, pattern) if loader != nil { if err = loader.Cleanup(); err != nil { - return nil, fmt.Errorf("error on cleaning up loader: %w", err) + return nil, fmt.Errorf("error cleaning up loader: %w", err) } } } diff --git a/kustomize/commands/internal/util/util_test.go b/kustomize/commands/internal/util/util_test.go index 5e6a57e4c..11038b3e0 100644 --- a/kustomize/commands/internal/util/util_test.go +++ b/kustomize/commands/internal/util/util_test.go @@ -8,6 +8,7 @@ import ( "reflect" "testing" + "github.com/stretchr/testify/require" "sigs.k8s.io/kustomize/api/ifc" "sigs.k8s.io/kustomize/kyaml/filesys" ) @@ -21,26 +22,18 @@ func TestConvertToMap(t *testing.T) { expected["g"] = "h:k" result, err := ConvertToMap(args, "annotation") - if err != nil { - t.Errorf("unexpected error: %v", err.Error()) - } + require.NoError(t, err, "unexpected error") eq := reflect.DeepEqual(expected, result) - if !eq { - t.Errorf("Converted map does not match expected, expected: %v, result: %v\n", expected, result) - } + require.True(t, eq, "Converted map does not match expected") } func TestConvertToMapError(t *testing.T) { args := "a:b,c:\"d\",:f:g" _, err := ConvertToMap(args, "annotation") - if err == nil { - t.Errorf("expected an error") - } - if err.Error() != "invalid annotation: ':f:g' (need k:v pair where v may be quoted)" { - t.Errorf("incorrect error: %v", err.Error()) - } + require.Error(t, err, "expected error but did not receive one") + require.Equal(t, "invalid annotation: ':f:g' (need k:v pair where v may be quoted)", err.Error(), "incorrect error") } func TestConvertSliceToMap(t *testing.T) { @@ -52,14 +45,10 @@ func TestConvertSliceToMap(t *testing.T) { expected["g"] = "h:k" result, err := ConvertSliceToMap(args, "annotation") - if err != nil { - t.Errorf("unexpected error: %v", err.Error()) - } + require.NoError(t, err, "unexpected error") eq := reflect.DeepEqual(expected, result) - if !eq { - t.Errorf("Converted map does not match expected, expected: %v, result: %v\n", expected, result) - } + require.True(t, eq, "Converted map does not match expected") } func TestGlobPatternsWithLoaderRemoteFile(t *testing.T) { @@ -72,42 +61,29 @@ func TestGlobPatternsWithLoaderRemoteFile(t *testing.T) { // test load remote file resources, err := GlobPatternsWithLoader(fSys, ldr, []string{httpPath}, false) - if err != nil { - t.Fatalf("unexpected load error: %v", err) - } - if len(resources) != 1 || resources[0] != httpPath { - t.Fatalf("incorrect resources: %v", resources) - } + require.NoError(t, err, "unexpected load error") + require.Equal(t, 1, len(resources), "incorrect resources") + require.Equal(t, httpPath, resources[0], "incorrect resources") // test load local and remote file resources, err = GlobPatternsWithLoader(fSys, ldr, []string{httpPath, "/test.yml"}, false) - if err != nil { - t.Fatalf("unexpected load error: %v", err) - } - if len(resources) != 2 || resources[0] != httpPath || resources[1] != "/test.yml" { - t.Fatalf("incorrect resources: %v", resources) - } + require.NoError(t, err, "unexpected load error") + require.Equal(t, 2, len(resources), "incorrect resources") + require.Equal(t, httpPath, resources[0], "incorrect resources") + require.Equal(t, "/test.yml", resources[1], "incorrect resources") // test load invalid file invalidURL := "http://invalid" resources, err = GlobPatternsWithLoader(fSys, ldr, []string{invalidURL}, false) - if err == nil { - t.Fatalf("expected error but did not receive one") - } else if err.Error() != invalidURL+" has no match: "+invalidURL+" not exist" { - t.Fatalf("unexpected load error: %v", err) - } - if len(resources) > 0 { - t.Fatalf("incorrect resources: %v", resources) - } + require.Error(t, err, "expected error but did not receive one") + require.Equal(t, "pattern "+invalidURL+" has no match: "+invalidURL+" not exist", err.Error(), "unexpected load error") + require.Equal(t, 0, len(resources), "incorrect resources") // test load unreachable remote file with skipped verification resources, err = GlobPatternsWithLoader(fSys, ldr, []string{invalidURL}, true) - if err != nil { - t.Fatalf("unexpected load error: %v", err) - } - if len(resources) != 1 || resources[0] != invalidURL { - t.Fatalf("incorrect resources: %v", resources) - } + require.NoError(t, err, "unexpected load error") + require.Equal(t, 1, len(resources), "incorrect resources") + require.Equal(t, invalidURL, resources[0], "incorrect resources") } type fakeLoader struct { From cb5b24171556287d0519961fd20dfecc2659b80c Mon Sep 17 00:00:00 2001 From: Yigit Demirbas Date: Wed, 27 Sep 2023 12:49:19 +0200 Subject: [PATCH 4/7] modify add resource test to accomodate changes --- kustomize/commands/edit/add/addresource_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kustomize/commands/edit/add/addresource_test.go b/kustomize/commands/edit/add/addresource_test.go index d35f16262..928c89cfc 100644 --- a/kustomize/commands/edit/add/addresource_test.go +++ b/kustomize/commands/edit/add/addresource_test.go @@ -103,5 +103,5 @@ func TestAddResourceFileNotFound(t *testing.T) { args := []string{resourceFileName} err := cmd.RunE(cmd, args) - assert.EqualError(t, err, resourceFileName+" has no match: must build at directory: not a valid directory: '"+resourceFileName+"' doesn't exist") + assert.EqualError(t, err, "pattern "+resourceFileName+" has no match: must build at directory: not a valid directory: '"+resourceFileName+"' doesn't exist") } From 0d854a5144f24ddf58f0f4f6357cbf03a570313c Mon Sep 17 00:00:00 2001 From: Yigit Demirbas Date: Wed, 27 Sep 2023 12:52:09 +0200 Subject: [PATCH 5/7] modify error message to original --- kustomize/commands/edit/add/addresource_test.go | 2 +- kustomize/commands/internal/util/util.go | 2 +- kustomize/commands/internal/util/util_test.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/kustomize/commands/edit/add/addresource_test.go b/kustomize/commands/edit/add/addresource_test.go index 928c89cfc..d35f16262 100644 --- a/kustomize/commands/edit/add/addresource_test.go +++ b/kustomize/commands/edit/add/addresource_test.go @@ -103,5 +103,5 @@ func TestAddResourceFileNotFound(t *testing.T) { args := []string{resourceFileName} err := cmd.RunE(cmd, args) - assert.EqualError(t, err, "pattern "+resourceFileName+" has no match: must build at directory: not a valid directory: '"+resourceFileName+"' doesn't exist") + assert.EqualError(t, err, resourceFileName+" has no match: must build at directory: not a valid directory: '"+resourceFileName+"' doesn't exist") } diff --git a/kustomize/commands/internal/util/util.go b/kustomize/commands/internal/util/util.go index d13b6a529..aa42e3213 100644 --- a/kustomize/commands/internal/util/util.go +++ b/kustomize/commands/internal/util/util.go @@ -55,7 +55,7 @@ func GlobPatternsWithLoader(fSys filesys.FileSystem, ldr ifc.Loader, patterns [] loader, err := ldr.New(pattern) if err != nil { - return nil, fmt.Errorf("pattern %s has no match: %w", pattern, err) + return nil, fmt.Errorf("%s has no match: %w", pattern, err) } result = append(result, pattern) diff --git a/kustomize/commands/internal/util/util_test.go b/kustomize/commands/internal/util/util_test.go index 11038b3e0..16fae2660 100644 --- a/kustomize/commands/internal/util/util_test.go +++ b/kustomize/commands/internal/util/util_test.go @@ -76,7 +76,7 @@ func TestGlobPatternsWithLoaderRemoteFile(t *testing.T) { invalidURL := "http://invalid" resources, err = GlobPatternsWithLoader(fSys, ldr, []string{invalidURL}, false) require.Error(t, err, "expected error but did not receive one") - require.Equal(t, "pattern "+invalidURL+" has no match: "+invalidURL+" not exist", err.Error(), "unexpected load error") + require.Equal(t, invalidURL+" has no match: "+invalidURL+" not exist", err.Error(), "unexpected load error") require.Equal(t, 0, len(resources), "incorrect resources") // test load unreachable remote file with skipped verification From 5936a892a762cba834f8dfb6d78c588584b3cd09 Mon Sep 17 00:00:00 2001 From: Yigit Demirbas Date: Mon, 2 Oct 2023 09:06:07 +0200 Subject: [PATCH 6/7] fix imports after rebase --- kustomize/commands/create/create.go | 2 +- kustomize/commands/edit/add/addresource.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/kustomize/commands/create/create.go b/kustomize/commands/create/create.go index 28532d8bc..81a05dbab 100644 --- a/kustomize/commands/create/create.go +++ b/kustomize/commands/create/create.go @@ -99,7 +99,7 @@ func runCreate(opts createFlags, fSys filesys.FileSystem, rf *resource.Factory) var resources []string var err error if opts.resources != "" { - resources, err = util.GlobPatternsWithLoader(fSys, loader.NewFileLoaderAtCwd(fSys), strings.Split(opts.resources, ","), false) + resources, err = util.GlobPatternsWithLoader(fSys, ldrhelper.NewFileLoaderAtCwd(fSys), strings.Split(opts.resources, ","), false) if err != nil { return err } diff --git a/kustomize/commands/edit/add/addresource.go b/kustomize/commands/edit/add/addresource.go index 928454a48..c8354a4fc 100644 --- a/kustomize/commands/edit/add/addresource.go +++ b/kustomize/commands/edit/add/addresource.go @@ -53,7 +53,7 @@ func (o *addResourceOptions) Validate(args []string) error { // RunAddResource runs addResource command (do real work). func (o *addResourceOptions) RunAddResource(fSys filesys.FileSystem) error { - resources, err := util.GlobPatternsWithLoader(fSys, loader.NewFileLoaderAtCwd(fSys), o.resourceFilePaths, o.skipValidation) + resources, err := util.GlobPatternsWithLoader(fSys, ldrhelper.NewFileLoaderAtCwd(fSys), o.resourceFilePaths, o.skipValidation) if err != nil { return err } From 713842330e6c0fb26a63f30241d0f7cc4b7f9f2c Mon Sep 17 00:00:00 2001 From: Yigit Demirbas Date: Tue, 3 Oct 2023 09:02:34 +0200 Subject: [PATCH 7/7] rename flag to --no-verify --- kustomize/commands/edit/add/addresource.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/kustomize/commands/edit/add/addresource.go b/kustomize/commands/edit/add/addresource.go index c8354a4fc..529f2b119 100644 --- a/kustomize/commands/edit/add/addresource.go +++ b/kustomize/commands/edit/add/addresource.go @@ -16,7 +16,7 @@ import ( type addResourceOptions struct { resourceFilePaths []string - skipValidation bool + noVerify bool } // newCmdAddResource adds the name of a file containing a resource to the kustomization file. @@ -36,7 +36,7 @@ func newCmdAddResource(fSys filesys.FileSystem) *cobra.Command { return o.RunAddResource(fSys) }, } - cmd.Flags().BoolVar(&o.skipValidation, "skip-validation", false, + cmd.Flags().BoolVar(&o.noVerify, "no-verify", false, "skip validation for resources", ) return cmd @@ -53,7 +53,7 @@ func (o *addResourceOptions) Validate(args []string) error { // RunAddResource runs addResource command (do real work). func (o *addResourceOptions) RunAddResource(fSys filesys.FileSystem) error { - resources, err := util.GlobPatternsWithLoader(fSys, ldrhelper.NewFileLoaderAtCwd(fSys), o.resourceFilePaths, o.skipValidation) + resources, err := util.GlobPatternsWithLoader(fSys, ldrhelper.NewFileLoaderAtCwd(fSys), o.resourceFilePaths, o.noVerify) if err != nil { return err }