Merge pull request #4615 from annasong20/add-remote-resource-tests

Fix Issue #4588

api/internal/builtins/AnnotationsTransformer.go

@@ -1,8 +1,6 @@
 // Code generated by pluginator on AnnotationsTransformer; DO NOT EDIT.
 // pluginator {unknown  1970-01-01T00:00:00Z  }
 
-
-
 package builtins
 
 import (
@@ -18,7 +16,6 @@ type AnnotationsTransformerPlugin struct {
 	FieldSpecs  []types.FieldSpec `json:"fieldSpecs,omitempty" yaml:"fieldSpecs,omitempty"`
 }
 
-
 func (p *AnnotationsTransformerPlugin) Config(
 	_ *resmap.PluginHelpers, c []byte) (err error) {
 	p.Annotations = nil
@@ -37,5 +34,5 @@ func (p *AnnotationsTransformerPlugin) Transform(m resmap.ResMap) error {
 }
 
 func NewAnnotationsTransformerPlugin() resmap.TransformerPlugin {
-  return &AnnotationsTransformerPlugin{}
+	return &AnnotationsTransformerPlugin{}
 }

api/krusty/remoteload_test.go

@@ -9,76 +9,20 @@ import (
 	"path/filepath"
 	"testing"
 
-	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
-	"sigs.k8s.io/kustomize/api/internal/utils"
 	"sigs.k8s.io/kustomize/api/krusty"
 	"sigs.k8s.io/kustomize/api/loader"
+	"sigs.k8s.io/kustomize/api/resmap"
 	"sigs.k8s.io/kustomize/kyaml/filesys"
 )
 
-func TestRemoteLoad(t *testing.T) {
+const resourcesField = `resources:
-	fSys := filesys.MakeFsOnDisk()
+- %s`
-	b := krusty.MakeKustomizer(krusty.MakeDefaultOptions())
+const resourceErrorFormat = "accumulating resources: accumulation err='accumulating resources from '%s': "
-	m, err := b.Run(
+const fileError = "evalsymlink failure"
-		fSys,
+const repoFindError = "URL is a git repository"
-		"github.com/kubernetes-sigs/kustomize/examples/multibases/dev/?ref=v1.0.6")
-	if utils.IsErrTimeout(err) {
-		// Don't fail on timeouts.
-		t.SkipNow()
-	}
-	if !assert.NoError(t, err) {
-		t.FailNow()
-	}
-	yml, err := m.AsYaml()
-	assert.NoError(t, err)
-	assert.Equal(t, `apiVersion: v1
-kind: Pod
-metadata:
-  labels:
-    app: myapp
-  name: dev-myapp-pod
-spec:
-  containers:
-  - image: nginx:1.7.9
-    name: nginx
-`, string(yml))
-}
 
-func TestRemoteResource(t *testing.T) {
+const multibaseDevExampleBuild = `apiVersion: v1
-	fSys := filesys.MakeFsOnDisk()
-	b := krusty.MakeKustomizer(krusty.MakeDefaultOptions())
-	tmpDir, err := filesys.NewTmpConfirmedDir()
-	assert.NoError(t, err)
-	assert.NoError(t, fSys.WriteFile(filepath.Join(tmpDir.String(), "kustomization.yaml"), []byte(`
-resources:
-- github.com/kubernetes-sigs/kustomize/examples/multibases/dev/?ref=v1.0.6
-`)))
-	m, err := b.Run(
-		fSys,
-		tmpDir.String())
-	if utils.IsErrTimeout(err) {
-		// Don't fail on timeouts.
-		t.SkipNow()
-	}
-	assert.NoError(t, err)
-	yml, err := m.AsYaml()
-	assert.NoError(t, err)
-	assert.Equal(t, `apiVersion: v1
-kind: Pod
-metadata:
-  labels:
-    app: myapp
-  name: dev-myapp-pod
-spec:
-  containers:
-  - image: nginx:1.7.9
-    name: nginx
-`, string(yml))
-	assert.NoError(t, fSys.RemoveAll(tmpDir.String()))
-}
-
-func TestRemoteResourceGitHTTP(t *testing.T) {
-	output := `apiVersion: v1
 kind: Pod
 metadata:
   labels:
@@ -89,100 +33,319 @@ spec:
   - image: nginx:1.7.9
     name: nginx
 `
-	tests := []struct {
+
-		input []byte
+type remoteResourceCase struct {
-	}{
+	skip          bool
-		{
+	kustomization string
-			input: []byte(`
+	error         bool
+	expected      string // if error, expected is error string
+}
+
+func createKustDir(content string, require *require.Assertions) (filesys.FileSystem, filesys.ConfirmedDir) {
+	fSys := filesys.MakeFsOnDisk()
+	tmpDir, err := filesys.NewTmpConfirmedDir()
+	require.NoError(err)
+	require.NoError(fSys.WriteFile(filepath.Join(tmpDir.String(), "kustomization.yaml"), []byte(content)))
+	return fSys, tmpDir
+}
+
+func checkYaml(actual resmap.ResMap, expected string, require *require.Assertions) {
+	yml, err := actual.AsYaml()
+	require.NoError(err)
+	require.Equal(expected, string(yml))
+}
+
+func testRemoteResource(require *require.Assertions, test *remoteResourceCase) {
+	fSys, tmpDir := createKustDir(test.kustomization, require)
+
+	b := krusty.MakeKustomizer(krusty.MakeDefaultOptions())
+	m, err := b.Run(
+		fSys,
+		tmpDir.String())
+
+	if test.error {
+		require.Error(err)
+		require.Contains(err.Error(), test.expected)
+	} else {
+		require.NoError(err)
+		checkYaml(m, test.expected, require)
+	}
+
+	require.NoError(fSys.RemoveAll(tmpDir.String()))
+}
+
+func TestRemoteLoad(t *testing.T) {
+	require := require.New(t)
+
+	fSys := filesys.MakeFsOnDisk()
+	b := krusty.MakeKustomizer(krusty.MakeDefaultOptions())
+
+	m, err := b.Run(
+		fSys,
+		"github.com/kubernetes-sigs/kustomize/examples/multibases/dev/?ref=v1.0.6")
+	require.NoError(err)
+	checkYaml(m, multibaseDevExampleBuild, require)
+}
+
+func TestRemoteResourceHttps(t *testing.T) {
+	tests := map[string]remoteResourceCase{
+		"basic": {
+			kustomization: `
 resources:
-- https://github.com/kubernetes-sigs/kustomize/examples/multibases/dev/?ref=v1.0.6
+- https://github.com/kubernetes-sigs/kustomize//examples/multibases/dev/?ref=v1.0.6`,
-`),
+			expected: multibaseDevExampleBuild,
 		},
-		{
+		".git repo suffix, no slash suffix": {
-			input: []byte(`
+			kustomization: `
 resources:
-- https://github.com/kubernetes-sigs/kustomize//examples/multibases/dev/?ref=v1.0.6
+- https://github.com/kubernetes-sigs/kustomize.git//examples/multibases/dev?ref=v1.0.6`,
-`),
+			expected: multibaseDevExampleBuild,
 		},
-		{
+		"repo": {
-			input: []byte(`
+			kustomization: `
 resources:
-- git::https://github.com/kubernetes-sigs/kustomize/examples/multibases/dev/?ref=v1.0.6
+- https://github.com/annasong20/kustomize-test.git?ref=main`,
-`),
+			expected: multibaseDevExampleBuild,
 		},
-		{
+		"raw remote file": {
-			input: []byte(`
+			kustomization: `
 resources:
-- git::https://github.com/kubernetes-sigs/kustomize//examples/multibases/dev/?ref=v1.0.6
+- https://raw.githubusercontent.com/kubernetes-sigs/kustomize/v3.1.0/examples/multibases/base/pod.yaml
-`),
+namePrefix: dev-`,
+			expected: multibaseDevExampleBuild,
 		},
 	}
 
-	for _, test := range tests {
+	for name, test := range tests {
-		fSys := filesys.MakeFsOnDisk()
+		test := test
-		b := krusty.MakeKustomizer(krusty.MakeDefaultOptions())
+		t.Run(name, func(t *testing.T) {
-		tmpDir, err := filesys.NewTmpConfirmedDir()
+			if test.skip {
-		assert.NoError(t, err)
+				t.SkipNow()
-		assert.NoError(t, fSys.WriteFile(filepath.Join(tmpDir.String(), "kustomization.yaml"), test.input))
+			}
-		m, err := b.Run(fSys, tmpDir.String())
+			testRemoteResource(require.New(t), &test)
-		if utils.IsErrTimeout(err) {
+		})
-			// Don't fail on timeouts.
-			t.SkipNow()
-		}
-		assert.NoError(t, err)
-		yml, err := m.AsYaml()
-		assert.NoError(t, err)
-		assert.Equal(t, output, string(yml))
-		assert.NoError(t, fSys.RemoveAll(tmpDir.String()))
 	}
 }
 
-func TestRemoteResourceWithHTTPError(t *testing.T) {
+func TestRemoteResourceSsh(t *testing.T) {
-	fSys := filesys.MakeFsOnDisk()
+	// skip all tests until server has ssh keys
-	b := krusty.MakeKustomizer(krusty.MakeDefaultOptions())
+	tests := map[string]remoteResourceCase{
-	tmpDir, err := filesys.NewTmpConfirmedDir()
+		"scp shorthand": {
-	assert.NoError(t, err)
+			skip: true,
+			kustomization: `
+resources:
+- git@github.com:kubernetes-sigs/kustomize//examples/multibases/dev/?ref=v1.0.6`,
+			expected: multibaseDevExampleBuild,
+		},
+		"full ssh, no ending slash": {
+			skip: true,
+			kustomization: `
+resources:
+- ssh://git@github.com/kubernetes-sigs/kustomize//examples/multibases/dev?ref=v1.0.6`,
+			expected: multibaseDevExampleBuild,
+		},
+		"repo": {
+			skip: true,
+			kustomization: `
+resources:
+- ssh://git@github.com/annasong20/kustomize-test.git?ref=main`,
+			expected: multibaseDevExampleBuild,
+		},
+	}
+
+	for name, test := range tests {
+		test := test
+		t.Run(name, func(t *testing.T) {
+			if test.skip {
+				t.SkipNow()
+			}
+			testRemoteResource(require.New(t), &test)
+		})
+	}
+}
+
+func TestRemoteResourcePort(t *testing.T) {
+	sshURL := "ssh://git@github.com:22/kubernetes-sigs/kustomize//examples/multibases/dev/?ref=v1.0.6"
+	httpsURL := "https://github.com:443/kubernetes-sigs/kustomize//examples/multibases/dev/?ref=v1.0.6"
+
+	// ports not currently supported; should implement in future
+	tests := map[string]remoteResourceCase{
+		"ssh": {
+			skip:          true,
+			kustomization: fmt.Sprintf(resourcesField, sshURL),
+			error:         true,
+			expected:      fmt.Sprintf(resourceErrorFormat+fileError, sshURL),
+		},
+		"https": {
+			kustomization: fmt.Sprintf(resourcesField, httpsURL),
+			error:         true,
+			expected:      fmt.Sprintf(resourceErrorFormat+repoFindError, httpsURL),
+		},
+	}
+	for name, test := range tests {
+		test := test
+		t.Run(name, func(t *testing.T) {
+			if test.skip {
+				t.SkipNow()
+			}
+			testRemoteResource(require.New(t), &test)
+		})
+	}
+}
+
+func TestRemoteResourceRepo(t *testing.T) {
+	tests := map[string]remoteResourceCase{
+		"https, no ref": {
+			kustomization: `
+resources:
+- https://github.com/annasong20/kustomize-test.git`,
+			expected: multibaseDevExampleBuild,
+		},
+		"ssh, no ref": {
+			skip: true,
+			kustomization: `
+resources:
+- git@github.com:annasong20/kustomize-test.git`,
+			expected: multibaseDevExampleBuild,
+		},
+	}
+
+	for name, test := range tests {
+		test := test
+		t.Run(name, func(t *testing.T) {
+			if test.skip {
+				t.SkipNow()
+			}
+			testRemoteResource(require.New(t), &test)
+		})
+	}
+}
+
+func TestRemoteResourceParameters(t *testing.T) {
+	httpsNoParam := "https://github.com/kubernetes-sigs/kustomize//examples/multibases/dev/"
+	httpsMasterBranch := "https://github.com/kubernetes-sigs/kustomize//examples/multibases/dev?ref=master"
+	sshNoParams := "git@github.com:kubernetes-sigs/kustomize//examples/multibases/dev"
+
+	// some query parameter combinations not currently supported; should implement in future
+	tests := map[string]remoteResourceCase{
+		"https no params": {
+			kustomization: fmt.Sprintf(resourcesField, httpsNoParam),
+			error:         true,
+			expected:      fmt.Sprintf(resourceErrorFormat+repoFindError, httpsNoParam),
+		},
+		"https master": {
+			kustomization: fmt.Sprintf(resourcesField, httpsMasterBranch),
+			error:         true,
+			expected:      fmt.Sprintf(resourceErrorFormat+repoFindError, httpsMasterBranch),
+		},
+		"https master and no submodules": {
+			kustomization: `
+resources:
+- https://github.com/kubernetes-sigs/kustomize//examples/multibases/dev?ref=master&submodules=false`,
+			expected: multibaseDevExampleBuild,
+		},
+		"https all params": {
+			kustomization: `
+resources:
+- https://github.com/kubernetes-sigs/kustomize//examples/multibases/dev?ref=v1.0.6&timeout=10&submodules=true`,
+			expected: multibaseDevExampleBuild,
+		},
+		"ssh no params": {
+			skip:          true,
+			kustomization: fmt.Sprintf(resourcesField, sshNoParams),
+			error:         true,
+			expected:      fmt.Sprintf(resourceErrorFormat+fileError, sshNoParams),
+		},
+		"ssh all params": {
+			skip: true,
+			kustomization: `
+resources:
+- ssh://git@github.com/annasong20/kustomize-test.git?ref=main&timeout=10&submodules=true`,
+			expected: multibaseDevExampleBuild,
+		},
+	}
+
+	for name, test := range tests {
+		test := test
+		t.Run(name, func(t *testing.T) {
+			if test.skip {
+				t.SkipNow()
+			}
+			testRemoteResource(require.New(t), &test)
+		})
+	}
+}
+
+func TestRemoteResourceGoGetter(t *testing.T) {
+	tests := map[string]remoteResourceCase{
+		"git detector with / subdirectory separator": {
+			kustomization: `
+resources:
+- github.com/kubernetes-sigs/kustomize/examples/multibases/dev/?ref=v1.0.6`,
+			expected: multibaseDevExampleBuild,
+		},
+		"git detector for repo": {
+			kustomization: `
+resources:
+- github.com/annasong20/kustomize-test`,
+			expected: multibaseDevExampleBuild,
+		},
+		"https with / subdirectory separator": {
+			kustomization: `
+resources:
+- https://github.com/kubernetes-sigs/kustomize/examples/multibases/dev/?ref=v1.0.6`,
+			expected: multibaseDevExampleBuild,
+		},
+		"git forced protocol": {
+			kustomization: `
+resources:
+- git::https://github.com/kubernetes-sigs/kustomize//examples/multibases/dev/?ref=v1.0.6`,
+			expected: multibaseDevExampleBuild,
+		},
+		"git forced protocol with / subdirectory separator": {
+			kustomization: `
+resources:
+- git::https://github.com/kubernetes-sigs/kustomize/examples/multibases/dev/?ref=v1.0.6`,
+			expected: multibaseDevExampleBuild,
+		},
+	}
+
+	for name, test := range tests {
+		test := test
+		t.Run(name, func(t *testing.T) {
+			if test.skip {
+				t.SkipNow()
+			}
+			testRemoteResource(require.New(t), &test)
+		})
+	}
+}
+
+func TestRemoteResourceWithHttpError(t *testing.T) {
+	require := require.New(t)
 
 	url404 := "https://github.com/thisisa404.yaml"
-	kusto := filepath.Join(tmpDir.String(), "kustomization.yaml")
+	fSys, tmpDir := createKustDir(fmt.Sprintf(resourcesField, url404), require)
-	assert.NoError(t, fSys.WriteFile(kusto, []byte(fmt.Sprintf(`
+	b := krusty.MakeKustomizer(krusty.MakeDefaultOptions())
-resources:
-- %s
-`, url404))))
 
-	_, err = b.Run(fSys, tmpDir.String())
+	_, err := b.Run(fSys, tmpDir.String())
-	if utils.IsErrTimeout(err) {
-		// Don't fail on timeouts.
-		t.SkipNow()
-	}
 
 	httpErr := fmt.Errorf("%w: status code %d (%s)", loader.ErrorHTTP, 404, http.StatusText(404))
 	accuFromErr := fmt.Errorf("accumulating resources from '%s': %w", url404, httpErr)
 	expectedErr := fmt.Errorf("accumulating resources: %w", accuFromErr)
-	assert.EqualErrorf(t, err, expectedErr.Error(), url404)
+	require.EqualErrorf(err, expectedErr.Error(), url404)
+
+	require.NoError(fSys.RemoveAll(tmpDir.String()))
 }
 
 func TestRemoteResourceAnnoOrigin(t *testing.T) {
-	fSys := filesys.MakeFsOnDisk()
+	test := remoteResourceCase{
-	b := krusty.MakeKustomizer(krusty.MakeDefaultOptions())
+		kustomization: `
-	tmpDir, err := filesys.NewTmpConfirmedDir()
-	assert.NoError(t, err)
-	assert.NoError(t, fSys.WriteFile(filepath.Join(tmpDir.String(), "kustomization.yaml"), []byte(`
 resources:
 - github.com/kubernetes-sigs/kustomize/examples/multibases/dev/?ref=v1.0.6
 buildMetadata: [originAnnotations]
-`)))
+`,
-	m, err := b.Run(
+		expected: `apiVersion: v1
-		fSys,
-		tmpDir.String())
-	if utils.IsErrTimeout(err) {
-		// Don't fail on timeouts.
-		t.SkipNow()
-	}
-	assert.NoError(t, err)
-	yml, err := m.AsYaml()
-	assert.NoError(t, err)
-	assert.Equal(t, `apiVersion: v1
 kind: Pod
 metadata:
   annotations:
@@ -197,24 +360,28 @@ spec:
   containers:
   - image: nginx:1.7.9
     name: nginx
-`, string(yml))
+`,
-	assert.NoError(t, fSys.RemoveAll(tmpDir.String()))
+	}
+
+	testRemoteResource(require.New(t), &test)
 }
 
 func TestRemoteResourceAsBaseWithAnnoOrigin(t *testing.T) {
+	require := require.New(t)
+
 	fSys := filesys.MakeFsOnDisk()
 	b := krusty.MakeKustomizer(krusty.MakeDefaultOptions())
 	tmpDir, err := filesys.NewTmpConfirmedDir()
-	assert.NoError(t, err)
+	require.NoError(err)
 	base := filepath.Join(tmpDir.String(), "base")
 	prod := filepath.Join(tmpDir.String(), "prod")
-	assert.NoError(t, fSys.Mkdir(base))
+	require.NoError(fSys.Mkdir(base))
-	assert.NoError(t, fSys.Mkdir(prod))
+	require.NoError(fSys.Mkdir(prod))
-	assert.NoError(t, fSys.WriteFile(filepath.Join(base, "kustomization.yaml"), []byte(`
+	require.NoError(fSys.WriteFile(filepath.Join(base, "kustomization.yaml"), []byte(`
 resources:
 - github.com/kubernetes-sigs/kustomize/examples/multibases/dev/?ref=v1.0.6
 `)))
-	assert.NoError(t, fSys.WriteFile(filepath.Join(prod, "kustomization.yaml"), []byte(`
+	require.NoError(fSys.WriteFile(filepath.Join(prod, "kustomization.yaml"), []byte(`
 resources:
 - ../base
 namePrefix: prefix-
@@ -224,14 +391,9 @@ buildMetadata: [originAnnotations]
 	m, err := b.Run(
 		fSys,
 		prod)
-	if utils.IsErrTimeout(err) {
+	require.NoError(err)
-		// Don't fail on timeouts.
+
-		t.SkipNow()
+	expected := `apiVersion: v1
-	}
-	assert.NoError(t, err)
-	yml, err := m.AsYaml()
-	assert.NoError(t, err)
-	assert.Equal(t, `apiVersion: v1
 kind: Pod
 metadata:
   annotations:
@@ -246,6 +408,8 @@ spec:
   containers:
   - image: nginx:1.7.9
     name: nginx
-`, string(yml))
+`
-	assert.NoError(t, fSys.RemoveAll(tmpDir.String()))
+	checkYaml(m, expected, require)
+
+	require.NoError(fSys.RemoveAll(tmpDir.String()))
 }