kubernetes-sigs/kustomize
1
0
Fork 0
mirror of https://github.com/kubernetes-sigs/kustomize.git synced 2026-05-17 10:15:22 +00:00
Code Issues Packages Projects Releases 1 Wiki Activity

Rewrite remoteload_test integration tests (#4783)

Browse Source 
* Better error message when git clone fails

* support file:// URLs

* rewrite remoteload_test

* lint and test fix

* fixes for kverey's comments

* document file:// remote load

* replace assert with require where appropriate

* add tests for file:// without git suffix

* fixes plus pr review from natasha

* fixes for review, lint

* revert changes to error handling

* fix skipped test
This commit is contained in:
Yunchi Luo
2022-09-19 12:13:18 -04:00
committed by GitHub
parent d6e40a3f6c
commit e62480d11c
16 changed files with 633 additions and 450 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
api/internal/git/cloner.go
View File

@@ -22,15 +22,11 @@ func ClonerUsingGitExec(repoSpec *RepoSpec) error {
if err = r.run("init"); err != nil {
return err
}
if err = r.run(
"remote", "add", "origin", repoSpec.CloneSpec()); err != nil {
return err
}
ref := "HEAD"
if repoSpec.Ref != "" {
ref = repoSpec.Ref
}
if err = r.run("fetch", "--depth=1", "origin", ref); err != nil {
if err = r.run("fetch", "--depth=1", repoSpec.CloneSpec(), ref); err != nil {
return err
}
if err = r.run("checkout", "FETCH_HEAD"); err != nil {

4
api/internal/git/gitrunner.go
View File

@@ -46,9 +46,9 @@ func (r gitRunner) run(args ...string) error {
cmd.String(),
r.duration,
func() error {
_, err := cmd.CombinedOutput()
out, err := cmd.CombinedOutput()
if err != nil {
return errors.Wrapf(err, "git cmd = '%s'", cmd.String())
return errors.Wrapf(err, "failed to run '%s': %s", cmd.String(), string(out))
}
return err
})

19
api/internal/git/repospec.go
View File

@@ -119,8 +119,12 @@ func parseGitURL(n string) (
return
}
host, n = parseHostSpec(n)
isLocal := strings.HasPrefix(host, "file://")
if !isLocal {
gitSuff = gitSuffix
}
if strings.Contains(n, gitSuffix) {
gitSuff = gitSuffix
index := strings.Index(n, gitSuffix)
orgRepo = n[0:index]
n = n[index+len(gitSuffix):]
@@ -131,6 +135,19 @@ func parseGitURL(n string) (
return
}
if isLocal {
if idx := strings.Index(n, "//"); idx > 0 {
orgRepo = n[:idx]
n = n[idx+2:]
path, gitRef, gitTimeout, gitSubmodules = peelQuery(n)
return
}
path, gitRef, gitTimeout, gitSubmodules = peelQuery(n)
orgRepo = path
path = ""
return
}
i := strings.Index(n, "/")
if i < 1 {
path, gitRef, gitTimeout, gitSubmodules = peelQuery(n)
@@ -200,7 +217,7 @@ func parseHostSpec(n string) (string, string) {
// Start accumulating the host part.
for _, p := range []string{
// Order matters here.
"git::", "gh:", "ssh://", "https://", "http://",
"git::", "gh:", "ssh://", "https://", "http://", "file://",
"git@", "github.com:", "github.com/"} {
if len(p) < len(n) && strings.ToLower(n[:len(p)]) == p {
n = n[len(p):]

128
api/internal/git/repospec_test.go
View File

@@ -11,6 +11,7 @@ import (
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestNewRepoSpecFromUrl_Permute(t *testing.T) {
@@ -61,6 +62,7 @@ func TestNewRepoSpecFromUrl_Permute(t *testing.T) {
rs, err := NewRepoSpecFromURL(uri)
if err != nil {
t.Errorf("problem %v", err)
continue
}
if rs.Host != hostSpec {
bad = append(bad, []string{"host", uri, rs.Host, hostSpec})
@@ -120,6 +122,7 @@ func TestNewRepoSpecFromUrl_Smoke(t *testing.T) {
repoSpec RepoSpec
cloneSpec string
absPath string
skip string
}{
{
name: "t1",
@@ -285,11 +288,134 @@ func TestNewRepoSpecFromUrl_Smoke(t *testing.T) {
GitSuffix: ".git",
},
},
{
name: "t15",
input: "https://github.com/kubernetes-sigs/kustomize//examples/multibases/dev/?ref=v1.0.6",
cloneSpec: "https://github.com/kubernetes-sigs/kustomize.git",
absPath: notCloned.Join("/examples/multibases/dev"),
repoSpec: RepoSpec{
Host: "https://github.com/",
OrgRepo: "kubernetes-sigs/kustomize",
Path: "/examples/multibases/dev/",
Ref: "v1.0.6",
GitSuffix: ".git",
},
},
{
name: "t16",
input: "file://a/b/c/someRepo.git/somepath?ref=someBranch",
cloneSpec: "file://a/b/c/someRepo.git",
absPath: notCloned.Join("somepath"),
repoSpec: RepoSpec{
Host: "file://",
OrgRepo: "a/b/c/someRepo",
Path: "somepath",
Ref: "someBranch",
GitSuffix: ".git",
},
},
{
name: "t17",
input: "file://a/b/c/someRepo//somepath?ref=someBranch",
cloneSpec: "file://a/b/c/someRepo",
absPath: notCloned.Join("somepath"),
repoSpec: RepoSpec{
Host: "file://",
OrgRepo: "a/b/c/someRepo",
Path: "somepath",
Ref: "someBranch",
},
},
{
name: "t18",
input: "file://a/b/c/someRepo?ref=someBranch",
cloneSpec: "file://a/b/c/someRepo",
absPath: notCloned.String(),
repoSpec: RepoSpec{
Host: "file://",
OrgRepo: "a/b/c/someRepo",
Ref: "someBranch",
},
},
{
name: "t19",
input: "file:///a/b/c/someRepo?ref=someBranch",
cloneSpec: "file:///a/b/c/someRepo",
absPath: notCloned.String(),
repoSpec: RepoSpec{
Host: "file://",
OrgRepo: "/a/b/c/someRepo",
Ref: "someBranch",
},
},
{
name: "t20",
input: "ssh://git@github.com/kubernetes-sigs/kustomize//examples/multibases/dev?ref=v1.0.6",
cloneSpec: "git@github.com:kubernetes-sigs/kustomize.git",
absPath: notCloned.Join("examples/multibases/dev"),
repoSpec: RepoSpec{
Host: "git@github.com:",
OrgRepo: "kubernetes-sigs/kustomize",
Path: "/examples/multibases/dev",
Ref: "v1.0.6",
GitSuffix: ".git",
},
},
{
name: "t21",
input: "file:///a/b/c/someRepo",
cloneSpec: "file:///a/b/c/someRepo",
absPath: notCloned.String(),
repoSpec: RepoSpec{
Host: "file://",
OrgRepo: "/a/b/c/someRepo",
},
},
{
name: "t22",
input: "file:///",
cloneSpec: "file:///",
absPath: notCloned.String(),
repoSpec: RepoSpec{
Host: "file://",
OrgRepo: "/",
},
},
{
name: "t23",
skip: "the `//` repo separator does not work",
input: "https://fake-git-hosting.org/path/to/repo//examples/multibases/dev",
cloneSpec: "https://fake-git-hosting.org/path/to.git",
absPath: notCloned.Join("/examples/multibases/dev"),
repoSpec: RepoSpec{
Host: "https://fake-git-hosting.org/",
OrgRepo: "path/to/repo",
Path: "/examples/multibases/dev",
GitSuffix: ".git",
},
},
{
name: "t24",
skip: "the `//` repo separator does not work",
input: "ssh://alice@acme.co/path/to/repo//examples/multibases/dev",
cloneSpec: "ssh://alice@acme.co/path/to/repo.git",
absPath: notCloned.Join("/examples/multibases/dev"),
repoSpec: RepoSpec{
Host: "ssh://alice@acme.co",
OrgRepo: "path/to/repo",
Path: "/examples/multibases/dev",
GitSuffix: ".git",
},
},
}
for _, tc := range testcases {
t.Run(tc.name, func(t *testing.T) {
if tc.skip != "" {
t.Skip(tc.skip)
}
rs, err := NewRepoSpecFromURL(tc.input)
assert.NoError(t, err)
require.NoError(t, err)
assert.Equal(t, tc.cloneSpec, rs.CloneSpec(), "cloneSpec mismatch")
assert.Equal(t, tc.absPath, rs.AbsPath(), "absPath mismatch")
// some values have defaults. Clear them here so test cases remain compact.

2
api/krusty/originannotation_test.go
View File

@@ -817,7 +817,7 @@ buildMetadata: [originAnnotations]
metadata:
annotations:
config.kubernetes.io/origin: |
repo: https://github.com/kubernetes-sigs/kustomize
repo: https://github.com/kubernetes-sigs/kustomize.git
ref: v1.0.6
configuredIn: examples/ldap/base/kustomization.yaml
configuredBy:

434
api/krusty/remoteload_test.go
View File

@@ -1,434 +0,0 @@
// Copyright 2019 The Kubernetes Authors.
// SPDX-License-Identifier: Apache-2.0
package krusty_test
import (
"bytes"
"encoding/base64"
"fmt"
"io"
"net/http"
"os"
"path/filepath"
"strconv"
"strings"
"testing"
"github.com/stretchr/testify/require"
"sigs.k8s.io/kustomize/api/krusty"
"sigs.k8s.io/kustomize/api/loader"
"sigs.k8s.io/kustomize/api/resmap"
"sigs.k8s.io/kustomize/kyaml/filesys"
"sigs.k8s.io/kustomize/kyaml/yaml"
)
const resourcesField = `resources:
- %s`
const resourceErrorFormat = "accumulating resources: accumulation err='accumulating resources from '%s': "
const fileError = "evalsymlink failure"
const repoFindError = "URL is a git repository"
const multibaseDevExampleBuild = `apiVersion: v1
kind: Pod
metadata:
labels:
app: myapp
name: dev-myapp-pod
spec:
containers:
- image: nginx:1.7.9
name: nginx
`
type remoteResourceCase struct {
local bool // only run locally; doesn't behave as expected on server
kustomization string
error bool
expected string // if error, expected is error string
}
func createKustDir(content string, require *require.Assertions) (filesys.FileSystem, filesys.ConfirmedDir) {
fSys := filesys.MakeFsOnDisk()
tmpDir, err := filesys.NewTmpConfirmedDir()
require.NoError(err)
require.NoError(fSys.WriteFile(filepath.Join(tmpDir.String(), "kustomization.yaml"), []byte(content)))
return fSys, tmpDir
}
func checkYaml(actual resmap.ResMap, expected string, require *require.Assertions) {
yml, err := actual.AsYaml()
require.NoError(err)
require.Equal(expected, string(yml))
}
func testRemoteResource(require *require.Assertions, test *remoteResourceCase) {
fSys, tmpDir := createKustDir(test.kustomization, require)
b := krusty.MakeKustomizer(krusty.MakeDefaultOptions())
m, err := b.Run(
fSys,
tmpDir.String())
if test.error {
require.Error(err)
require.Contains(err.Error(), test.expected)
} else {
require.NoError(err)
checkYaml(m, test.expected, require)
}
require.NoError(fSys.RemoveAll(tmpDir.String()))
}
func isLocalEnv(require *require.Assertions) bool {
// make variable that determines whether to run local-only tests
if value, exists := os.LookupEnv("IS_LOCAL"); exists {
isLocal, err := strconv.ParseBool(strings.TrimSpace(value))
require.NoError(err)
return isLocal
}
return false
}
func runResourceTests(t *testing.T, cases map[string]*remoteResourceCase) {
t.Helper()
req := require.New(t)
for name, test := range cases {
savedTest := test // test assignment changes; need assignment in this scope (iteration) of range
t.Run(name, func(t *testing.T) {
if savedTest.local && !isLocalEnv(req) {
t.SkipNow()
}
configureGitSSHCommand(t)
testRemoteResource(req, test)
})
}
}
func configureGitSSHCommand(t *testing.T) {
t.Helper()
// This contains a read-only Deploy Key for the kustomize repo.
node, err := yaml.ReadFile("testdata/repo_read_only_ssh_key.yaml")
require.NoError(t, err)
keyB64, err := node.GetString("key")
require.NoError(t, err)
key, err := base64.StdEncoding.DecodeString(keyB64)
require.NoError(t, err)
// Write the key to a temp file and use it in SSH
f, err := os.CreateTemp("", "kustomize_ssh")
require.NoError(t, err)
_, err = io.Copy(f, bytes.NewReader(key))
require.NoError(t, err)
cmd := fmt.Sprintf("ssh -i %s", f.Name())
const SSHCommandKey = "GIT_SSH_COMMAND"
t.Setenv(SSHCommandKey, cmd)
t.Cleanup(func() {
_ = os.Remove(f.Name())
})
}
func TestRemoteLoad(t *testing.T) {
req := require.New(t)
fSys := filesys.MakeFsOnDisk()
b := krusty.MakeKustomizer(krusty.MakeDefaultOptions())
m, err := b.Run(
fSys,
"github.com/kubernetes-sigs/kustomize/examples/multibases/dev/?ref=v1.0.6")
req.NoError(err)
checkYaml(m, multibaseDevExampleBuild, req)
}
func TestRemoteResourceHttps(t *testing.T) {
tests := map[string]*remoteResourceCase{
"basic": {
kustomization: `
resources:
- https://github.com/kubernetes-sigs/kustomize//examples/multibases/dev/?ref=v1.0.6`,
expected: multibaseDevExampleBuild,
},
".git repo suffix, no slash suffix": {
kustomization: `
resources:
- https://github.com/kubernetes-sigs/kustomize.git//examples/multibases/dev?ref=v1.0.6`,
expected: multibaseDevExampleBuild,
},
"repo": {
kustomization: `
resources:
- https://github.com/annasong20/kustomize-test.git?ref=main`,
expected: multibaseDevExampleBuild,
},
"raw remote file": {
kustomization: `
resources:
- https://raw.githubusercontent.com/kubernetes-sigs/kustomize/v3.1.0/examples/multibases/base/pod.yaml
namePrefix: dev-`,
expected: multibaseDevExampleBuild,
},
}
runResourceTests(t, tests)
}
func TestRemoteResourceSsh(t *testing.T) {
tests := map[string]*remoteResourceCase{
"scp shorthand": {
kustomization: `
resources:
- git@github.com:kubernetes-sigs/kustomize//examples/multibases/dev/?ref=v1.0.6`,
expected: multibaseDevExampleBuild,
},
"full ssh, no ending slash": {
kustomization: `
resources:
- ssh://git@github.com/kubernetes-sigs/kustomize//examples/multibases/dev?ref=v1.0.6`,
expected: multibaseDevExampleBuild,
},
"repo": {
local: true,
kustomization: `
resources:
- ssh://git@github.com/annasong20/kustomize-test.git?ref=main`,
expected: multibaseDevExampleBuild,
},
}
runResourceTests(t, tests)
}
func TestRemoteResourcePort(t *testing.T) {
sshURL := "ssh://git@github.com:22/kubernetes-sigs/kustomize//examples/multibases/dev/?ref=v1.0.6"
httpsURL := "https://github.com:443/kubernetes-sigs/kustomize//examples/multibases/dev/?ref=v1.0.6"
// TODO: ports not currently supported; implement in future
tests := map[string]*remoteResourceCase{
"ssh": {
local: true,
kustomization: fmt.Sprintf(resourcesField, sshURL),
error: true,
expected: fmt.Sprintf(resourceErrorFormat+fileError, sshURL),
},
"https": {
kustomization: fmt.Sprintf(resourcesField, httpsURL),
error: true,
expected: fmt.Sprintf(resourceErrorFormat+repoFindError, httpsURL),
},
}
runResourceTests(t, tests)
}
func TestRemoteResourceRepo(t *testing.T) {
tests := map[string]*remoteResourceCase{
"https, no ref": {
// TODO: fix flaky test that sporadically throws errors on server
local: true,
kustomization: `
resources:
- https://github.com/annasong20/kustomize-test.git`,
expected: multibaseDevExampleBuild,
},
"ssh, no ref": {
local: true,
kustomization: `
resources:
- git@github.com:annasong20/kustomize-test.git`,
expected: multibaseDevExampleBuild,
},
}
runResourceTests(t, tests)
}
func TestRemoteResourceParameters(t *testing.T) {
httpsNoParam := "https://github.com/kubernetes-sigs/kustomize//examples/multibases/dev/"
httpsMasterBranch := "https://github.com/kubernetes-sigs/kustomize//examples/multibases/dev?ref=master"
sshNoParams := "git@github.com:kubernetes-sigs/kustomize//examples/multibases/dev"
// TODO: cases with expected errors are query parameter combinations that aren't supported yet; implement in future
// TODO: fix flaky tests (non-ssh tests that we skip) that sporadically fail on server
tests := map[string]*remoteResourceCase{
"https no params": {
local: true,
kustomization: fmt.Sprintf(resourcesField, httpsNoParam),
error: true,
expected: fmt.Sprintf(resourceErrorFormat+repoFindError, httpsNoParam),
},
"https master": {
local: true,
kustomization: fmt.Sprintf(resourcesField, httpsMasterBranch),
error: true,
expected: fmt.Sprintf(resourceErrorFormat+repoFindError, httpsMasterBranch),
},
"https master and no submodules": {
local: true,
kustomization: `
resources:
- https://github.com/kubernetes-sigs/kustomize//examples/multibases/dev?ref=master&submodules=false`,
expected: multibaseDevExampleBuild,
},
"https all params": {
local: true,
kustomization: `
resources:
- https://github.com/kubernetes-sigs/kustomize//examples/multibases/dev?ref=v1.0.6&timeout=10&submodules=true`,
expected: multibaseDevExampleBuild,
},
"ssh no params": {
local: true,
kustomization: fmt.Sprintf(resourcesField, sshNoParams),
error: true,
expected: fmt.Sprintf(resourceErrorFormat+fileError, sshNoParams),
},
"ssh all params": {
local: true,
kustomization: `
resources:
- ssh://git@github.com/annasong20/kustomize-test.git?ref=main&timeout=10&submodules=true`,
expected: multibaseDevExampleBuild,
},
}
runResourceTests(t, tests)
}
func TestRemoteResourceGoGetter(t *testing.T) {
// TODO: fix flaky tests (the ones that we skip) that fail sporadically on server
tests := map[string]*remoteResourceCase{
"git detector with / subdirectory separator": {
local: true,
kustomization: `
resources:
- github.com/kubernetes-sigs/kustomize/examples/multibases/dev/?ref=v1.0.6`,
expected: multibaseDevExampleBuild,
},
"git detector for repo": {
local: true,
kustomization: `
resources:
- github.com/annasong20/kustomize-test`,
expected: multibaseDevExampleBuild,
},
"https with / subdirectory separator": {
local: true,
kustomization: `
resources:
- https://github.com/kubernetes-sigs/kustomize/examples/multibases/dev/?ref=v1.0.6`,
expected: multibaseDevExampleBuild,
},
"git forced protocol": {
kustomization: `
resources:
- git::https://github.com/kubernetes-sigs/kustomize//examples/multibases/dev/?ref=v1.0.6`,
expected: multibaseDevExampleBuild,
},
"git forced protocol with / subdirectory separator": {
local: true,
kustomization: `
resources:
- git::https://github.com/kubernetes-sigs/kustomize/examples/multibases/dev/?ref=v1.0.6`,
expected: multibaseDevExampleBuild,
},
}
runResourceTests(t, tests)
}
func TestRemoteResourceWithHttpError(t *testing.T) {
req := require.New(t)
url404 := "https://github.com/thisisa404.yaml"
fSys, tmpDir := createKustDir(fmt.Sprintf(resourcesField, url404), req)
b := krusty.MakeKustomizer(krusty.MakeDefaultOptions())
_, err := b.Run(fSys, tmpDir.String())
httpErr := fmt.Errorf("%w: status code %d (%s)", loader.ErrHTTP, 404, http.StatusText(404))
accuFromErr := fmt.Errorf("accumulating resources from '%s': %w", url404, httpErr)
expectedErr := fmt.Errorf("accumulating resources: %w", accuFromErr)
req.EqualErrorf(err, expectedErr.Error(), url404)
req.NoError(fSys.RemoveAll(tmpDir.String()))
}
func TestRemoteResourceAnnoOrigin(t *testing.T) {
test := remoteResourceCase{
kustomization: `
resources:
- github.com/kubernetes-sigs/kustomize/examples/multibases/dev/?ref=v1.0.6
buildMetadata: [originAnnotations]
`,
expected: `apiVersion: v1
kind: Pod
metadata:
annotations:
config.kubernetes.io/origin: |
path: examples/multibases/base/pod.yaml
repo: https://github.com/kubernetes-sigs/kustomize
ref: v1.0.6
labels:
app: myapp
name: dev-myapp-pod
spec:
containers:
- image: nginx:1.7.9
name: nginx
`,
}
testRemoteResource(require.New(t), &test)
}
func TestRemoteResourceAsBaseWithAnnoOrigin(t *testing.T) {
req := require.New(t)
fSys := filesys.MakeFsOnDisk()
b := krusty.MakeKustomizer(krusty.MakeDefaultOptions())
tmpDir, err := filesys.NewTmpConfirmedDir()
req.NoError(err)
base := filepath.Join(tmpDir.String(), "base")
prod := filepath.Join(tmpDir.String(), "prod")
req.NoError(fSys.Mkdir(base))
req.NoError(fSys.Mkdir(prod))
req.NoError(fSys.WriteFile(filepath.Join(base, "kustomization.yaml"), []byte(`
resources:
- github.com/kubernetes-sigs/kustomize/examples/multibases/dev/?ref=v1.0.6
`)))
req.NoError(fSys.WriteFile(filepath.Join(prod, "kustomization.yaml"), []byte(`
resources:
- ../base
namePrefix: prefix-
buildMetadata: [originAnnotations]
`)))
m, err := b.Run(
fSys,
prod)
req.NoError(err)
expected := `apiVersion: v1
kind: Pod
metadata:
annotations:
config.kubernetes.io/origin: |
path: examples/multibases/base/pod.yaml
repo: https://github.com/kubernetes-sigs/kustomize
ref: v1.0.6
labels:
app: myapp
name: prefix-dev-myapp-pod
spec:
containers:
- image: nginx:1.7.9
name: nginx
`
checkYaml(m, expected, req)
req.NoError(fSys.RemoveAll(tmpDir.String()))
}

443
api/krusty/remoteloader_test.go Normal file
View File

@@ -0,0 +1,443 @@
// Copyright 2022 The Kubernetes Authors.
// SPDX-License-Identifier: Apache-2.0
package krusty_test
import (
"bytes"
"encoding/base64"
"fmt"
"io"
"os"
"os/exec"
"path/filepath"
"strings"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"sigs.k8s.io/kustomize/api/krusty"
"sigs.k8s.io/kustomize/api/loader"
"sigs.k8s.io/kustomize/api/resmap"
"sigs.k8s.io/kustomize/kyaml/filesys"
"sigs.k8s.io/kustomize/kyaml/yaml"
)
func TestRemoteLoad_LocalProtocol(t *testing.T) {
type testRepos struct {
root string
simple string
noSuffix string
multiBaseDev string
withSubmodule string
}
// creates git repos under a root temporary directory with the following structure
// root/
// simple.git/ - base with just a pod
// nosuffix/ - same as simple.git/ without the .git suffix
// multibase.git/ - base with a dev overlay
// with-submodule.git/ - includes `simple` as a submodule
// submodule/ - the submodule referencing `simple
createGitRepos := func(t *testing.T) testRepos {
t.Helper()
bash := func(script string) {
cmd := exec.Command("sh", "-c", script)
o, err := cmd.CombinedOutput()
if err != nil {
t.Fatalf("error running %v\nerr: %v\n%s", script, err, string(o))
}
}
root := t.TempDir()
bash(fmt.Sprintf(`
set -eux
export ROOT="%s"
export GIT_AUTHOR_EMAIL=nobody@kustomize.io
export GIT_AUTHOR_NAME=Nobody
export GIT_COMMITTER_EMAIL=nobody@kustomize.io
export GIT_COMMITTER_NAME=Nobody
cp -r testdata/remoteload/simple $ROOT/simple.git
(
cd $ROOT/simple.git
git init --initial-branch=main
git add .
git commit -m "import"
git checkout -b change-image
cat >>kustomization.yaml <<EOF
images:
- name: nginx
newName: nginx
newTag: "2"
EOF
git commit -am "image change"
git checkout main
)
cp -r $ROOT/simple.git $ROOT/nosuffix
cp -r testdata/remoteload/multibase $ROOT/multibase.git
(
cd $ROOT/multibase.git
git init --initial-branch=main
git add .
git commit -m "import"
)
(
mkdir $ROOT/with-submodule.git
cd $ROOT/with-submodule.git
git init --initial-branch=main
git submodule add $ROOT/simple.git submodule
git commit -m "import"
)
`, root))
return testRepos{
root: root,
// The strings below aren't currently used, and more serve as documentation.
simple: "simple.git",
noSuffix: "nosuffix",
multiBaseDev: "multibase.git",
withSubmodule: "with-submodule.git",
}
}
const simpleBuild = `apiVersion: v1
kind: Pod
metadata:
labels:
app: myapp
name: myapp-pod
spec:
containers:
- image: nginx:1.7.9
name: nginx
`
var simpleBuildWithNginx2 = strings.ReplaceAll(simpleBuild, "nginx:1.7.9", "nginx:2")
var multibaseDevExampleBuild = strings.ReplaceAll(simpleBuild, "myapp-pod", "dev-myapp-pod")
repos := createGitRepos(t)
tests := []struct {
name string
kustomization string
expected string
err string
skip bool
}{
{
name: "simple",
kustomization: `
resources:
- file://$ROOT/simple.git
`,
expected: simpleBuild,
},
{
name: "without git suffix",
kustomization: `
resources:
- file://$ROOT/nosuffix
`,
expected: simpleBuild,
},
{
name: "has path",
kustomization: `
resources:
- file://$ROOT/multibase.git/dev
`,
expected: multibaseDevExampleBuild,
},
{
name: "has path without git suffix",
kustomization: `
resources:
- file://$ROOT/multibase//dev
`,
expected: multibaseDevExampleBuild,
},
{
name: "has ref",
kustomization: `
resources:
- "file://$ROOT/simple.git?ref=change-image"
`,
expected: simpleBuildWithNginx2,
},
{
// Version is the same as ref
name: "has version",
kustomization: `
resources:
- file://$ROOT/simple.git?version=change-image
`,
expected: simpleBuildWithNginx2,
},
{
name: "has submodule",
kustomization: `
resources:
- file://$ROOT/with-submodule.git/submodule
`,
expected: simpleBuild,
},
{
name: "has timeout",
kustomization: `
resources:
- file://$ROOT/simple.git?timeout=500
`,
expected: simpleBuild,
},
{
name: "triple slash absolute path",
kustomization: `
resources:
- file:///$ROOT/simple.git
`,
expected: simpleBuild,
},
{
name: "has submodule but not initialized",
kustomization: `
resources:
- file://$ROOT/with-submodule.git/submodule?submodules=0
`,
err: "unable to find one of 'kustomization.yaml', 'kustomization.yml' or 'Kustomization' in directory",
},
{
name: "has origin annotation",
skip: true, // The annotated path should be "pod.yaml" but is "notCloned/pod.yaml"
kustomization: `
resources:
- file://$ROOT/simple.git
buildMetadata: [originAnnotations]
`,
expected: `apiVersion: v1
kind: Pod
metadata:
annotations:
config.kubernetes.io/origin: |
path: pod.yaml
repo: file://$ROOT/simple.git
labels:
app: myapp
name: myapp-pod
spec:
containers:
- image: nginx:1.7.9
name: nginx
`,
},
{
name: "has ref path timeout and origin annotation",
kustomization: `
resources:
- file://$ROOT/multibase.git/dev?version=main&timeout=500
buildMetadata: [originAnnotations]
`,
expected: `apiVersion: v1
kind: Pod
metadata:
annotations:
config.kubernetes.io/origin: |
path: base/pod.yaml
repo: file://$ROOT/multibase.git
ref: main
labels:
app: myapp
name: dev-myapp-pod
spec:
containers:
- image: nginx:1.7.9
name: nginx
`,
},
{
name: "repo does not exist ",
kustomization: `
resources:
- file:///not/a/real/repo
`,
err: "fatal: '/not/a/real/repo' does not appear to be a git repository",
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
if test.skip {
t.SkipNow()
}
kust := strings.ReplaceAll(test.kustomization, "$ROOT", repos.root)
fSys, tmpDir := createKustDir(t, kust)
b := krusty.MakeKustomizer(krusty.MakeDefaultOptions())
m, err := b.Run(
fSys,
tmpDir.String())
if test.err != "" {
require.Error(t, err)
require.Contains(t, err.Error(), test.err)
} else {
require.NoError(t, err)
checkYaml(t, m, strings.ReplaceAll(test.expected, "$ROOT", repos.root))
}
})
}
}
func TestRemoteLoad_RemoteProtocols(t *testing.T) {
// Slow remote tests with long timeouts.
// TODO: If these end up flaking, they should retry. If not, remove this TODO.
tests := []struct {
name string
kustomization string
err string
errT error
beforeTest func(t *testing.T)
}{
{
name: "https",
kustomization: `
resources:
- https://github.com/kubernetes-sigs/kustomize//examples/multibases/dev/?submodules=0&ref=kustomize%2Fv4.5.7&timeout=300
`,
},
{
name: "git double-colon https",
kustomization: `
resources:
- git::https://github.com/kubernetes-sigs/kustomize//examples/multibases/dev/?submodules=0&ref=kustomize%2Fv4.5.7&timeout=300
`,
},
{
name: "https raw remote file",
kustomization: `
resources:
- https://raw.githubusercontent.com/kubernetes-sigs/kustomize/v3.1.0/examples/multibases/base/pod.yaml?timeout=300
namePrefix: dev-
`,
},
{
name: "https without scheme",
kustomization: `
resources:
- github.com/kubernetes-sigs/kustomize/examples/multibases/dev?submodules=0&ref=kustomize%2Fv4.5.7&timeout=300
`,
},
{
name: "ssh",
beforeTest: configureGitSSHCommand,
kustomization: `
resources:
- git@github.com/kubernetes-sigs/kustomize/examples/multibases/dev?submodules=0&ref=kustomize%2Fv4.5.7&timeout=300
`,
},
{
name: "ssh with colon",
beforeTest: configureGitSSHCommand,
kustomization: `
resources:
- git@github.com:kubernetes-sigs/kustomize/examples/multibases/dev?submodules=0&ref=kustomize%2Fv4.5.7&timeout=300
`,
},
{
name: "ssh scheme",
beforeTest: configureGitSSHCommand,
kustomization: `
resources:
- ssh://git@github.com/kubernetes-sigs/kustomize/examples/multibases/dev?submodules=0&ref=kustomize%2Fv4.5.7&timeout=300
`,
},
{
name: "http error",
kustomization: `
resources:
- https://github.com/thisisa404.yaml
`,
err: "accumulating resources: accumulating resources from 'https://github.com/thisisa404.yaml': HTTP Error: status code 404 (Not Found)",
errT: loader.ErrHTTP,
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
if test.beforeTest != nil {
test.beforeTest(t)
}
fSys, tmpDir := createKustDir(t, test.kustomization)
b := krusty.MakeKustomizer(krusty.MakeDefaultOptions())
m, err := b.Run(
fSys,
tmpDir.String())
if test.err != "" {
require.Error(t, err)
assert.Contains(t, err.Error(), test.err)
if test.errT != nil {
assert.ErrorIs(t, err, test.errT)
}
} else {
require.NoError(t, err)
const multibaseDevExampleBuild = `apiVersion: v1
kind: Pod
metadata:
labels:
app: myapp
name: dev-myapp-pod
spec:
containers:
- image: nginx:1.7.9
name: nginx
`
checkYaml(t, m, multibaseDevExampleBuild)
}
})
}
}
func configureGitSSHCommand(t *testing.T) {
t.Helper()
// This contains a read-only Deploy Key for the kustomize repo.
node, err := yaml.ReadFile("testdata/repo_read_only_ssh_key.yaml")
require.NoError(t, err)
keyB64, err := node.GetString("key")
require.NoError(t, err)
key, err := base64.StdEncoding.DecodeString(keyB64)
require.NoError(t, err)
// Write the key to a temp file and use it in SSH
f, err := os.CreateTemp("", "kustomize_ssh")
require.NoError(t, err)
_, err = io.Copy(f, bytes.NewReader(key))
require.NoError(t, err)
cmd := fmt.Sprintf("ssh -i %s -o IdentitiesOnly=yes", f.Name())
const SSHCommandKey = "GIT_SSH_COMMAND"
t.Setenv(SSHCommandKey, cmd)
t.Cleanup(func() {
_ = os.Remove(f.Name())
})
}
func createKustDir(t *testing.T, content string) (filesys.FileSystem, filesys.ConfirmedDir) {
t.Helper()
fSys := filesys.MakeFsOnDisk()
tmpDir, err := filesys.NewTmpConfirmedDir()
require.NoError(t, err)
require.NoError(t, fSys.WriteFile(filepath.Join(tmpDir.String(), "kustomization.yaml"), []byte(content)))
return fSys, tmpDir
}
func checkYaml(t *testing.T, actual resmap.ResMap, expected string) {
t.Helper()
yml, err := actual.AsYaml()
assert.NoError(t, err)
assert.Equal(t, expected, string(yml))
}

2
api/krusty/testdata/remoteload/multibase/base/kustomization.yaml vendored Normal file
View File

@@ -0,0 +1,2 @@
resources:
- pod.yaml

10
api/krusty/testdata/remoteload/multibase/base/pod.yaml vendored Normal file
View File

@@ -0,0 +1,10 @@
apiVersion: v1
kind: Pod
metadata:
name: myapp-pod
labels:
app: myapp
spec:
containers:
- name: nginx
image: nginx:1.7.9

3
api/krusty/testdata/remoteload/multibase/dev/kustomization.yaml vendored Normal file
View File

@@ -0,0 +1,3 @@
resources:
- ../base
namePrefix: dev-

2
api/krusty/testdata/remoteload/simple/kustomization.yaml vendored Normal file
View File

@@ -0,0 +1,2 @@
resources:
- pod.yaml

10
api/krusty/testdata/remoteload/simple/pod.yaml vendored Normal file
View File

@@ -0,0 +1,10 @@
apiVersion: v1
kind: Pod
metadata:
name: myapp-pod
labels:
app: myapp
spec:
containers:
- name: nginx
image: nginx:1.7.9

2
api/krusty/transformerannotation_test.go
View File

@@ -288,7 +288,7 @@ kind: Pod
metadata:
annotations:
alpha.config.kubernetes.io/transformations: |
- repo: https://github.com/kubernetes-sigs/kustomize
- repo: https://github.com/kubernetes-sigs/kustomize.git
ref: v1.0.6
configuredIn: examples/multibases/production/kustomization.yaml
configuredBy:

2
api/resource/origin.go
View File

@@ -52,7 +52,7 @@ func (origin *Origin) Append(path string) *Origin {
originCopy := origin.Copy()
repoSpec, err := git.NewRepoSpecFromURL(path)
if err == nil {
originCopy.Repo = repoSpec.Host + repoSpec.OrgRepo
originCopy.Repo = repoSpec.CloneSpec()
absPath := repoSpec.AbsPath()
path = absPath[strings.Index(absPath[1:], "/")+1:][1:]
originCopy.Path = ""

4
api/resource/origin_test.go
View File

@@ -31,14 +31,14 @@ func TestOriginAppend(t *testing.T) {
},
path: "github.com/kubernetes-sigs/kustomize/examples/multibases/dev/",
expected: `path: examples/multibases/dev
repo: https://github.com/kubernetes-sigs/kustomize
repo: https://github.com/kubernetes-sigs/kustomize.git
`,
},
}
for _, test := range tests {
actual, err := test.in.Append(test.path).String()
assert.NoError(t, err)
assert.Equal(t, actual, test.expected)
assert.Equal(t, test.expected, actual)
}
}

10
examples/remoteBuild.md
View File

@@ -12,6 +12,7 @@ following query string parameters can also be specified:
* `ref` - a [`git fetch`-able ref](https://git-scm.com/docs/git-fetch), typically a branch, tag, or full commit hash
(short hashes are not supported)
* `version` - same as `ref`. If `ref` is provided, this is ignored.
* `timeout` (default `27s`) - a number in seconds, or a go duration. specifies
the timeout for fetching the resource
* `submodules` (default `true`) - a boolean specifying whether to clone
@@ -25,7 +26,14 @@ will essentially clone the git repo via HTTPS, checkout `v1.0.6` and run
SSH clones are also supported either with `git@github.com:owner/repo` or
`ssh://git@github.com/owner/repo` URLs.
`file:///` clones are not supported.
`file:///` clones are supported. For
example, `file:///path/to/repo//someSubdir?ref=v1.0.6`, references the absolute
path to the repo at `/path/to/repo`, and a kustomization directory
at `someSubdir` within that repo. `//` to delimits the root of the repo.
Kustomize will clone the repo to a temporary directory and do a clean checkout
of the `ref`. This behavior is differs from a direct path reference
like `/path/to/repo/someSubdir`, in which case Kustomize will not use Git at
all, and process the files at the path directly.
## remote files
Resources can reference remote files via their raw GitHub urls, such