Reduce k8ds deps

This commit is contained in:
jregan
2019-05-27 08:21:10 -07:00
parent 3af0f9776f
commit 47c965481f
39 changed files with 563 additions and 698 deletions

View File

@@ -1,18 +1,5 @@
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// Copyright 2019 The Kubernetes Authors.
// SPDX-License-Identifier: Apache-2.0
package loader
@@ -92,6 +79,9 @@ type fileLoader struct {
// Restricts behavior of Load function.
loadRestrictor LoadRestrictorFunc
// Used to validate various k8s data fields.
validator ifc.Validator
// If this is non-nil, the files were
// obtained from the given repository.
repoSpec *git.RepoSpec
@@ -110,41 +100,44 @@ const CWD = "."
// NewFileLoaderAtCwd returns a loader that loads from ".".
// A convenience for kustomize edit commands.
func NewFileLoaderAtCwd(fSys fs.FileSystem) *fileLoader {
func NewFileLoaderAtCwd(v ifc.Validator, fSys fs.FileSystem) *fileLoader {
return newLoaderOrDie(
RestrictionRootOnly, fSys, CWD)
RestrictionRootOnly, v, fSys, CWD)
}
// NewFileLoaderAtRoot returns a loader that loads from "/".
// A convenience for tests.
func NewFileLoaderAtRoot(fSys fs.FileSystem) *fileLoader {
func NewFileLoaderAtRoot(v ifc.Validator, fSys fs.FileSystem) *fileLoader {
return newLoaderOrDie(
RestrictionRootOnly, fSys, string(filepath.Separator))
RestrictionRootOnly, v, fSys, string(filepath.Separator))
}
// Root returns the absolute path that is prepended to any
// relative paths used in Load.
func (l *fileLoader) Root() string {
return l.root.String()
func (fl *fileLoader) Root() string {
return fl.root.String()
}
func newLoaderOrDie(
lr LoadRestrictorFunc, fSys fs.FileSystem, path string) *fileLoader {
lr LoadRestrictorFunc, v ifc.Validator,
fSys fs.FileSystem, path string) *fileLoader {
root, err := demandDirectoryRoot(fSys, path)
if err != nil {
log.Fatalf("unable to make loader at '%s'; %v", path, err)
}
return newLoaderAtConfirmedDir(
lr, root, fSys, nil, git.ClonerUsingGitExec)
lr, v, root, fSys, nil, git.ClonerUsingGitExec)
}
// newLoaderAtConfirmedDir returns a new fileLoader with given root.
func newLoaderAtConfirmedDir(
lr LoadRestrictorFunc,
v ifc.Validator,
root fs.ConfirmedDir, fSys fs.FileSystem,
referrer *fileLoader, cloner git.Cloner) *fileLoader {
return &fileLoader{
loadRestrictor: lr,
validator: v,
root: root,
referrer: referrer,
fSys: fSys,
@@ -175,39 +168,41 @@ func demandDirectoryRoot(
// New returns a new Loader, rooted relative to current loader,
// or rooted in a temp directory holding a git repo clone.
func (l *fileLoader) New(path string) (ifc.Loader, error) {
func (fl *fileLoader) New(path string) (ifc.Loader, error) {
if path == "" {
return nil, fmt.Errorf("new root cannot be empty")
}
repoSpec, err := git.NewRepoSpecFromUrl(path)
if err == nil {
// Treat this as git repo clone request.
if err := l.errIfRepoCycle(repoSpec); err != nil {
if err := fl.errIfRepoCycle(repoSpec); err != nil {
return nil, err
}
return newLoaderAtGitClone(repoSpec, l.fSys, l.referrer, l.cloner)
return newLoaderAtGitClone(
repoSpec, fl.validator, fl.fSys, fl.referrer, fl.cloner)
}
if filepath.IsAbs(path) {
return nil, fmt.Errorf("new root '%s' cannot be absolute", path)
}
root, err := demandDirectoryRoot(l.fSys, l.root.Join(path))
root, err := demandDirectoryRoot(fl.fSys, fl.root.Join(path))
if err != nil {
return nil, err
}
if err := l.errIfGitContainmentViolation(root); err != nil {
if err := fl.errIfGitContainmentViolation(root); err != nil {
return nil, err
}
if err := l.errIfArgEqualOrHigher(root); err != nil {
if err := fl.errIfArgEqualOrHigher(root); err != nil {
return nil, err
}
return newLoaderAtConfirmedDir(
l.loadRestrictor, root, l.fSys, l, l.cloner), nil
fl.loadRestrictor, fl.validator, root, fl.fSys, fl, fl.cloner), nil
}
// newLoaderAtGitClone returns a new Loader pinned to a temporary
// directory holding a cloned git repo.
func newLoaderAtGitClone(
repoSpec *git.RepoSpec, fSys fs.FileSystem,
repoSpec *git.RepoSpec,
v ifc.Validator, fSys fs.FileSystem,
referrer *fileLoader, cloner git.Cloner) (ifc.Loader, error) {
err := cloner(repoSpec)
if err != nil {
@@ -229,6 +224,7 @@ func newLoaderAtGitClone(
return &fileLoader{
// Clones never allowed to escape root.
loadRestrictor: RestrictionRootOnly,
validator: v,
root: root,
referrer: referrer,
repoSpec: repoSpec,
@@ -238,9 +234,9 @@ func newLoaderAtGitClone(
}, nil
}
func (l *fileLoader) errIfGitContainmentViolation(
func (fl *fileLoader) errIfGitContainmentViolation(
base fs.ConfirmedDir) error {
containingRepo := l.containingRepo()
containingRepo := fl.containingRepo()
if containingRepo == nil {
return nil
}
@@ -256,64 +252,64 @@ func (l *fileLoader) errIfGitContainmentViolation(
// Looks back through referrers for a git repo, returning nil
// if none found.
func (l *fileLoader) containingRepo() *git.RepoSpec {
if l.repoSpec != nil {
return l.repoSpec
func (fl *fileLoader) containingRepo() *git.RepoSpec {
if fl.repoSpec != nil {
return fl.repoSpec
}
if l.referrer == nil {
if fl.referrer == nil {
return nil
}
return l.referrer.containingRepo()
return fl.referrer.containingRepo()
}
// errIfArgEqualOrHigher tests whether the argument,
// is equal to or above the root of any ancestor.
func (l *fileLoader) errIfArgEqualOrHigher(
func (fl *fileLoader) errIfArgEqualOrHigher(
candidateRoot fs.ConfirmedDir) error {
if l.root.HasPrefix(candidateRoot) {
if fl.root.HasPrefix(candidateRoot) {
return fmt.Errorf(
"cycle detected: candidate root '%s' contains visited root '%s'",
candidateRoot, l.root)
candidateRoot, fl.root)
}
if l.referrer == nil {
if fl.referrer == nil {
return nil
}
return l.referrer.errIfArgEqualOrHigher(candidateRoot)
return fl.referrer.errIfArgEqualOrHigher(candidateRoot)
}
// TODO(monopole): Distinguish branches?
// I.e. Allow a distinction between git URI with
// path foo and tag bar and a git URI with the same
// path but a different tag?
func (l *fileLoader) errIfRepoCycle(newRepoSpec *git.RepoSpec) error {
func (fl *fileLoader) errIfRepoCycle(newRepoSpec *git.RepoSpec) error {
// TODO(monopole): Use parsed data instead of Raw().
if l.repoSpec != nil &&
strings.HasPrefix(l.repoSpec.Raw(), newRepoSpec.Raw()) {
if fl.repoSpec != nil &&
strings.HasPrefix(fl.repoSpec.Raw(), newRepoSpec.Raw()) {
return fmt.Errorf(
"cycle detected: URI '%s' referenced by previous URI '%s'",
newRepoSpec.Raw(), l.repoSpec.Raw())
newRepoSpec.Raw(), fl.repoSpec.Raw())
}
if l.referrer == nil {
if fl.referrer == nil {
return nil
}
return l.referrer.errIfRepoCycle(newRepoSpec)
return fl.referrer.errIfRepoCycle(newRepoSpec)
}
// Load returns the content of file at the given path,
// else an error. Relative paths are taken relative
// to the root.
func (l *fileLoader) Load(path string) ([]byte, error) {
func (fl *fileLoader) Load(path string) ([]byte, error) {
if !filepath.IsAbs(path) {
path = l.root.Join(path)
path = fl.root.Join(path)
}
path, err := l.loadRestrictor(l.fSys, l.root, path)
path, err := fl.loadRestrictor(fl.fSys, fl.root, path)
if err != nil {
return nil, err
}
return l.fSys.ReadFile(path)
return fl.fSys.ReadFile(path)
}
// Cleanup runs the cleaner.
func (l *fileLoader) Cleanup() error {
return l.cleaner()
func (fl *fileLoader) Cleanup() error {
return fl.cleaner()
}

View File

@@ -29,6 +29,7 @@ import (
"sigs.k8s.io/kustomize/pkg/git"
"sigs.k8s.io/kustomize/pkg/ifc"
"sigs.k8s.io/kustomize/pkg/pgmconfig"
"sigs.k8s.io/kustomize/pkg/validators"
)
type testData struct {
@@ -63,8 +64,12 @@ func MakeFakeFs(td []testData) fs.FileSystem {
return fSys
}
func makeLoader() *fileLoader {
return NewFileLoaderAtRoot(validators.MakeFakeValidator(), MakeFakeFs(testCases))
}
func TestLoaderLoad(t *testing.T) {
l1 := NewFileLoaderAtRoot(MakeFakeFs(testCases))
l1 := makeLoader()
if "/" != l1.Root() {
t.Fatalf("incorrect root: '%s'\n", l1.Root())
}
@@ -103,7 +108,7 @@ func TestLoaderLoad(t *testing.T) {
}
func TestLoaderNewSubDir(t *testing.T) {
l1, err := NewFileLoaderAtRoot(MakeFakeFs(testCases)).New("foo/project")
l1, err := makeLoader().New("foo/project")
if err != nil {
t.Fatalf("unexpected err: %v\n", err)
}
@@ -125,7 +130,7 @@ func TestLoaderNewSubDir(t *testing.T) {
}
func TestLoaderBadRelative(t *testing.T) {
l1, err := NewFileLoaderAtRoot(MakeFakeFs(testCases)).New("foo/project/subdir1")
l1, err := makeLoader().New("foo/project/subdir1")
if err != nil {
t.Fatalf("unexpected err: %v\n", err)
}
@@ -195,7 +200,7 @@ func TestLoaderBadRelative(t *testing.T) {
}
func TestLoaderMisc(t *testing.T) {
l := NewFileLoaderAtRoot(MakeFakeFs(testCases))
l := makeLoader()
_, err := l.New("")
if err == nil {
t.Fatalf("Expected error for empty root location not returned")
@@ -297,7 +302,8 @@ func TestRestrictionRootOnlyInRealLoader(t *testing.T) {
var l ifc.Loader
l = newLoaderOrDie(RestrictionRootOnly, fSys, dir)
l = newLoaderOrDie(
RestrictionRootOnly, validators.MakeFakeValidator(), fSys, dir)
l = doSanityChecksAndDropIntoBase(t, l)
@@ -330,7 +336,8 @@ func TestRestrictionNoneInRealLoader(t *testing.T) {
var l ifc.Loader
l = newLoaderOrDie(RestrictionNone, fSys, dir)
l = newLoaderOrDie(
RestrictionNone, validators.MakeFakeValidator(), fSys, dir)
l = doSanityChecksAndDropIntoBase(t, l)
@@ -392,7 +399,7 @@ whatever
t.Fatalf("unexpected err: %v\n", err)
}
l, err := newLoaderAtGitClone(
repoSpec, fSys, nil,
repoSpec, validators.MakeFakeValidator(), fSys, nil,
git.DoNothingCloner(fs.ConfirmedDir(coRoot)))
if err != nil {
t.Fatalf("unexpected err: %v\n", err)
@@ -434,7 +441,8 @@ func TestLoaderDisallowsLocalBaseFromRemoteOverlay(t *testing.T) {
// Establish that a local overlay can navigate
// to the local bases.
l1 = newLoaderOrDie(RestrictionRootOnly, fSys, cloneRoot+"/foo/overlay")
l1 = newLoaderOrDie(
RestrictionRootOnly, validators.MakeFakeValidator(), fSys, cloneRoot+"/foo/overlay")
if l1.Root() != cloneRoot+"/foo/overlay" {
t.Fatalf("unexpected root %s", l1.Root())
}
@@ -470,7 +478,7 @@ func TestLoaderDisallowsLocalBaseFromRemoteOverlay(t *testing.T) {
t.Fatalf("unexpected err: %v\n", err)
}
l1, err = newLoaderAtGitClone(
repoSpec, fSys, nil,
repoSpec, validators.MakeFakeValidator(), fSys, nil,
git.DoNothingCloner(fs.ConfirmedDir(cloneRoot)))
if err != nil {
t.Fatalf("unexpected err: %v\n", err)
@@ -509,7 +517,7 @@ func TestLocalLoaderReferencingGitBase(t *testing.T) {
t.Fatalf("unexpected err: %v\n", err)
}
l1 := newLoaderAtConfirmedDir(
RestrictionRootOnly, root, fSys, nil,
RestrictionRootOnly, validators.MakeFakeValidator(), root, fSys, nil,
git.DoNothingCloner(fs.ConfirmedDir(cloneRoot)))
if l1.Root() != topDir {
t.Fatalf("unexpected root %s", l1.Root())

200
pkg/loader/kv.go Normal file
View File

@@ -0,0 +1,200 @@
// Copyright 2019 The Kubernetes Authors.
// SPDX-License-Identifier: Apache-2.0
package loader
import (
"bufio"
"bytes"
"fmt"
"os"
"path"
"strings"
"unicode"
"unicode/utf8"
"github.com/pkg/errors"
"sigs.k8s.io/kustomize/pkg/ifc"
"sigs.k8s.io/kustomize/pkg/types"
)
var utf8bom = []byte{0xEF, 0xBB, 0xBF}
func (fl *fileLoader) Validator() ifc.Validator {
return fl.validator
}
func (fl *fileLoader) LoadKvPairs(
args types.GeneratorArgs) (all []types.Pair, err error) {
pairs, err := fl.keyValuesFromEnvFiles(args.EnvSources)
if err != nil {
return nil, errors.Wrap(err, fmt.Sprintf(
"env source files: %v",
args.EnvSources))
}
all = append(all, pairs...)
pairs, err = keyValuesFromLiteralSources(args.LiteralSources)
if err != nil {
return nil, errors.Wrap(err, fmt.Sprintf(
"literal sources %v", args.LiteralSources))
}
all = append(all, pairs...)
pairs, err = fl.keyValuesFromFileSources(args.FileSources)
if err != nil {
return nil, errors.Wrap(err, fmt.Sprintf(
"file sources: %v", args.FileSources))
}
return append(all, pairs...), nil
}
func keyValuesFromLiteralSources(sources []string) ([]types.Pair, error) {
var kvs []types.Pair
for _, s := range sources {
k, v, err := parseLiteralSource(s)
if err != nil {
return nil, err
}
kvs = append(kvs, types.Pair{Key: k, Value: v})
}
return kvs, nil
}
func (fl *fileLoader) keyValuesFromFileSources(sources []string) ([]types.Pair, error) {
var kvs []types.Pair
for _, s := range sources {
k, fPath, err := parseFileSource(s)
if err != nil {
return nil, err
}
content, err := fl.Load(fPath)
if err != nil {
return nil, err
}
kvs = append(kvs, types.Pair{Key: k, Value: string(content)})
}
return kvs, nil
}
func (fl *fileLoader) keyValuesFromEnvFiles(paths []string) ([]types.Pair, error) {
var kvs []types.Pair
for _, p := range paths {
content, err := fl.Load(p)
if err != nil {
return nil, err
}
more, err := fl.keyValuesFromLines(content)
if err != nil {
return nil, err
}
kvs = append(kvs, more...)
}
return kvs, nil
}
// keyValuesFromLines parses given content in to a list of key-value pairs.
func (fl *fileLoader) keyValuesFromLines(content []byte) ([]types.Pair, error) {
var kvs []types.Pair
scanner := bufio.NewScanner(bytes.NewReader(content))
currentLine := 0
for scanner.Scan() {
// Process the current line, retrieving a key/value pair if
// possible.
scannedBytes := scanner.Bytes()
kv, err := fl.keyValuesFromLine(scannedBytes, currentLine)
if err != nil {
return nil, err
}
currentLine++
if len(kv.Key) == 0 {
// no key means line was empty or a comment
continue
}
kvs = append(kvs, kv)
}
return kvs, nil
}
// KeyValuesFromLine returns a kv with blank key if the line is empty or a comment.
// The value will be retrieved from the environment if necessary.
func (fl *fileLoader) keyValuesFromLine(line []byte, currentLine int) (types.Pair, error) {
kv := types.Pair{}
if !utf8.Valid(line) {
return kv, fmt.Errorf("line %d has invalid utf8 bytes : %v", line, string(line))
}
// We trim UTF8 BOM from the first line of the file but no others
if currentLine == 0 {
line = bytes.TrimPrefix(line, utf8bom)
}
// trim the line from all leading whitespace first
line = bytes.TrimLeftFunc(line, unicode.IsSpace)
// If the line is empty or a comment, we return a blank key/value pair.
if len(line) == 0 || line[0] == '#' {
return kv, nil
}
data := strings.SplitN(string(line), "=", 2)
key := data[0]
if err := fl.validator.IsEnvVarName(key); err != nil {
return kv, err
}
if len(data) == 2 {
kv.Value = data[1]
} else {
// No value (no `=` in the line) is a signal to obtain the value
// from the environment.
kv.Value = os.Getenv(key)
}
kv.Key = key
return kv, nil
}
// ParseFileSource parses the source given.
//
// Acceptable formats include:
// 1. source-path: the basename will become the key name
// 2. source-name=source-path: the source-name will become the key name and
// source-path is the path to the key file.
//
// Key names cannot include '='.
func parseFileSource(source string) (keyName, filePath string, err error) {
numSeparators := strings.Count(source, "=")
switch {
case numSeparators == 0:
return path.Base(source), source, nil
case numSeparators == 1 && strings.HasPrefix(source, "="):
return "", "", fmt.Errorf("key name for file path %v missing", strings.TrimPrefix(source, "="))
case numSeparators == 1 && strings.HasSuffix(source, "="):
return "", "", fmt.Errorf("file path for key name %v missing", strings.TrimSuffix(source, "="))
case numSeparators > 1:
return "", "", errors.New("key names or file paths cannot contain '='")
default:
components := strings.Split(source, "=")
return components[0], components[1], nil
}
}
// ParseLiteralSource parses the source key=val pair into its component pieces.
// This functionality is distinguished from strings.SplitN(source, "=", 2) since
// it returns an error in the case of empty keys, values, or a missing equals sign.
func parseLiteralSource(source string) (keyName, value string, err error) {
// leading equal is invalid
if strings.Index(source, "=") == 0 {
return "", "", fmt.Errorf("invalid literal source %v, expected key=value", source)
}
// split after the first equal (so values can have the = character)
items := strings.SplitN(source, "=", 2)
if len(items) != 2 {
return "", "", fmt.Errorf("invalid literal source %v, expected key=value", source)
}
return items[0], strings.Trim(items[1], "\"'"), nil
}

91
pkg/loader/kv_test.go Normal file
View File

@@ -0,0 +1,91 @@
// Copyright 2019 The Kubernetes Authors.
// SPDX-License-Identifier: Apache-2.0
package loader
import (
"reflect"
"testing"
"sigs.k8s.io/kustomize/pkg/fs"
"sigs.k8s.io/kustomize/pkg/types"
"sigs.k8s.io/kustomize/pkg/validators"
)
func TestKeyValuesFromLines(t *testing.T) {
tests := []struct {
desc string
content string
expectedPairs []types.Pair
expectedErr bool
}{
{
desc: "valid kv content parse",
content: `
k1=v1
k2=v2
`,
expectedPairs: []types.Pair{
{Key: "k1", Value: "v1"},
{Key: "k2", Value: "v2"},
},
expectedErr: false,
},
{
desc: "content with comments",
content: `
k1=v1
#k2=v2
`,
expectedPairs: []types.Pair{
{Key: "k1", Value: "v1"},
},
expectedErr: false,
},
// TODO: add negative testcases
}
l := NewFileLoaderAtRoot(
validators.MakeFakeValidator(), fs.MakeFakeFS())
for _, test := range tests {
pairs, err := l.keyValuesFromLines([]byte(test.content))
if test.expectedErr && err == nil {
t.Fatalf("%s should not return error", test.desc)
}
if !reflect.DeepEqual(pairs, test.expectedPairs) {
t.Errorf("%s should succeed, got:%v exptected:%v", test.desc, pairs, test.expectedPairs)
}
}
}
func TestKeyValuesFromFileSources(t *testing.T) {
tests := []struct {
description string
sources []string
expected []types.Pair
}{
{
description: "create kvs from file sources",
sources: []string{"files/app-init.ini"},
expected: []types.Pair{
{
Key: "app-init.ini",
Value: "FOO=bar",
},
},
},
}
fSys := fs.MakeFakeFS()
fSys.WriteFile("/files/app-init.ini", []byte("FOO=bar"))
l := NewFileLoaderAtRoot(validators.MakeFakeValidator(), fSys)
for _, tc := range tests {
kvs, err := l.keyValuesFromFileSources(tc.sources)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if !reflect.DeepEqual(kvs, tc.expected) {
t.Fatalf("in testcase: %q updated:\n%#v\ndoesn't match expected:\n%#v\n", tc.description, kvs, tc.expected)
}
}
}

View File

@@ -31,17 +31,18 @@ import (
// the remote bases will all be root-only restricted.
func NewLoader(
lr LoadRestrictorFunc,
v ifc.Validator,
target string, fSys fs.FileSystem) (ifc.Loader, error) {
repoSpec, err := git.NewRepoSpecFromUrl(target)
if err == nil {
// The target qualifies as a remote git target.
return newLoaderAtGitClone(
repoSpec, fSys, nil, git.ClonerUsingGitExec)
repoSpec, v, fSys, nil, git.ClonerUsingGitExec)
}
root, err := demandDirectoryRoot(fSys, target)
if err != nil {
return nil, err
}
return newLoaderAtConfirmedDir(
lr, root, fSys, nil, git.ClonerUsingGitExec), nil
lr, v, root, fSys, nil, git.ClonerUsingGitExec), nil
}