Merge pull request #398 from monopole/injectDecoder

Introduce k8sdeps package to isolate k8s deps.

pkg/commands/build.go

@@ -20,6 +20,7 @@ import (
 	"errors"
 	"io"
 	"log"
+	"sigs.k8s.io/kustomize/pkg/ifc"
 	"strings"
 
 	"github.com/spf13/cobra"
@@ -57,7 +58,9 @@ Use different transformer configurations by passing files to kustomize
 `
 
 // newCmdBuild creates a new build command.
-func newCmdBuild(out io.Writer, fs fs.FileSystem) *cobra.Command {
+func newCmdBuild(
+	out io.Writer, fs fs.FileSystem,
+	decoder ifc.Decoder) *cobra.Command {
 	var o buildOptions
 	var p string
 
@@ -71,7 +74,7 @@ func newCmdBuild(out io.Writer, fs fs.FileSystem) *cobra.Command {
 			if err != nil {
 				return err
 			}
-			return o.RunBuild(out, fs)
+			return o.RunBuild(out, fs, decoder)
 		},
 	}
 	cmd.Flags().StringVarP(
@@ -114,14 +117,18 @@ func (o *buildOptions) Validate(args []string, p string, fs fs.FileSystem) error
 }
 
 // RunBuild runs build command.
-func (o *buildOptions) RunBuild(out io.Writer, fSys fs.FileSystem) error {
+func (o *buildOptions) RunBuild(
+	out io.Writer, fSys fs.FileSystem,
+	decoder ifc.Decoder) error {
 	rootLoader, err := loader.NewLoader(o.kustomizationPath, "", fSys)
 	if err != nil {
 		return err
 	}
 	defer rootLoader.Cleanup()
 	kt, err := target.NewKustTarget(
-		rootLoader, fSys, makeTransformerconfig(fSys, o.transformerconfigPaths))
+		rootLoader, fSys,
+		makeTransformerconfig(fSys, o.transformerconfigPaths),
+		decoder)
 	if err != nil {
 		return err
 	}

pkg/commands/build_test.go

@@ -22,6 +22,7 @@ import (
 	"os"
 	"path/filepath"
 	"reflect"
+	"sigs.k8s.io/kustomize/pkg/internal/k8sdeps"
 	"strings"
 	"testing"
 
@@ -124,7 +125,7 @@ func runBuildTestCase(t *testing.T, testcaseName string, updateKustomizeExpected
 		kustomizationPath: testcase.Filename,
 	}
 	buf := bytes.NewBuffer([]byte{})
-	err = ops.RunBuild(buf, fSys)
+	err = ops.RunBuild(buf, fSys, k8sdeps.NewKustDecoder())
 	switch {
 	case err != nil && len(testcase.ExpectedError) == 0:
 		t.Errorf("unexpected error: %v", err)

pkg/commands/commands.go

@@ -20,6 +20,7 @@ package commands
 import (
 	"flag"
 	"os"
+	"sigs.k8s.io/kustomize/pkg/internal/k8sdeps"
 
 	"github.com/spf13/cobra"
 	"sigs.k8s.io/kustomize/pkg/fs"
@@ -43,7 +44,7 @@ See https://sigs.k8s.io/kustomize
 
 	c.AddCommand(
 		// TODO: Make consistent API for newCmd* functions.
-		newCmdBuild(stdOut, fsys),
+		newCmdBuild(stdOut, fsys, k8sdeps.NewKustDecoder()),
 		newCmdEdit(fsys),
 		newCmdConfig(fsys),
 		newCmdVersion(stdOut),

pkg/ifc/ifc.go

@@ -0,0 +1,26 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package ifc holds miscellaneous interfaces used by kustomize.
+package ifc
+
+// Decoder unmarshalls byte input into an object.
+type Decoder interface {
+	// SetInput accepts new input.
+	SetInput([]byte)
+	// Decode yields the next object from the input, else io.EOF
+	Decode(interface{}) error
+}

pkg/internal/k8sdeps/decoder.go

@@ -0,0 +1,47 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package k8sdeps
+
+import (
+	"bytes"
+	"errors"
+	"k8s.io/apimachinery/pkg/util/yaml"
+)
+
+// KustDecoder unmarshalls bytes to objects.
+type KustDecoder struct {
+	d *yaml.YAMLOrJSONDecoder
+}
+
+// NewKustDecoder returns a new KustDecoder.
+func NewKustDecoder() *KustDecoder {
+	return &KustDecoder{}
+}
+
+// SetInput initializes an apimachinery decoder.
+func (k *KustDecoder) SetInput(in []byte) {
+	k.d = yaml.NewYAMLOrJSONDecoder(
+		bytes.NewReader(in), 1024)
+}
+
+// Decode delegates to the apimachinery decoder.
+func (k *KustDecoder) Decode(into interface{}) error {
+	if k.d == nil {
+		return errors.New("no decoder")
+	}
+	return k.d.Decode(into)
+}

pkg/internal/k8sdeps/doc.go

@@ -0,0 +1,76 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// It's possible that kustomize's features will be vendored into
+// the kubernetes/kubernetes repo and made available to kubectl
+// commands, while at the same time the kustomize program will
+// continue to exist as an independent CLI.  Vendoring snapshots
+// would be taken just before a kubectl release.
+//
+// This creates a problem in that freestanding-kustomize depends on
+// (for example):
+//
+//   https://github.com/kubernetes/apimachinery/
+//       tree/master/pkg/util/yaml
+//
+// It vendors that package into
+//   sigs.k8s.io/kustomize/vendor/k8s.io/apimachinery/
+//
+// Whereas kubectl-kustomize would have to depend on the "staging"
+// version of this code, located at
+//
+//   https://github.com/kubernetes/kubernetes/
+//       blob/master/staging/src/k8s.io/apimachinery/pkg/util/yaml
+//
+// which is "vendored" via symlinks:
+//   k8s.io/kubernetes/vendor/k8s.io/apimachinery
+// is a symlink to
+//   ../../staging/src/k8s.io/apimachinery
+//
+// The staging version is the canonical, under-development
+// version of the code that kubectl depends on, whereas the packages
+// at kubernetes/apimachinery are periodic snapshots of staging made
+// for outside tools to depend on.
+//
+// apimachinery isn't the only package that poses this problem, just
+// using it as a specific example.
+//
+// The kubectl binary cannot vendor in kustomize code that in
+// turn vendors in the non-staging packages.
+//
+// One way to fix some of this would be to copy code - a hard fork.
+// This has all the problems associated with a hard forking.
+//
+// Another way would be to break the kustomize repo into three:
+//
+// (1) kustomize - repo with the main() function,
+//     vendoring (2) and (3).
+//
+// (2) kustomize-libs - packages used by (1) with no
+//     apimachinery dependence.
+//
+// (3) kustomize-k8sdeps - A thin code layer that depends
+//     on (vendors) apimachinery to provide thin implementations
+//     to interfaces used in (2).
+//
+// The kubectl repo would then vendor from (2) only, and have
+// a local implementation of (3).  With that in mind, it's clear
+// that (3) doesn't have to be a repo; the kustomize version of
+// the thin layer can live directly in (1).
+//
+// This package is the code in (3), meant for kustomize.
+
+package k8sdeps

pkg/resmap/resmap.go

@@ -21,6 +21,7 @@ import (
 	"bytes"
 	"fmt"
 	"reflect"
+	"sigs.k8s.io/kustomize/pkg/ifc"
 	"sort"
 
 	"github.com/ghodss/yaml"
@@ -151,14 +152,16 @@ func (m ResMap) FilterBy(inputId resource.ResId) ResMap {
 }
 
 // NewResMapFromFiles returns a ResMap given a resource path slice.
-func NewResMapFromFiles(loader loader.Loader, paths []string) (ResMap, error) {
+func NewResMapFromFiles(
+	loader loader.Loader, paths []string,
+	d ifc.Decoder) (ResMap, error) {
 	var result []ResMap
 	for _, path := range paths {
 		content, err := loader.Load(path)
 		if err != nil {
 			return nil, errors.Wrap(err, "Load from path "+path+" failed")
 		}
-		res, err := newResMapFromBytes(content)
+		res, err := newResMapFromBytes(content, d)
 		if err != nil {
 			return nil, internal.Handler(err, path)
 		}
@@ -168,8 +171,8 @@ func NewResMapFromFiles(loader loader.Loader, paths []string) (ResMap, error) {
 }
 
 // newResMapFromBytes decodes a list of objects in byte array format.
-func newResMapFromBytes(b []byte) (ResMap, error) {
+func newResMapFromBytes(b []byte, d ifc.Decoder) (ResMap, error) {
-	resources, err := resource.NewResourceSliceFromBytes(b)
+	resources, err := resource.NewResourceSliceFromBytes(b, d)
 	if err != nil {
 		return nil, err
 	}

pkg/resmap/resmap_test.go

@@ -19,6 +19,7 @@ package resmap
 import (
 	"fmt"
 	"reflect"
+	"sigs.k8s.io/kustomize/pkg/internal/k8sdeps"
 	"testing"
 
 	"sigs.k8s.io/kustomize/pkg/gvk"
@@ -106,7 +107,9 @@ metadata:
 			}),
 	}
 
-	m, _ := NewResMapFromFiles(l, []string{"/home/seans/project/deployment.yaml"})
+	m, _ := NewResMapFromFiles(
+		l, []string{"/home/seans/project/deployment.yaml"},
+		k8sdeps.NewKustDecoder())
 	if len(m) != 2 {
 		t.Fatalf("%#v should contain 2 appResource, but got %d", m, len(m))
 	}
@@ -145,7 +148,7 @@ metadata:
 				},
 			}),
 	}
-	m, err := newResMapFromBytes(encoded)
+	m, err := newResMapFromBytes(encoded, k8sdeps.NewKustDecoder())
 	fmt.Printf("%v\n", m)
 	if err != nil {
 		t.Fatalf("unexpected error: %v", err)

pkg/resource/resource.go

@@ -18,7 +18,6 @@ limitations under the License.
 package resource
 
 import (
-	"bytes"
 	"encoding/json"
 	"fmt"
 	"io"
@@ -27,8 +26,8 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime"
-	"k8s.io/apimachinery/pkg/util/yaml"
 	"sigs.k8s.io/kustomize/pkg/gvk"
+	"sigs.k8s.io/kustomize/pkg/ifc"
 	internal "sigs.k8s.io/kustomize/pkg/internal/error"
 	"sigs.k8s.io/kustomize/pkg/loader"
 	"sigs.k8s.io/kustomize/pkg/patch"
@@ -68,14 +67,15 @@ func NewResourceFromUnstruct(u unstructured.Unstructured) *Resource {
 // NewResourceSliceFromPatches returns a slice of resources given a patch path
 // slice from a kustomization file.
 func NewResourceSliceFromPatches(
-	ldr loader.Loader, paths []patch.StrategicMerge) ([]*Resource, error) {
+	ldr loader.Loader, paths []patch.StrategicMerge,
+	decoder ifc.Decoder) ([]*Resource, error) {
 	var result []*Resource
 	for _, path := range paths {
 		content, err := ldr.Load(string(path))
 		if err != nil {
 			return nil, err
 		}
-		res, err := NewResourceSliceFromBytes(content)
+		res, err := NewResourceSliceFromBytes(content, decoder)
 		if err != nil {
 			return nil, internal.Handler(err, string(path))
 		}
@@ -85,8 +85,9 @@ func NewResourceSliceFromPatches(
 }
 
 // NewResourceSliceFromBytes unmarshalls bytes into a Resource slice.
-func NewResourceSliceFromBytes(in []byte) ([]*Resource, error) {
+func NewResourceSliceFromBytes(
-	decoder := yaml.NewYAMLOrJSONDecoder(bytes.NewReader(in), 1024)
+	in []byte, decoder ifc.Decoder) ([]*Resource, error) {
+	decoder.SetInput(in)
 	var result []*Resource
 	var err error
 	for err == nil || isEmptyYamlError(err) {

pkg/resource/resource_test.go

@@ -18,6 +18,7 @@ package resource
 
 import (
 	"reflect"
+	"sigs.k8s.io/kustomize/pkg/internal/k8sdeps"
 	"testing"
 
 	"sigs.k8s.io/kustomize/pkg/internal/loadertest"
@@ -121,7 +122,8 @@ WOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOT: woot
 		},
 	}
 	for _, test := range tests {
-		rs, err := NewResourceSliceFromPatches(l, test.input)
+		rs, err := NewResourceSliceFromPatches(
+			l, test.input, k8sdeps.NewKustDecoder())
 		if test.expectedErr && err == nil {
 			t.Fatalf("%v: should return error", test.name)
 		}
@@ -211,7 +213,8 @@ WOOOOOOOOOOOOOOOOOOOOOOOOT:  woot
 	}
 
 	for _, test := range tests {
-		rs, err := NewResourceSliceFromBytes(test.input)
+		rs, err := NewResourceSliceFromBytes(
+			test.input, k8sdeps.NewKustDecoder())
 		if test.expectedErr && err == nil {
 			t.Fatalf("%v: should return error", test.name)
 		}

pkg/target/kusttarget.go

@@ -21,6 +21,7 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
+	"sigs.k8s.io/kustomize/pkg/ifc"
 
 	"github.com/ghodss/yaml"
 	"github.com/golang/glog"
@@ -43,6 +44,7 @@ import (
 // KustTarget encapsulates the entirety of a kustomization build.
 type KustTarget struct {
 	kustomization *types.Kustomization
+	decoder       ifc.Decoder
 	ldr           loader.Loader
 	fSys          fs.FileSystem
 	tcfg          *transformerconfig.TransformerConfig
@@ -51,7 +53,8 @@ type KustTarget struct {
 // NewKustTarget returns a new instance of KustTarget primed with a Loader.
 func NewKustTarget(
 	ldr loader.Loader, fSys fs.FileSystem,
-	tcfg *transformerconfig.TransformerConfig) (*KustTarget, error) {
+	tcfg *transformerconfig.TransformerConfig,
+	d ifc.Decoder) (*KustTarget, error) {
 	content, err := ldr.Load(constants.KustomizationFileName)
 	if err != nil {
 		return nil, err
@@ -67,6 +70,7 @@ func NewKustTarget(
 		ldr:           ldr,
 		fSys:          fSys,
 		tcfg:          tcfg,
+		decoder:       d,
 	}, nil
 }
 
@@ -157,7 +161,7 @@ func (kt *KustTarget) loadCustomizedResMap() (resmap.ResMap, error) {
 		kt.kustomization.PatchesStrategicMerge,
 		kt.kustomization.Patches...)
 	patches, err := resource.NewResourceSliceFromPatches(
-		kt.ldr, kt.kustomization.PatchesStrategicMerge)
+		kt.ldr, kt.kustomization.PatchesStrategicMerge, kt.decoder)
 	if err != nil {
 		errs.Append(errors.Wrap(err, "NewResourceSliceFromPatches"))
 	}
@@ -194,7 +198,8 @@ func (kt *KustTarget) loadCustomizedResMap() (resmap.ResMap, error) {
 // Gets Bases and Resources as advertised.
 func (kt *KustTarget) loadResMapFromBasesAndResources() (resmap.ResMap, error) {
 	bases, errs := kt.loadCustomizedBases()
-	resources, err := resmap.NewResMapFromFiles(kt.ldr, kt.kustomization.Resources)
+	resources, err := resmap.NewResMapFromFiles(
+		kt.ldr, kt.kustomization.Resources, kt.decoder)
 	if err != nil {
 		errs.Append(errors.Wrap(err, "rawResources failed to read Resources"))
 	}
@@ -215,7 +220,7 @@ func (kt *KustTarget) loadCustomizedBases() (resmap.ResMap, *interror.Kustomizat
 			errs.Append(errors.Wrap(err, "couldn't make ldr for "+path))
 			continue
 		}
-		target, err := NewKustTarget(ldr, kt.fSys, kt.tcfg)
+		target, err := NewKustTarget(ldr, kt.fSys, kt.tcfg, kt.decoder)
 		if err != nil {
 			errs.Append(errors.Wrap(err, "couldn't make target for "+path))
 			continue
@@ -244,7 +249,7 @@ func (kt *KustTarget) loadBasesAsFlatList() ([]*KustTarget, error) {
 			errs.Append(err)
 			continue
 		}
-		target, err := NewKustTarget(ldr, kt.fSys, kt.tcfg)
+		target, err := NewKustTarget(ldr, kt.fSys, kt.tcfg, kt.decoder)
 		if err != nil {
 			errs.Append(err)
 			continue

pkg/target/kusttarget_test.go

@@ -19,6 +19,7 @@ package target
 import (
 	"encoding/base64"
 	"reflect"
+	"sigs.k8s.io/kustomize/pkg/internal/k8sdeps"
 	"strings"
 	"testing"
 
@@ -204,7 +205,9 @@ func TestResources1(t *testing.T) {
 	l := makeLoader1(t)
 	fakeFs := fs.MakeFakeFS()
 	fakeFs.Mkdir("/")
-	kt, err := NewKustTarget(l, fakeFs, transformerconfig.MakeDefaultTransformerConfig())
+	kt, err := NewKustTarget(
+		l, fakeFs, transformerconfig.MakeDefaultTransformerConfig(),
+		k8sdeps.NewKustDecoder())
 	if err != nil {
 		t.Fatalf("unexpected construction error %v", err)
 	}
@@ -227,7 +230,9 @@ func TestResourceNotFound(t *testing.T) {
 	}
 	fakeFs := fs.MakeFakeFS()
 	fakeFs.Mkdir("/")
-	kt, err := NewKustTarget(l, fakeFs, transformerconfig.MakeDefaultTransformerConfig())
+	kt, err := NewKustTarget(
+		l, fakeFs, transformerconfig.MakeDefaultTransformerConfig(),
+		k8sdeps.NewKustDecoder())
 	if err != nil {
 		t.Fatalf("Unexpected construction error %v", err)
 	}
@@ -248,7 +253,9 @@ func TestSecretTimeout(t *testing.T) {
 	}
 	fakeFs := fs.MakeFakeFS()
 	fakeFs.Mkdir("/")
-	kt, err := NewKustTarget(l, fakeFs, transformerconfig.MakeDefaultTransformerConfig())
+	kt, err := NewKustTarget(
+		l, fakeFs, transformerconfig.MakeDefaultTransformerConfig(),
+		k8sdeps.NewKustDecoder())
 	if err != nil {
 		t.Fatalf("Unexpected construction error %v", err)
 	}