Support various target and resource with go-getter

api/loader/fileloader.go

@@ -95,6 +95,13 @@ type fileLoader struct {
 	// Used to clone repositories.
 	cloner git.Cloner
 
+	// If this is non-nil, the files were
+	// obtained from the given resource
+	rscSpec *resourceSpec
+
+	// Used to get resources
+	getter resourceGetter
+
 	// Used to clean up, as needed.
 	cleaner func() error
 }
@@ -127,20 +134,21 @@ func newLoaderOrDie(
 		log.Fatalf("unable to make loader at '%s'; %v", path, err)
 	}
 	return newLoaderAtConfirmedDir(
-		lr, root, fSys, nil, git.ClonerUsingGitExec)
+		lr, root, fSys, nil, git.ClonerUsingGitExec, getResource)
 }
 
 // newLoaderAtConfirmedDir returns a new fileLoader with given root.
 func newLoaderAtConfirmedDir(
 	lr LoadRestrictorFunc,
 	root filesys.ConfirmedDir, fSys filesys.FileSystem,
-	referrer *fileLoader, cloner git.Cloner) *fileLoader {
+	referrer *fileLoader, cloner git.Cloner, getter resourceGetter) *fileLoader {
 	return &fileLoader{
 		loadRestrictor: lr,
 		root:           root,
 		referrer:       referrer,
 		fSys:           fSys,
 		cloner:         cloner,
+		getter:         getter,
 		cleaner:        func() error { return nil },
 	}
 }
@@ -170,37 +178,44 @@ func (fl *fileLoader) New(path string) (ifc.Loader, error) {
 	if path == "" {
 		return nil, fmt.Errorf("new root cannot be empty")
 	}
-	repoSpec, err := git.NewRepoSpecFromUrl(path)
-	if err == nil {
+
+	ldr, errGet := newResourceGetter(path, fl.fSys, nil, fl.cloner, fl.getter)
+	if errGet == nil {
+		return ldr, nil
+	}
+
+	repoSpec, errGit := git.NewRepoSpecFromUrl(path)
+	if errGit == nil {
 		// Treat this as git repo clone request.
-		if err := fl.errIfRepoCycle(repoSpec); err != nil {
-			return nil, err
+		if errGit := fl.errIfRepoCycle(repoSpec); errGit != nil {
+			return nil, errGit
 		}
 		return newLoaderAtGitClone(
-			repoSpec, fl.fSys, fl, fl.cloner)
+			repoSpec, fl.fSys, fl, fl.cloner, fl.getter)
 	}
+
 	if filepath.IsAbs(path) {
 		return nil, fmt.Errorf("new root '%s' cannot be absolute", path)
 	}
-	root, err := demandDirectoryRoot(fl.fSys, fl.root.Join(path))
-	if err != nil {
-		return nil, err
+	root, errDir := demandDirectoryRoot(fl.fSys, fl.root.Join(path))
+	if errDir != nil {
+		return nil, fmt.Errorf("Error loading %s with git: %v, dir: %v, get: %v", path, errGit, errDir, errGet)
 	}
-	if err := fl.errIfGitContainmentViolation(root); err != nil {
-		return nil, err
+	if errDir := fl.errIfGitContainmentViolation(root); errDir != nil {
+		return nil, errDir
 	}
-	if err := fl.errIfArgEqualOrHigher(root); err != nil {
-		return nil, err
+	if errDir := fl.errIfArgEqualOrHigher(root); errDir != nil {
+		return nil, errDir
 	}
 	return newLoaderAtConfirmedDir(
-		fl.loadRestrictor, root, fl.fSys, fl, fl.cloner), nil
+		fl.loadRestrictor, root, fl.fSys, fl, fl.cloner, fl.getter), nil
 }
 
 // newLoaderAtGitClone returns a new Loader pinned to a temporary
 // directory holding a cloned git repo.
 func newLoaderAtGitClone(
 	repoSpec *git.RepoSpec, fSys filesys.FileSystem,
-	referrer *fileLoader, cloner git.Cloner) (ifc.Loader, error) {
+	referrer *fileLoader, cloner git.Cloner, getter resourceGetter) (ifc.Loader, error) {
 	cleaner := repoSpec.Cleaner(fSys)
 	err := cloner(repoSpec)
 	if err != nil {
@@ -230,6 +245,7 @@ func newLoaderAtGitClone(
 		repoSpec:       repoSpec,
 		fSys:           fSys,
 		cloner:         cloner,
+		getter:         getter,
 		cleaner:        cleaner,
 	}, nil
 }

api/loader/fileloader_test.go

@@ -388,7 +388,7 @@ whatever
 	}
 	l, err := newLoaderAtGitClone(
 		repoSpec, fSys, nil,
-		git.DoNothingCloner(filesys.ConfirmedDir(coRoot)))
+		git.DoNothingCloner(filesys.ConfirmedDir(coRoot)), getNothing)
 	if err != nil {
 		t.Fatalf("unexpected err: %v\n", err)
 	}
@@ -467,7 +467,7 @@ func TestLoaderDisallowsLocalBaseFromRemoteOverlay(t *testing.T) {
 	}
 	l1, err = newLoaderAtGitClone(
 		repoSpec, fSys, nil,
-		git.DoNothingCloner(filesys.ConfirmedDir(cloneRoot)))
+		git.DoNothingCloner(filesys.ConfirmedDir(cloneRoot)), getNothing)
 	if err != nil {
 		t.Fatalf("unexpected err: %v\n", err)
 	}
@@ -506,7 +506,7 @@ func TestLocalLoaderReferencingGitBase(t *testing.T) {
 	}
 	l1 := newLoaderAtConfirmedDir(
 		RestrictionRootOnly, root, fSys, nil,
-		git.DoNothingCloner(filesys.ConfirmedDir(cloneRoot)))
+		git.DoNothingCloner(filesys.ConfirmedDir(cloneRoot)), getNothing)
 	if l1.Root() != topDir {
 		t.Fatalf("unexpected root %s", l1.Root())
 	}
@@ -532,7 +532,7 @@ func TestRepoDirectCycleDetection(t *testing.T) {
 	}
 	l1 := newLoaderAtConfirmedDir(
 		RestrictionRootOnly, root, fSys, nil,
-		git.DoNothingCloner(filesys.ConfirmedDir(cloneRoot)))
+		git.DoNothingCloner(filesys.ConfirmedDir(cloneRoot)), getNothing)
 	p1 := "github.com/someOrg/someRepo/foo"
 	rs1, err := git.NewRepoSpecFromUrl(p1)
 	if err != nil {
@@ -561,7 +561,7 @@ func TestRepoIndirectCycleDetection(t *testing.T) {
 	}
 	l0 := newLoaderAtConfirmedDir(
 		RestrictionRootOnly, root, fSys, nil,
-		git.DoNothingCloner(filesys.ConfirmedDir(cloneRoot)))
+		git.DoNothingCloner(filesys.ConfirmedDir(cloneRoot)), getNothing)
 
 	p1 := "github.com/someOrg/someRepo1"
 	p2 := "github.com/someOrg/someRepo2"

api/loader/loader.go

@@ -5,6 +5,8 @@
 package loader
 
 import (
+	"fmt"
+
 	"sigs.k8s.io/kustomize/api/filesys"
 	"sigs.k8s.io/kustomize/api/ifc"
 	"sigs.k8s.io/kustomize/api/internal/git"
@@ -19,16 +21,23 @@ import (
 func NewLoader(
 	lr LoadRestrictorFunc,
 	target string, fSys filesys.FileSystem) (ifc.Loader, error) {
-	repoSpec, err := git.NewRepoSpecFromUrl(target)
-	if err == nil {
+
+	ldr, errGet := newResourceGetter(target, fSys, nil, git.ClonerUsingGitExec, getResource)
+	if errGet == nil {
+		return ldr, nil
+	}
+
+	repoSpec, errGit := git.NewRepoSpecFromUrl(target)
+	if errGit == nil {
 		// The target qualifies as a remote git target.
 		return newLoaderAtGitClone(
-			repoSpec, fSys, nil, git.ClonerUsingGitExec)
+			repoSpec, fSys, nil, git.ClonerUsingGitExec, getResource)
 	}
-	root, err := demandDirectoryRoot(fSys, target)
-	if err != nil {
-		return nil, err
+
+	root, errDir := demandDirectoryRoot(fSys, target)
+	if errDir == nil {
+		return newLoaderAtConfirmedDir(lr, root, fSys, nil, git.ClonerUsingGitExec, getResource), nil
 	}
-	return newLoaderAtConfirmedDir(
-		lr, root, fSys, nil, git.ClonerUsingGitExec), nil
+
+	return nil, fmt.Errorf("Error creating new loader with git: %v, dir: %v, get: %v", errGit, errDir, errGet)
 }

api/loader/resource.go

@@ -0,0 +1,103 @@
+// Copyright 2020 The Kubernetes Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package loader
+
+import (
+	"context"
+	"log"
+	"os"
+
+	"github.com/yujunz/go-getter"
+	"sigs.k8s.io/kustomize/api/filesys"
+	"sigs.k8s.io/kustomize/api/ifc"
+	"sigs.k8s.io/kustomize/api/internal/git"
+)
+
+type resourceSpec struct {
+	// Raw is the original resource in kustomization.yaml
+	Raw string
+
+	// Dir is where the resource is saved
+	Dir filesys.ConfirmedDir
+}
+
+// Getter is a function that can gets resource
+type resourceGetter func(rs *resourceSpec) error
+
+func newResourceGetter(raw string, fSys filesys.FileSystem, referrer *fileLoader, cloner git.Cloner, getter resourceGetter) (ifc.Loader, error) {
+	rs := &resourceSpec{
+		Raw: raw,
+	}
+
+	cleaner := func() error {
+		return fSys.RemoveAll(rs.Dir.String())
+	}
+
+	if err := getter(rs); err != nil {
+		cleaner()
+		return nil, err
+	}
+
+	// TODO(yujunz): check file or directory
+
+	return &fileLoader{
+		loadRestrictor: RestrictionRootOnly,
+		// TODO(yujunz): limit to getter root
+		root:     rs.Dir,
+		referrer: referrer,
+		fSys:     fSys,
+		cloner:   cloner,
+		rscSpec:  rs,
+		getter:   getter,
+		cleaner:  cleaner,
+	}, nil
+}
+
+func getResource(rs *resourceSpec) error {
+	var err error
+
+	rs.Dir, err = filesys.NewTmpConfirmedDir()
+	if err != nil {
+		return err
+	}
+
+	// Get the pwd
+	pwd, err := os.Getwd()
+	if err != nil {
+		log.Fatalf("Error getting wd: %s", err)
+	}
+
+	opts := []getter.ClientOption{}
+	client := &getter.Client{
+		Ctx:  context.TODO(),
+		Src:  rs.Raw,
+		Dst:  rs.Dir.String(),
+		Pwd:  pwd,
+		Mode: getter.ClientModeAny,
+		Detectors: []getter.Detector{
+			new(getter.GitHubDetector),
+			new(getter.GitDetector),
+			new(getter.BitBucketDetector),
+		},
+		Options: opts,
+	}
+	return client.Get()
+}
+
+func getNothing(rs *resourceSpec) error {
+	var err error
+	rs.Dir, err = filesys.NewTmpConfirmedDir()
+	if err != nil {
+		return err
+	}
+
+	// Get the pwd
+	pwd, err := os.Getwd()
+	if err != nil {
+		log.Fatalf("Error getting wd: %s", err)
+	}
+
+	_, err = getter.Detect(rs.Raw, pwd, []getter.Detector{})
+	return err
+}