Add kustomize build {repoUrl}

pkg/commands/build.go

@@ -18,7 +18,6 @@ package commands
 
 import (
 	"io"
-	"path/filepath"
 
 	"github.com/spf13/cobra"
 
@@ -75,17 +74,11 @@ func (o *buildOptions) Validate(args []string) error {
 
 // RunBuild runs build command.
 func (o *buildOptions) RunBuild(out io.Writer, fSys fs.FileSystem) error {
-	l := loader.NewFileLoader(fSys)
-
-	absPath, err := filepath.Abs(o.kustomizationPath)
-	if err != nil {
-		return err
-	}
-
-	rootLoader, err := l.New(absPath)
+	rootLoader, err := loader.NewLoader(o.kustomizationPath, fSys)
 	if err != nil {
 		return err
 	}
+	defer rootLoader.Cleanup()
 
 	application, err := app.NewApplication(rootLoader, fSys)
 	if err != nil {

pkg/commands/diff.go

@@ -19,7 +19,6 @@ package commands
 import (
 	"errors"
 	"io"
-	"path/filepath"
 
 	"github.com/spf13/cobra"
 
@@ -68,17 +67,11 @@ func (o *diffOptions) Validate(args []string) error {
 // RunDiff gets the differences between Application.MakeCustomizedResMap() and Application.MakeUncustomizedResMap().
 func (o *diffOptions) RunDiff(out, errOut io.Writer, fSys fs.FileSystem) error {
 
-	l := loader.NewFileLoader(fSys)
-
-	absPath, err := filepath.Abs(o.kustomizationPath)
-	if err != nil {
-		return err
-	}
-
-	rootLoader, err := l.New(absPath)
+	rootLoader, err := loader.NewLoader(o.kustomizationPath, fSys)
 	if err != nil {
 		return err
 	}
+	defer rootLoader.Cleanup()
 
 	application, err := app.NewApplication(rootLoader, fSys)
 	if err != nil {

pkg/internal/loadertest/fakeloader.go

@@ -63,3 +63,8 @@ func (f FakeLoader) New(newRoot string) (loader.Loader, error) {
 func (f FakeLoader) Load(location string) ([]byte, error) {
 	return f.delegate.Load(location)
 }
+
+// Cleanup does nothing
+func (f FakeLoader) Cleanup() error {
+	return nil
+}

pkg/loader/fileloader.go

@@ -53,6 +53,9 @@ func (l *fileLoader) Root() string {
 // Example: "/home/seans/project" or "/home/seans/project/"
 // NOT "/home/seans/project/file.yaml".
 func (l *fileLoader) New(newRoot string) (Loader, error) {
+	if isRepoUrl(newRoot) {
+		return newGithubLoader(newRoot, l.fSys)
+	}
 	if !l.IsAbsPath(l.root, newRoot) {
 		return nil, fmt.Errorf("Not abs path: l.root='%s', loc='%s'\n", l.root, newRoot)
 	}
@@ -109,3 +112,8 @@ func (l *fileLoader) Load(location string) ([]byte, error) {
 	}
 	return l.fSys.ReadFile(fullLocation)
 }
+
+// Cleanup does nothing
+func (l *fileLoader) Cleanup() error {
+	return nil
+}

pkg/loader/githubloader.go

@@ -0,0 +1,96 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package loader
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/hashicorp/go-getter"
+
+	"github.com/kubernetes-sigs/kustomize/pkg/fs"
+)
+
+// githubLoader loads files from a checkout github repo
+type githubLoader struct {
+	repo        string
+	checkoutDir string
+	fSys        fs.FileSystem
+	loader      *fileLoader
+}
+
+// Root returns the root location for this Loader.
+func (l *githubLoader) Root() string {
+	return l.checkoutDir
+}
+
+// New delegates to fileLoader.New
+func (l *githubLoader) New(newRoot string) (Loader, error) {
+	return l.loader.New(newRoot)
+}
+
+// Load delegates to fileLoader.Load
+func (l *githubLoader) Load(location string) ([]byte, error) {
+	return l.loader.Load(location)
+}
+
+// Cleanup removes the checked out repo
+func (l *githubLoader) Cleanup() error {
+	return os.RemoveAll(l.checkoutDir)
+}
+
+// newGithubLoader returns a new fileLoader with given github Url.
+func newGithubLoader(repoUrl string, fs fs.FileSystem) (*githubLoader, error) {
+	dir, err := ioutil.TempDir("", "kustomize-")
+	if err != nil {
+		return nil, err
+	}
+	target := filepath.Join(dir, "repo")
+	err = checkout(repoUrl, target)
+	if err != nil {
+		return nil, err
+	}
+	l := newFileLoaderAtRoot(target, fs)
+	return &githubLoader{
+		repo:        repoUrl,
+		checkoutDir: target,
+		fSys:        fs,
+		loader:      l,
+	}, nil
+}
+
+// isRepoUrl checks if a string is a repo Url
+func isRepoUrl(s string) bool {
+	return strings.Contains(s, ".com") || strings.Contains(s, ".org") || strings.Contains(s, "https://")
+}
+
+// Checkout clones a github repo with specified commit/tag/branch
+func checkout(url, dir string) error {
+	pwd, err := os.Getwd()
+	if err != nil {
+		return err
+	}
+	client := &getter.Client{
+		Src:  url,
+		Dst:  dir,
+		Pwd:  pwd,
+		Mode: getter.ClientModeDir,
+	}
+	return client.Get()
+}

pkg/loader/loader.go

@@ -17,6 +17,13 @@ limitations under the License.
 // Package loader has a data loading interface and various implementations.
 package loader
 
+import (
+	"fmt"
+	"path/filepath"
+
+	"github.com/kubernetes-sigs/kustomize/pkg/fs"
+)
+
 // Loader interface exposes methods to read bytes.
 type Loader interface {
 	// Root returns the root location for this Loader.
@@ -25,4 +32,29 @@ type Loader interface {
 	New(newRoot string) (Loader, error)
 	// Load returns the bytes read from the location or an error.
 	Load(location string) ([]byte, error)
+	// Cleanup cleans the loader
+	Cleanup() error
+}
+
+// NewLoader returns a Loader given a target
+// The target can be a local disk directory or a github Url
+func NewLoader(target string, fSys fs.FileSystem) (Loader, error) {
+	if isRepoUrl(target) {
+		return newGithubLoader(target, fSys)
+	}
+
+	l := NewFileLoader(fSys)
+	absPath, err := filepath.Abs(target)
+	if err != nil {
+		return nil, err
+	}
+
+	if !l.IsAbsPath(l.root, absPath) {
+		return nil, fmt.Errorf("Not abs path: l.root='%s', loc='%s'\n", l.root, absPath)
+	}
+	root, err := l.fullLocation(l.root, absPath)
+	if err != nil {
+		return nil, err
+	}
+	return newFileLoaderAtRoot(root, l.fSys), nil
 }

vendor/github.com/hashicorp/go-getter/get.go

@@ -63,7 +63,7 @@ func init() {
 		"file":  new(FileGetter),
 		"git":   new(GitGetter),
 		"hg":    new(HgGetter),
-		"s3":    new(S3Getter),
+		// "s3":    new(S3Getter),
 		"http":  httpGetter,
 		"https": httpGetter,
 	}

vendor/github.com/hashicorp/go-getter/get_s3.go

@@ -1,270 +0,0 @@
-package getter
-
-import (
-	"fmt"
-	"io"
-	"net/url"
-	"os"
-	"path/filepath"
-	"strings"
-
-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/aws/credentials"
-	"github.com/aws/aws-sdk-go/aws/credentials/ec2rolecreds"
-	"github.com/aws/aws-sdk-go/aws/ec2metadata"
-	"github.com/aws/aws-sdk-go/aws/session"
-	"github.com/aws/aws-sdk-go/service/s3"
-)
-
-// S3Getter is a Getter implementation that will download a module from
-// a S3 bucket.
-type S3Getter struct{}
-
-func (g *S3Getter) ClientMode(u *url.URL) (ClientMode, error) {
-	// Parse URL
-	region, bucket, path, _, creds, err := g.parseUrl(u)
-	if err != nil {
-		return 0, err
-	}
-
-	// Create client config
-	config := g.getAWSConfig(region, u, creds)
-	sess := session.New(config)
-	client := s3.New(sess)
-
-	// List the object(s) at the given prefix
-	req := &s3.ListObjectsInput{
-		Bucket: aws.String(bucket),
-		Prefix: aws.String(path),
-	}
-	resp, err := client.ListObjects(req)
-	if err != nil {
-		return 0, err
-	}
-
-	for _, o := range resp.Contents {
-		// Use file mode on exact match.
-		if *o.Key == path {
-			return ClientModeFile, nil
-		}
-
-		// Use dir mode if child keys are found.
-		if strings.HasPrefix(*o.Key, path+"/") {
-			return ClientModeDir, nil
-		}
-	}
-
-	// There was no match, so just return file mode. The download is going
-	// to fail but we will let S3 return the proper error later.
-	return ClientModeFile, nil
-}
-
-func (g *S3Getter) Get(dst string, u *url.URL) error {
-	// Parse URL
-	region, bucket, path, _, creds, err := g.parseUrl(u)
-	if err != nil {
-		return err
-	}
-
-	// Remove destination if it already exists
-	_, err = os.Stat(dst)
-	if err != nil && !os.IsNotExist(err) {
-		return err
-	}
-
-	if err == nil {
-		// Remove the destination
-		if err := os.RemoveAll(dst); err != nil {
-			return err
-		}
-	}
-
-	// Create all the parent directories
-	if err := os.MkdirAll(filepath.Dir(dst), 0755); err != nil {
-		return err
-	}
-
-	config := g.getAWSConfig(region, u, creds)
-	sess := session.New(config)
-	client := s3.New(sess)
-
-	// List files in path, keep listing until no more objects are found
-	lastMarker := ""
-	hasMore := true
-	for hasMore {
-		req := &s3.ListObjectsInput{
-			Bucket: aws.String(bucket),
-			Prefix: aws.String(path),
-		}
-		if lastMarker != "" {
-			req.Marker = aws.String(lastMarker)
-		}
-
-		resp, err := client.ListObjects(req)
-		if err != nil {
-			return err
-		}
-
-		hasMore = aws.BoolValue(resp.IsTruncated)
-
-		// Get each object storing each file relative to the destination path
-		for _, object := range resp.Contents {
-			lastMarker = aws.StringValue(object.Key)
-			objPath := aws.StringValue(object.Key)
-
-			// If the key ends with a backslash assume it is a directory and ignore
-			if strings.HasSuffix(objPath, "/") {
-				continue
-			}
-
-			// Get the object destination path
-			objDst, err := filepath.Rel(path, objPath)
-			if err != nil {
-				return err
-			}
-			objDst = filepath.Join(dst, objDst)
-
-			if err := g.getObject(client, objDst, bucket, objPath, ""); err != nil {
-				return err
-			}
-		}
-	}
-
-	return nil
-}
-
-func (g *S3Getter) GetFile(dst string, u *url.URL) error {
-	region, bucket, path, version, creds, err := g.parseUrl(u)
-	if err != nil {
-		return err
-	}
-
-	config := g.getAWSConfig(region, u, creds)
-	sess := session.New(config)
-	client := s3.New(sess)
-	return g.getObject(client, dst, bucket, path, version)
-}
-
-func (g *S3Getter) getObject(client *s3.S3, dst, bucket, key, version string) error {
-	req := &s3.GetObjectInput{
-		Bucket: aws.String(bucket),
-		Key:    aws.String(key),
-	}
-	if version != "" {
-		req.VersionId = aws.String(version)
-	}
-
-	resp, err := client.GetObject(req)
-	if err != nil {
-		return err
-	}
-
-	// Create all the parent directories
-	if err := os.MkdirAll(filepath.Dir(dst), 0755); err != nil {
-		return err
-	}
-
-	f, err := os.Create(dst)
-	if err != nil {
-		return err
-	}
-	defer f.Close()
-
-	_, err = io.Copy(f, resp.Body)
-	return err
-}
-
-func (g *S3Getter) getAWSConfig(region string, url *url.URL, creds *credentials.Credentials) *aws.Config {
-	conf := &aws.Config{}
-	if creds == nil {
-		// Grab the metadata URL
-		metadataURL := os.Getenv("AWS_METADATA_URL")
-		if metadataURL == "" {
-			metadataURL = "http://169.254.169.254:80/latest"
-		}
-
-		creds = credentials.NewChainCredentials(
-			[]credentials.Provider{
-				&credentials.EnvProvider{},
-				&credentials.SharedCredentialsProvider{Filename: "", Profile: ""},
-				&ec2rolecreds.EC2RoleProvider{
-					Client: ec2metadata.New(session.New(&aws.Config{
-						Endpoint: aws.String(metadataURL),
-					})),
-				},
-			})
-	}
-
-	if creds != nil {
-		conf.Endpoint = &url.Host
-		conf.S3ForcePathStyle = aws.Bool(true)
-		if url.Scheme == "http" {
-			conf.DisableSSL = aws.Bool(true)
-		}
-	}
-
-	conf.Credentials = creds
-	if region != "" {
-		conf.Region = aws.String(region)
-	}
-
-	return conf
-}
-
-func (g *S3Getter) parseUrl(u *url.URL) (region, bucket, path, version string, creds *credentials.Credentials, err error) {
-	// This just check whether we are dealing with S3 or
-	// any other S3 compliant service. S3 has a predictable
-	// url as others do not
-	if strings.Contains(u.Host, "amazonaws.com") {
-		// Expected host style: s3.amazonaws.com. They always have 3 parts,
-		// although the first may differ if we're accessing a specific region.
-		hostParts := strings.Split(u.Host, ".")
-		if len(hostParts) != 3 {
-			err = fmt.Errorf("URL is not a valid S3 URL")
-			return
-		}
-
-		// Parse the region out of the first part of the host
-		region = strings.TrimPrefix(strings.TrimPrefix(hostParts[0], "s3-"), "s3")
-		if region == "" {
-			region = "us-east-1"
-		}
-
-		pathParts := strings.SplitN(u.Path, "/", 3)
-		if len(pathParts) != 3 {
-			err = fmt.Errorf("URL is not a valid S3 URL")
-			return
-		}
-
-		bucket = pathParts[1]
-		path = pathParts[2]
-		version = u.Query().Get("version")
-
-	} else {
-		pathParts := strings.SplitN(u.Path, "/", 3)
-		if len(pathParts) != 3 {
-			err = fmt.Errorf("URL is not a valid S3 complaint URL")
-			return
-		}
-		bucket = pathParts[1]
-		path = pathParts[2]
-		version = u.Query().Get("version")
-		region = u.Query().Get("region")
-		if region == "" {
-			region = "us-east-1"
-		}
-	}
-
-	_, hasAwsId := u.Query()["aws_access_key_id"]
-	_, hasAwsSecret := u.Query()["aws_access_key_secret"]
-	_, hasAwsToken := u.Query()["aws_access_token"]
-	if hasAwsId || hasAwsSecret || hasAwsToken {
-		creds = credentials.NewStaticCredentials(
-			u.Query().Get("aws_access_key_id"),
-			u.Query().Get("aws_access_key_secret"),
-			u.Query().Get("aws_access_token"),
-		)
-	}
-
-	return
-}

vendor/github.com/hashicorp/go-getter/get_s3_test.go

@@ -1,250 +0,0 @@
-package getter
-
-import (
-	"net/url"
-	"os"
-	"path/filepath"
-	"testing"
-
-	"github.com/aws/aws-sdk-go/aws/awserr"
-)
-
-func init() {
-	// These are well known restricted IAM keys to a HashiCorp-managed bucket
-	// in a private AWS account that only has access to the open source test
-	// resources.
-	//
-	// We do the string concat below to avoid AWS autodetection of a key. This
-	// key is locked down an IAM policy that is read-only so we're purposely
-	// exposing it.
-	os.Setenv("AWS_ACCESS_KEY", "AKIAITTDR"+"WY2STXOZE2A")
-	os.Setenv("AWS_SECRET_KEY", "oMwSyqdass2kPF"+"/7ORZA9dlb/iegz+89B0Cy01Ea")
-}
-
-func TestS3Getter_impl(t *testing.T) {
-	var _ Getter = new(S3Getter)
-}
-
-func TestS3Getter(t *testing.T) {
-	g := new(S3Getter)
-	dst := tempDir(t)
-
-	// With a dir that doesn't exist
-	err := g.Get(
-		dst, testURL("https://s3.amazonaws.com/hc-oss-test/go-getter/folder"))
-	if err != nil {
-		t.Fatalf("err: %s", err)
-	}
-
-	// Verify the main file exists
-	mainPath := filepath.Join(dst, "main.tf")
-	if _, err := os.Stat(mainPath); err != nil {
-		t.Fatalf("err: %s", err)
-	}
-}
-
-func TestS3Getter_subdir(t *testing.T) {
-	g := new(S3Getter)
-	dst := tempDir(t)
-
-	// With a dir that doesn't exist
-	err := g.Get(
-		dst, testURL("https://s3.amazonaws.com/hc-oss-test/go-getter/folder/subfolder"))
-	if err != nil {
-		t.Fatalf("err: %s", err)
-	}
-
-	// Verify the main file exists
-	subPath := filepath.Join(dst, "sub.tf")
-	if _, err := os.Stat(subPath); err != nil {
-		t.Fatalf("err: %s", err)
-	}
-}
-
-func TestS3Getter_GetFile(t *testing.T) {
-	g := new(S3Getter)
-	dst := tempFile(t)
-
-	// Download
-	err := g.GetFile(
-		dst, testURL("https://s3.amazonaws.com/hc-oss-test/go-getter/folder/main.tf"))
-	if err != nil {
-		t.Fatalf("err: %s", err)
-	}
-
-	// Verify the main file exists
-	if _, err := os.Stat(dst); err != nil {
-		t.Fatalf("err: %s", err)
-	}
-	assertContents(t, dst, "# Main\n")
-}
-
-func TestS3Getter_GetFile_badParams(t *testing.T) {
-	g := new(S3Getter)
-	dst := tempFile(t)
-
-	// Download
-	err := g.GetFile(
-		dst,
-		testURL("https://s3.amazonaws.com/hc-oss-test/go-getter/folder/main.tf?aws_access_key_id=foo&aws_access_key_secret=bar&aws_access_token=baz"))
-	if err == nil {
-		t.Fatalf("expected error, got none")
-	}
-
-	if reqerr, ok := err.(awserr.RequestFailure); !ok || reqerr.StatusCode() != 403 {
-		t.Fatalf("expected InvalidAccessKeyId error")
-	}
-}
-
-func TestS3Getter_GetFile_notfound(t *testing.T) {
-	g := new(S3Getter)
-	dst := tempFile(t)
-
-	// Download
-	err := g.GetFile(
-		dst, testURL("https://s3.amazonaws.com/hc-oss-test/go-getter/folder/404.tf"))
-	if err == nil {
-		t.Fatalf("expected error, got none")
-	}
-}
-
-func TestS3Getter_ClientMode_dir(t *testing.T) {
-	g := new(S3Getter)
-
-	// Check client mode on a key prefix with only a single key.
-	mode, err := g.ClientMode(
-		testURL("https://s3.amazonaws.com/hc-oss-test/go-getter/folder"))
-	if err != nil {
-		t.Fatalf("err: %s", err)
-	}
-	if mode != ClientModeDir {
-		t.Fatal("expect ClientModeDir")
-	}
-}
-
-func TestS3Getter_ClientMode_file(t *testing.T) {
-	g := new(S3Getter)
-
-	// Check client mode on a key prefix which contains sub-keys.
-	mode, err := g.ClientMode(
-		testURL("https://s3.amazonaws.com/hc-oss-test/go-getter/folder/main.tf"))
-	if err != nil {
-		t.Fatalf("err: %s", err)
-	}
-	if mode != ClientModeFile {
-		t.Fatal("expect ClientModeFile")
-	}
-}
-
-func TestS3Getter_ClientMode_notfound(t *testing.T) {
-	g := new(S3Getter)
-
-	// Check the client mode when a non-existent key is looked up. This does not
-	// return an error, but rather should just return the file mode so that S3
-	// can return an appropriate error later on. This also checks that the
-	// prefix is handled properly (e.g., "/fold" and "/folder" don't put the
-	// client mode into "dir".
-	mode, err := g.ClientMode(
-		testURL("https://s3.amazonaws.com/hc-oss-test/go-getter/fold"))
-	if err != nil {
-		t.Fatalf("err: %s", err)
-	}
-	if mode != ClientModeFile {
-		t.Fatal("expect ClientModeFile")
-	}
-}
-
-func TestS3Getter_ClientMode_collision(t *testing.T) {
-	g := new(S3Getter)
-
-	// Check that the client mode is "file" if there is both an object and a
-	// folder with a common prefix (i.e., a "collision" in the namespace).
-	mode, err := g.ClientMode(
-		testURL("https://s3.amazonaws.com/hc-oss-test/go-getter/collision/foo"))
-	if err != nil {
-		t.Fatalf("err: %s", err)
-	}
-	if mode != ClientModeFile {
-		t.Fatal("expect ClientModeFile")
-	}
-}
-
-func TestS3Getter_Url(t *testing.T) {
-	var s3tests = []struct {
-		name    string
-		url     string
-		region  string
-		bucket  string
-		path    string
-		version string
-	}{
-		{
-			name:    "AWSv1234",
-			url:     "s3::https://s3-eu-west-1.amazonaws.com/bucket/foo/bar.baz?version=1234",
-			region:  "eu-west-1",
-			bucket:  "bucket",
-			path:    "foo/bar.baz",
-			version: "1234",
-		},
-		{
-			name:    "localhost-1",
-			url:     "s3::http://127.0.0.1:9000/test-bucket/hello.txt?aws_access_key_id=TESTID&aws_access_key_secret=TestSecret&region=us-east-2&version=1",
-			region:  "us-east-2",
-			bucket:  "test-bucket",
-			path:    "hello.txt",
-			version: "1",
-		},
-		{
-			name:    "localhost-2",
-			url:     "s3::http://127.0.0.1:9000/test-bucket/hello.txt?aws_access_key_id=TESTID&aws_access_key_secret=TestSecret&version=1",
-			region:  "us-east-1",
-			bucket:  "test-bucket",
-			path:    "hello.txt",
-			version: "1",
-		},
-		{
-			name:    "localhost-3",
-			url:     "s3::http://127.0.0.1:9000/test-bucket/hello.txt?aws_access_key_id=TESTID&aws_access_key_secret=TestSecret",
-			region:  "us-east-1",
-			bucket:  "test-bucket",
-			path:    "hello.txt",
-			version: "",
-		},
-	}
-
-	for i, pt := range s3tests {
-		t.Run(pt.name, func(t *testing.T) {
-			g := new(S3Getter)
-			forced, src := getForcedGetter(pt.url)
-			u, err := url.Parse(src)
-
-			if err != nil {
-				t.Errorf("test %d: unexpected error: %s", i, err)
-			}
-			if forced != "s3" {
-				t.Fatalf("expected forced protocol to be s3")
-			}
-
-			region, bucket, path, version, creds, err := g.parseUrl(u)
-
-			if err != nil {
-				t.Fatalf("err: %s", err)
-			}
-			if region != pt.region {
-				t.Fatalf("expected %s, got %s", pt.region, region)
-			}
-			if bucket != pt.bucket {
-				t.Fatalf("expected %s, got %s", pt.bucket, bucket)
-			}
-			if path != pt.path {
-				t.Fatalf("expected %s, got %s", pt.path, path)
-			}
-			if version != pt.version {
-				t.Fatalf("expected %s, got %s", pt.version, version)
-			}
-			if &creds == nil {
-				t.Fatalf("expected to not be nil")
-			}
-		})
-	}
-}