diff --git a/pkg/commands/build.go b/pkg/commands/build.go index 3b5941803..41b24cc00 100644 --- a/pkg/commands/build.go +++ b/pkg/commands/build.go @@ -18,7 +18,6 @@ package commands import ( "io" - "path/filepath" "github.com/spf13/cobra" @@ -75,17 +74,11 @@ func (o *buildOptions) Validate(args []string) error { // RunBuild runs build command. func (o *buildOptions) RunBuild(out io.Writer, fSys fs.FileSystem) error { - l := loader.NewFileLoader(fSys) - - absPath, err := filepath.Abs(o.kustomizationPath) - if err != nil { - return err - } - - rootLoader, err := l.New(absPath) + rootLoader, err := loader.NewLoader(o.kustomizationPath, fSys) if err != nil { return err } + defer rootLoader.Cleanup() application, err := app.NewApplication(rootLoader, fSys) if err != nil { diff --git a/pkg/commands/diff.go b/pkg/commands/diff.go index b7f372ae8..8b9053b1d 100644 --- a/pkg/commands/diff.go +++ b/pkg/commands/diff.go @@ -19,7 +19,6 @@ package commands import ( "errors" "io" - "path/filepath" "github.com/spf13/cobra" @@ -68,17 +67,11 @@ func (o *diffOptions) Validate(args []string) error { // RunDiff gets the differences between Application.MakeCustomizedResMap() and Application.MakeUncustomizedResMap(). func (o *diffOptions) RunDiff(out, errOut io.Writer, fSys fs.FileSystem) error { - l := loader.NewFileLoader(fSys) - - absPath, err := filepath.Abs(o.kustomizationPath) - if err != nil { - return err - } - - rootLoader, err := l.New(absPath) + rootLoader, err := loader.NewLoader(o.kustomizationPath, fSys) if err != nil { return err } + defer rootLoader.Cleanup() application, err := app.NewApplication(rootLoader, fSys) if err != nil { diff --git a/pkg/internal/loadertest/fakeloader.go b/pkg/internal/loadertest/fakeloader.go index 793c3e2e3..be3ac56b3 100644 --- a/pkg/internal/loadertest/fakeloader.go +++ b/pkg/internal/loadertest/fakeloader.go @@ -63,3 +63,8 @@ func (f FakeLoader) New(newRoot string) (loader.Loader, error) { func (f FakeLoader) Load(location string) ([]byte, error) { return f.delegate.Load(location) } + +// Cleanup does nothing +func (f FakeLoader) Cleanup() error { + return nil +} diff --git a/pkg/loader/fileloader.go b/pkg/loader/fileloader.go index 2d614305e..2fe58479d 100644 --- a/pkg/loader/fileloader.go +++ b/pkg/loader/fileloader.go @@ -53,6 +53,9 @@ func (l *fileLoader) Root() string { // Example: "/home/seans/project" or "/home/seans/project/" // NOT "/home/seans/project/file.yaml". func (l *fileLoader) New(newRoot string) (Loader, error) { + if isRepoUrl(newRoot) { + return newGithubLoader(newRoot, l.fSys) + } if !l.IsAbsPath(l.root, newRoot) { return nil, fmt.Errorf("Not abs path: l.root='%s', loc='%s'\n", l.root, newRoot) } @@ -109,3 +112,8 @@ func (l *fileLoader) Load(location string) ([]byte, error) { } return l.fSys.ReadFile(fullLocation) } + +// Cleanup does nothing +func (l *fileLoader) Cleanup() error { + return nil +} diff --git a/pkg/loader/githubloader.go b/pkg/loader/githubloader.go new file mode 100644 index 000000000..e48551d1b --- /dev/null +++ b/pkg/loader/githubloader.go @@ -0,0 +1,96 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package loader + +import ( + "io/ioutil" + "os" + "path/filepath" + "strings" + + "github.com/hashicorp/go-getter" + + "github.com/kubernetes-sigs/kustomize/pkg/fs" +) + +// githubLoader loads files from a checkout github repo +type githubLoader struct { + repo string + checkoutDir string + fSys fs.FileSystem + loader *fileLoader +} + +// Root returns the root location for this Loader. +func (l *githubLoader) Root() string { + return l.checkoutDir +} + +// New delegates to fileLoader.New +func (l *githubLoader) New(newRoot string) (Loader, error) { + return l.loader.New(newRoot) +} + +// Load delegates to fileLoader.Load +func (l *githubLoader) Load(location string) ([]byte, error) { + return l.loader.Load(location) +} + +// Cleanup removes the checked out repo +func (l *githubLoader) Cleanup() error { + return os.RemoveAll(l.checkoutDir) +} + +// newGithubLoader returns a new fileLoader with given github Url. +func newGithubLoader(repoUrl string, fs fs.FileSystem) (*githubLoader, error) { + dir, err := ioutil.TempDir("", "kustomize-") + if err != nil { + return nil, err + } + target := filepath.Join(dir, "repo") + err = checkout(repoUrl, target) + if err != nil { + return nil, err + } + l := newFileLoaderAtRoot(target, fs) + return &githubLoader{ + repo: repoUrl, + checkoutDir: target, + fSys: fs, + loader: l, + }, nil +} + +// isRepoUrl checks if a string is a repo Url +func isRepoUrl(s string) bool { + return strings.Contains(s, ".com") || strings.Contains(s, ".org") || strings.Contains(s, "https://") +} + +// Checkout clones a github repo with specified commit/tag/branch +func checkout(url, dir string) error { + pwd, err := os.Getwd() + if err != nil { + return err + } + client := &getter.Client{ + Src: url, + Dst: dir, + Pwd: pwd, + Mode: getter.ClientModeDir, + } + return client.Get() +} diff --git a/pkg/loader/loader.go b/pkg/loader/loader.go index 2db035089..74c5444c1 100644 --- a/pkg/loader/loader.go +++ b/pkg/loader/loader.go @@ -17,6 +17,13 @@ limitations under the License. // Package loader has a data loading interface and various implementations. package loader +import ( + "fmt" + "path/filepath" + + "github.com/kubernetes-sigs/kustomize/pkg/fs" +) + // Loader interface exposes methods to read bytes. type Loader interface { // Root returns the root location for this Loader. @@ -25,4 +32,29 @@ type Loader interface { New(newRoot string) (Loader, error) // Load returns the bytes read from the location or an error. Load(location string) ([]byte, error) + // Cleanup cleans the loader + Cleanup() error +} + +// NewLoader returns a Loader given a target +// The target can be a local disk directory or a github Url +func NewLoader(target string, fSys fs.FileSystem) (Loader, error) { + if isRepoUrl(target) { + return newGithubLoader(target, fSys) + } + + l := NewFileLoader(fSys) + absPath, err := filepath.Abs(target) + if err != nil { + return nil, err + } + + if !l.IsAbsPath(l.root, absPath) { + return nil, fmt.Errorf("Not abs path: l.root='%s', loc='%s'\n", l.root, absPath) + } + root, err := l.fullLocation(l.root, absPath) + if err != nil { + return nil, err + } + return newFileLoaderAtRoot(root, l.fSys), nil } diff --git a/vendor/github.com/hashicorp/go-getter/get.go b/vendor/github.com/hashicorp/go-getter/get.go index e6053d934..c5b6dd453 100644 --- a/vendor/github.com/hashicorp/go-getter/get.go +++ b/vendor/github.com/hashicorp/go-getter/get.go @@ -63,7 +63,7 @@ func init() { "file": new(FileGetter), "git": new(GitGetter), "hg": new(HgGetter), - "s3": new(S3Getter), + // "s3": new(S3Getter), "http": httpGetter, "https": httpGetter, } diff --git a/vendor/github.com/hashicorp/go-getter/get_s3.go b/vendor/github.com/hashicorp/go-getter/get_s3.go deleted file mode 100644 index ebb321741..000000000 --- a/vendor/github.com/hashicorp/go-getter/get_s3.go +++ /dev/null @@ -1,270 +0,0 @@ -package getter - -import ( - "fmt" - "io" - "net/url" - "os" - "path/filepath" - "strings" - - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/credentials" - "github.com/aws/aws-sdk-go/aws/credentials/ec2rolecreds" - "github.com/aws/aws-sdk-go/aws/ec2metadata" - "github.com/aws/aws-sdk-go/aws/session" - "github.com/aws/aws-sdk-go/service/s3" -) - -// S3Getter is a Getter implementation that will download a module from -// a S3 bucket. -type S3Getter struct{} - -func (g *S3Getter) ClientMode(u *url.URL) (ClientMode, error) { - // Parse URL - region, bucket, path, _, creds, err := g.parseUrl(u) - if err != nil { - return 0, err - } - - // Create client config - config := g.getAWSConfig(region, u, creds) - sess := session.New(config) - client := s3.New(sess) - - // List the object(s) at the given prefix - req := &s3.ListObjectsInput{ - Bucket: aws.String(bucket), - Prefix: aws.String(path), - } - resp, err := client.ListObjects(req) - if err != nil { - return 0, err - } - - for _, o := range resp.Contents { - // Use file mode on exact match. - if *o.Key == path { - return ClientModeFile, nil - } - - // Use dir mode if child keys are found. - if strings.HasPrefix(*o.Key, path+"/") { - return ClientModeDir, nil - } - } - - // There was no match, so just return file mode. The download is going - // to fail but we will let S3 return the proper error later. - return ClientModeFile, nil -} - -func (g *S3Getter) Get(dst string, u *url.URL) error { - // Parse URL - region, bucket, path, _, creds, err := g.parseUrl(u) - if err != nil { - return err - } - - // Remove destination if it already exists - _, err = os.Stat(dst) - if err != nil && !os.IsNotExist(err) { - return err - } - - if err == nil { - // Remove the destination - if err := os.RemoveAll(dst); err != nil { - return err - } - } - - // Create all the parent directories - if err := os.MkdirAll(filepath.Dir(dst), 0755); err != nil { - return err - } - - config := g.getAWSConfig(region, u, creds) - sess := session.New(config) - client := s3.New(sess) - - // List files in path, keep listing until no more objects are found - lastMarker := "" - hasMore := true - for hasMore { - req := &s3.ListObjectsInput{ - Bucket: aws.String(bucket), - Prefix: aws.String(path), - } - if lastMarker != "" { - req.Marker = aws.String(lastMarker) - } - - resp, err := client.ListObjects(req) - if err != nil { - return err - } - - hasMore = aws.BoolValue(resp.IsTruncated) - - // Get each object storing each file relative to the destination path - for _, object := range resp.Contents { - lastMarker = aws.StringValue(object.Key) - objPath := aws.StringValue(object.Key) - - // If the key ends with a backslash assume it is a directory and ignore - if strings.HasSuffix(objPath, "/") { - continue - } - - // Get the object destination path - objDst, err := filepath.Rel(path, objPath) - if err != nil { - return err - } - objDst = filepath.Join(dst, objDst) - - if err := g.getObject(client, objDst, bucket, objPath, ""); err != nil { - return err - } - } - } - - return nil -} - -func (g *S3Getter) GetFile(dst string, u *url.URL) error { - region, bucket, path, version, creds, err := g.parseUrl(u) - if err != nil { - return err - } - - config := g.getAWSConfig(region, u, creds) - sess := session.New(config) - client := s3.New(sess) - return g.getObject(client, dst, bucket, path, version) -} - -func (g *S3Getter) getObject(client *s3.S3, dst, bucket, key, version string) error { - req := &s3.GetObjectInput{ - Bucket: aws.String(bucket), - Key: aws.String(key), - } - if version != "" { - req.VersionId = aws.String(version) - } - - resp, err := client.GetObject(req) - if err != nil { - return err - } - - // Create all the parent directories - if err := os.MkdirAll(filepath.Dir(dst), 0755); err != nil { - return err - } - - f, err := os.Create(dst) - if err != nil { - return err - } - defer f.Close() - - _, err = io.Copy(f, resp.Body) - return err -} - -func (g *S3Getter) getAWSConfig(region string, url *url.URL, creds *credentials.Credentials) *aws.Config { - conf := &aws.Config{} - if creds == nil { - // Grab the metadata URL - metadataURL := os.Getenv("AWS_METADATA_URL") - if metadataURL == "" { - metadataURL = "http://169.254.169.254:80/latest" - } - - creds = credentials.NewChainCredentials( - []credentials.Provider{ - &credentials.EnvProvider{}, - &credentials.SharedCredentialsProvider{Filename: "", Profile: ""}, - &ec2rolecreds.EC2RoleProvider{ - Client: ec2metadata.New(session.New(&aws.Config{ - Endpoint: aws.String(metadataURL), - })), - }, - }) - } - - if creds != nil { - conf.Endpoint = &url.Host - conf.S3ForcePathStyle = aws.Bool(true) - if url.Scheme == "http" { - conf.DisableSSL = aws.Bool(true) - } - } - - conf.Credentials = creds - if region != "" { - conf.Region = aws.String(region) - } - - return conf -} - -func (g *S3Getter) parseUrl(u *url.URL) (region, bucket, path, version string, creds *credentials.Credentials, err error) { - // This just check whether we are dealing with S3 or - // any other S3 compliant service. S3 has a predictable - // url as others do not - if strings.Contains(u.Host, "amazonaws.com") { - // Expected host style: s3.amazonaws.com. They always have 3 parts, - // although the first may differ if we're accessing a specific region. - hostParts := strings.Split(u.Host, ".") - if len(hostParts) != 3 { - err = fmt.Errorf("URL is not a valid S3 URL") - return - } - - // Parse the region out of the first part of the host - region = strings.TrimPrefix(strings.TrimPrefix(hostParts[0], "s3-"), "s3") - if region == "" { - region = "us-east-1" - } - - pathParts := strings.SplitN(u.Path, "/", 3) - if len(pathParts) != 3 { - err = fmt.Errorf("URL is not a valid S3 URL") - return - } - - bucket = pathParts[1] - path = pathParts[2] - version = u.Query().Get("version") - - } else { - pathParts := strings.SplitN(u.Path, "/", 3) - if len(pathParts) != 3 { - err = fmt.Errorf("URL is not a valid S3 complaint URL") - return - } - bucket = pathParts[1] - path = pathParts[2] - version = u.Query().Get("version") - region = u.Query().Get("region") - if region == "" { - region = "us-east-1" - } - } - - _, hasAwsId := u.Query()["aws_access_key_id"] - _, hasAwsSecret := u.Query()["aws_access_key_secret"] - _, hasAwsToken := u.Query()["aws_access_token"] - if hasAwsId || hasAwsSecret || hasAwsToken { - creds = credentials.NewStaticCredentials( - u.Query().Get("aws_access_key_id"), - u.Query().Get("aws_access_key_secret"), - u.Query().Get("aws_access_token"), - ) - } - - return -} diff --git a/vendor/github.com/hashicorp/go-getter/get_s3_test.go b/vendor/github.com/hashicorp/go-getter/get_s3_test.go deleted file mode 100644 index 2d9da14cb..000000000 --- a/vendor/github.com/hashicorp/go-getter/get_s3_test.go +++ /dev/null @@ -1,250 +0,0 @@ -package getter - -import ( - "net/url" - "os" - "path/filepath" - "testing" - - "github.com/aws/aws-sdk-go/aws/awserr" -) - -func init() { - // These are well known restricted IAM keys to a HashiCorp-managed bucket - // in a private AWS account that only has access to the open source test - // resources. - // - // We do the string concat below to avoid AWS autodetection of a key. This - // key is locked down an IAM policy that is read-only so we're purposely - // exposing it. - os.Setenv("AWS_ACCESS_KEY", "AKIAITTDR"+"WY2STXOZE2A") - os.Setenv("AWS_SECRET_KEY", "oMwSyqdass2kPF"+"/7ORZA9dlb/iegz+89B0Cy01Ea") -} - -func TestS3Getter_impl(t *testing.T) { - var _ Getter = new(S3Getter) -} - -func TestS3Getter(t *testing.T) { - g := new(S3Getter) - dst := tempDir(t) - - // With a dir that doesn't exist - err := g.Get( - dst, testURL("https://s3.amazonaws.com/hc-oss-test/go-getter/folder")) - if err != nil { - t.Fatalf("err: %s", err) - } - - // Verify the main file exists - mainPath := filepath.Join(dst, "main.tf") - if _, err := os.Stat(mainPath); err != nil { - t.Fatalf("err: %s", err) - } -} - -func TestS3Getter_subdir(t *testing.T) { - g := new(S3Getter) - dst := tempDir(t) - - // With a dir that doesn't exist - err := g.Get( - dst, testURL("https://s3.amazonaws.com/hc-oss-test/go-getter/folder/subfolder")) - if err != nil { - t.Fatalf("err: %s", err) - } - - // Verify the main file exists - subPath := filepath.Join(dst, "sub.tf") - if _, err := os.Stat(subPath); err != nil { - t.Fatalf("err: %s", err) - } -} - -func TestS3Getter_GetFile(t *testing.T) { - g := new(S3Getter) - dst := tempFile(t) - - // Download - err := g.GetFile( - dst, testURL("https://s3.amazonaws.com/hc-oss-test/go-getter/folder/main.tf")) - if err != nil { - t.Fatalf("err: %s", err) - } - - // Verify the main file exists - if _, err := os.Stat(dst); err != nil { - t.Fatalf("err: %s", err) - } - assertContents(t, dst, "# Main\n") -} - -func TestS3Getter_GetFile_badParams(t *testing.T) { - g := new(S3Getter) - dst := tempFile(t) - - // Download - err := g.GetFile( - dst, - testURL("https://s3.amazonaws.com/hc-oss-test/go-getter/folder/main.tf?aws_access_key_id=foo&aws_access_key_secret=bar&aws_access_token=baz")) - if err == nil { - t.Fatalf("expected error, got none") - } - - if reqerr, ok := err.(awserr.RequestFailure); !ok || reqerr.StatusCode() != 403 { - t.Fatalf("expected InvalidAccessKeyId error") - } -} - -func TestS3Getter_GetFile_notfound(t *testing.T) { - g := new(S3Getter) - dst := tempFile(t) - - // Download - err := g.GetFile( - dst, testURL("https://s3.amazonaws.com/hc-oss-test/go-getter/folder/404.tf")) - if err == nil { - t.Fatalf("expected error, got none") - } -} - -func TestS3Getter_ClientMode_dir(t *testing.T) { - g := new(S3Getter) - - // Check client mode on a key prefix with only a single key. - mode, err := g.ClientMode( - testURL("https://s3.amazonaws.com/hc-oss-test/go-getter/folder")) - if err != nil { - t.Fatalf("err: %s", err) - } - if mode != ClientModeDir { - t.Fatal("expect ClientModeDir") - } -} - -func TestS3Getter_ClientMode_file(t *testing.T) { - g := new(S3Getter) - - // Check client mode on a key prefix which contains sub-keys. - mode, err := g.ClientMode( - testURL("https://s3.amazonaws.com/hc-oss-test/go-getter/folder/main.tf")) - if err != nil { - t.Fatalf("err: %s", err) - } - if mode != ClientModeFile { - t.Fatal("expect ClientModeFile") - } -} - -func TestS3Getter_ClientMode_notfound(t *testing.T) { - g := new(S3Getter) - - // Check the client mode when a non-existent key is looked up. This does not - // return an error, but rather should just return the file mode so that S3 - // can return an appropriate error later on. This also checks that the - // prefix is handled properly (e.g., "/fold" and "/folder" don't put the - // client mode into "dir". - mode, err := g.ClientMode( - testURL("https://s3.amazonaws.com/hc-oss-test/go-getter/fold")) - if err != nil { - t.Fatalf("err: %s", err) - } - if mode != ClientModeFile { - t.Fatal("expect ClientModeFile") - } -} - -func TestS3Getter_ClientMode_collision(t *testing.T) { - g := new(S3Getter) - - // Check that the client mode is "file" if there is both an object and a - // folder with a common prefix (i.e., a "collision" in the namespace). - mode, err := g.ClientMode( - testURL("https://s3.amazonaws.com/hc-oss-test/go-getter/collision/foo")) - if err != nil { - t.Fatalf("err: %s", err) - } - if mode != ClientModeFile { - t.Fatal("expect ClientModeFile") - } -} - -func TestS3Getter_Url(t *testing.T) { - var s3tests = []struct { - name string - url string - region string - bucket string - path string - version string - }{ - { - name: "AWSv1234", - url: "s3::https://s3-eu-west-1.amazonaws.com/bucket/foo/bar.baz?version=1234", - region: "eu-west-1", - bucket: "bucket", - path: "foo/bar.baz", - version: "1234", - }, - { - name: "localhost-1", - url: "s3::http://127.0.0.1:9000/test-bucket/hello.txt?aws_access_key_id=TESTID&aws_access_key_secret=TestSecret®ion=us-east-2&version=1", - region: "us-east-2", - bucket: "test-bucket", - path: "hello.txt", - version: "1", - }, - { - name: "localhost-2", - url: "s3::http://127.0.0.1:9000/test-bucket/hello.txt?aws_access_key_id=TESTID&aws_access_key_secret=TestSecret&version=1", - region: "us-east-1", - bucket: "test-bucket", - path: "hello.txt", - version: "1", - }, - { - name: "localhost-3", - url: "s3::http://127.0.0.1:9000/test-bucket/hello.txt?aws_access_key_id=TESTID&aws_access_key_secret=TestSecret", - region: "us-east-1", - bucket: "test-bucket", - path: "hello.txt", - version: "", - }, - } - - for i, pt := range s3tests { - t.Run(pt.name, func(t *testing.T) { - g := new(S3Getter) - forced, src := getForcedGetter(pt.url) - u, err := url.Parse(src) - - if err != nil { - t.Errorf("test %d: unexpected error: %s", i, err) - } - if forced != "s3" { - t.Fatalf("expected forced protocol to be s3") - } - - region, bucket, path, version, creds, err := g.parseUrl(u) - - if err != nil { - t.Fatalf("err: %s", err) - } - if region != pt.region { - t.Fatalf("expected %s, got %s", pt.region, region) - } - if bucket != pt.bucket { - t.Fatalf("expected %s, got %s", pt.bucket, bucket) - } - if path != pt.path { - t.Fatalf("expected %s, got %s", pt.path, path) - } - if version != pt.version { - t.Fatalf("expected %s, got %s", pt.version, version) - } - if &creds == nil { - t.Fatalf("expected to not be nil") - } - }) - } -}