From a094be45d9418598d93ea261b134d2e762959f16 Mon Sep 17 00:00:00 2001 From: Jingfang Liu Date: Fri, 16 Nov 2018 13:27:20 -0800 Subject: [PATCH] update vendor_kustomize.sh with run-in-gopath.sh (#545) --- build/vendor_kustomize.diff | 281 ++++++++++++++++++++++++++++++++++++ build/vendor_kustomize.sh | 150 +++---------------- 2 files changed, 302 insertions(+), 129 deletions(-) create mode 100755 build/vendor_kustomize.diff diff --git a/build/vendor_kustomize.diff b/build/vendor_kustomize.diff new file mode 100755 index 000000000..b116f0f3f --- /dev/null +++ b/build/vendor_kustomize.diff @@ -0,0 +1,281 @@ +commit 1b893558aa83ac6491e5ba416b493170a9045fec +Author: Jingfang Liu +Date: Mon Nov 12 10:26:12 2018 -0800 + + last change + +diff --git a/staging/src/k8s.io/cli-runtime/artifacts/kustomization/configMap.yaml b/staging/src/k8s.io/cli-runtime/artifacts/kustomization/configMap.yaml +new file mode 100644 +index 0000000000..0008853094 +--- /dev/null ++++ b/staging/src/k8s.io/cli-runtime/artifacts/kustomization/configMap.yaml +@@ -0,0 +1,8 @@ ++ ++apiVersion: v1 ++kind: ConfigMap ++metadata: ++ name: the-map ++data: ++ altGreeting: "Good Morning!" ++ enableRisky: "false" +diff --git a/staging/src/k8s.io/cli-runtime/artifacts/kustomization/deployment.yaml b/staging/src/k8s.io/cli-runtime/artifacts/kustomization/deployment.yaml +new file mode 100644 +index 0000000000..6e79409080 +--- /dev/null ++++ b/staging/src/k8s.io/cli-runtime/artifacts/kustomization/deployment.yaml +@@ -0,0 +1,30 @@ ++apiVersion: apps/v1 ++kind: Deployment ++metadata: ++ name: the-deployment ++spec: ++ replicas: 3 ++ template: ++ metadata: ++ labels: ++ deployment: hello ++ spec: ++ containers: ++ - name: the-container ++ image: monopole/hello:1 ++ command: ["/hello", ++ "--port=8080", ++ "--enableRiskyFeature=$(ENABLE_RISKY)"] ++ ports: ++ - containerPort: 8080 ++ env: ++ - name: ALT_GREETING ++ valueFrom: ++ configMapKeyRef: ++ name: the-map ++ key: altGreeting ++ - name: ENABLE_RISKY ++ valueFrom: ++ configMapKeyRef: ++ name: the-map ++ key: enableRisky +diff --git a/staging/src/k8s.io/cli-runtime/artifacts/kustomization/kustomization.yaml b/staging/src/k8s.io/cli-runtime/artifacts/kustomization/kustomization.yaml +new file mode 100644 +index 0000000000..6e1e3202d5 +--- /dev/null ++++ b/staging/src/k8s.io/cli-runtime/artifacts/kustomization/kustomization.yaml +@@ -0,0 +1,5 @@ ++nameprefix: test- ++ resources: ++- deployment.yaml ++- service.yaml ++- configMap.yaml +diff --git a/staging/src/k8s.io/cli-runtime/artifacts/kustomization/service.yaml b/staging/src/k8s.io/cli-runtime/artifacts/kustomization/service.yaml +new file mode 100644 +index 0000000000..2942cdb7df +--- /dev/null ++++ b/staging/src/k8s.io/cli-runtime/artifacts/kustomization/service.yaml +@@ -0,0 +1,13 @@ ++kind: Service ++apiVersion: v1 ++metadata: ++ name: the-service ++spec: ++ selector: ++ deployment: hello ++ type: LoadBalancer ++ ports: ++ - protocol: TCP ++ port: 8666 ++ targetPort: 8080 ++ +\ No newline at end of file +diff --git a/staging/src/k8s.io/cli-runtime/pkg/genericclioptions/resource/BUILD b/staging/src/k8s.io/cli-runtime/pkg/genericclioptions/resource/BUILD +index 22b34de008..b91d1c0130 100644 +--- a/staging/src/k8s.io/cli-runtime/pkg/genericclioptions/resource/BUILD ++++ b/staging/src/k8s.io/cli-runtime/pkg/genericclioptions/resource/BUILD +@@ -35,12 +35,15 @@ go_library( + "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/yaml:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/watch:go_default_library", ++ "//staging/src/k8s.io/cli-runtime/pkg/kustomize/k8sdeps:go_default_library", + "//staging/src/k8s.io/client-go/discovery:go_default_library", + "//staging/src/k8s.io/client-go/kubernetes/scheme:go_default_library", + "//staging/src/k8s.io/client-go/rest:go_default_library", + "//staging/src/k8s.io/client-go/restmapper:go_default_library", + "//vendor/golang.org/x/text/encoding/unicode:go_default_library", + "//vendor/golang.org/x/text/transform:go_default_library", ++ "//vendor/sigs.k8s.io/kustomize/pkg/commands/build:go_default_library", ++ "//vendor/sigs.k8s.io/kustomize/pkg/fs:go_default_library", + ], + ) + +diff --git a/staging/src/k8s.io/cli-runtime/pkg/genericclioptions/resource/builder_test.go b/staging/src/k8s.io/cli-runtime/pkg/genericclioptions/resource/builder_test.go +index 7fd526b33c..801f13f772 100644 +--- a/staging/src/k8s.io/cli-runtime/pkg/genericclioptions/resource/builder_test.go ++++ b/staging/src/k8s.io/cli-runtime/pkg/genericclioptions/resource/builder_test.go +@@ -465,27 +465,48 @@ func TestPathBuilderWithMultipleInvalid(t *testing.T) { + } + + func TestDirectoryBuilder(t *testing.T) { +- b := newDefaultBuilder(). +- FilenameParam(false, &FilenameOptions{Recursive: false, Filenames: []string{"../../../artifacts/guestbook"}}). +- NamespaceParam("test").DefaultNamespace() ++ tests := []struct { ++ directories []string ++ singleItem bool ++ number int ++ expectedNames []string ++ }{ ++ {[]string{"../../../artifacts/guestbook"}, false, 3, []string{"redis-master"}}, ++ {[]string{"../../../artifacts/kustomization"}, true, 3, []string{"test-the-deployment"}}, ++ {[]string{"../../../artifacts/guestbook", "../../../artifacts/kustomization"}, false, 6, []string{"redis-master", "test-the-deployment"}}, ++ } + +- test := &testVisitor{} +- singleItemImplied := false ++ for _, tt := range tests { ++ b := newDefaultBuilder(). ++ FilenameParam(false, &FilenameOptions{Recursive: false, Filenames: tt.directories}). ++ NamespaceParam("test").DefaultNamespace() + +- err := b.Do().IntoSingleItemImplied(&singleItemImplied).Visit(test.Handle) +- if err != nil || singleItemImplied || len(test.Infos) < 3 { +- t.Fatalf("unexpected response: %v %t %#v", err, singleItemImplied, test.Infos) +- } ++ test := &testVisitor{} ++ singleItemImplied := false + +- found := false +- for _, info := range test.Infos { +- if info.Name == "redis-master" && info.Namespace == "test" && info.Object != nil { +- found = true +- break ++ err := b.Do().IntoSingleItemImplied(&singleItemImplied).Visit(test.Handle) ++ if err != nil || singleItemImplied != tt.singleItem || len(test.Infos) < tt.number { ++ t.Fatalf("unexpected response: %v %t %#v", err, singleItemImplied, test.Infos) ++ } ++ ++ contained := func(name string) bool { ++ for _, info := range test.Infos { ++ if info.Name == name && info.Namespace == "test" && info.Object != nil { ++ return true ++ } ++ } ++ return false ++ } ++ ++ allFound := true ++ for _, name := range tt.expectedNames { ++ if !contained(name) { ++ allFound = false ++ } ++ } ++ if !allFound { ++ t.Errorf("unexpected responses: %#v", test.Infos) + } +- } +- if !found { +- t.Errorf("unexpected responses: %#v", test.Infos) + } + } + +diff --git a/staging/src/k8s.io/cli-runtime/pkg/genericclioptions/resource/visitor.go b/staging/src/k8s.io/cli-runtime/pkg/genericclioptions/resource/visitor.go +index 32c1a691a5..d7a37e1cde 100644 +--- a/staging/src/k8s.io/cli-runtime/pkg/genericclioptions/resource/visitor.go ++++ b/staging/src/k8s.io/cli-runtime/pkg/genericclioptions/resource/visitor.go +@@ -20,10 +20,12 @@ import ( + "bytes" + "fmt" + "io" ++ "io/ioutil" + "net/http" + "net/url" + "os" + "path/filepath" ++ "strings" + "time" + + "golang.org/x/text/encoding/unicode" +@@ -38,6 +40,9 @@ import ( + utilerrors "k8s.io/apimachinery/pkg/util/errors" + "k8s.io/apimachinery/pkg/util/yaml" + "k8s.io/apimachinery/pkg/watch" ++ "k8s.io/cli-runtime/pkg/kustomize/k8sdeps" ++ "sigs.k8s.io/kustomize/pkg/commands/build" ++ "sigs.k8s.io/kustomize/pkg/fs" + ) + + const ( +@@ -452,7 +457,10 @@ func ExpandPathsToFileVisitors(mapper *mapper, paths string, recursive bool, ext + if err != nil { + return err + } +- ++ if isKustomizationDir(path) { ++ visitors = append(visitors, NewKustomizationVisitor(mapper, path, schema)) ++ return filepath.SkipDir ++ } + if fi.IsDir() { + if path != paths && !recursive { + return filepath.SkipDir +@@ -463,7 +471,10 @@ func ExpandPathsToFileVisitors(mapper *mapper, paths string, recursive bool, ext + if path != paths && ignoreFile(path, extensions) { + return nil + } +- ++ if strings.HasSuffix(path, "kustomization.yaml") { ++ visitors = append(visitors, NewKustomizationVisitor(mapper, filepath.Dir(path), schema)) ++ return nil ++ } + visitor := &FileVisitor{ + Path: path, + StreamVisitor: NewStreamVisitor(nil, mapper, path, schema), +@@ -479,6 +490,14 @@ func ExpandPathsToFileVisitors(mapper *mapper, paths string, recursive bool, ext + return visitors, nil + } + ++func isKustomizationDir(path string) bool { ++ if _, err := os.Stat(filepath.Join(path, "kustomization.yaml")); err == nil { ++ return true ++ } ++ return false ++} ++ ++ + // FileVisitor is wrapping around a StreamVisitor, to handle open/close files + type FileVisitor struct { + Path string +@@ -507,6 +526,37 @@ func (v *FileVisitor) Visit(fn VisitorFunc) error { + return v.StreamVisitor.Visit(fn) + } + ++// KustomizationVisitor prorvides the output of kustomization build ++type KustomizationVisitor struct { ++ Path string ++ *StreamVisitor ++} ++ ++// Visit in a KustomizationVisitor build the kustomization output ++func (v *KustomizationVisitor) Visit(fn VisitorFunc) error { ++ fSys := fs.MakeRealFS() ++ f := k8sdeps.NewFactory() ++ var out bytes.Buffer ++ cmd := build.NewCmdBuild(&out, fSys, f.ResmapF, f.TransformerF) ++ cmd.SetArgs([]string{v.Path}) ++ // we want to silence usage, error output, and any future output from cobra ++ // we will get error output as a golang error from execute ++ cmd.SetOutput(ioutil.Discard) ++ _, err := cmd.ExecuteC() ++ if err != nil { ++ return err ++ } ++ v.StreamVisitor.Reader = bytes.NewReader(out.Bytes()) ++ return v.StreamVisitor.Visit(fn) ++} ++ ++func NewKustomizationVisitor(mapper *mapper, path string, schema ContentValidator) *KustomizationVisitor { ++ return &KustomizationVisitor{ ++ Path: path, ++ StreamVisitor: NewStreamVisitor(nil, mapper, path, schema), ++ } ++} ++ + // StreamVisitor reads objects from an io.Reader and walks them. A stream visitor can only be + // visited once. + // TODO: depends on objects being in JSON format before being passed to decode - need to implement diff --git a/build/vendor_kustomize.sh b/build/vendor_kustomize.sh index 6efa167e7..71b9ac54c 100755 --- a/build/vendor_kustomize.sh +++ b/build/vendor_kustomize.sh @@ -17,6 +17,8 @@ set -e set -x +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" + # vendor_kustomize.sh creates the change in kubernetes repo for vendoring kustomize function setUpWorkspace { @@ -26,27 +28,27 @@ function setUpWorkspace { } function cloneK8s { - mkdir -p $GOPATH/src/k8s.io - cd $GOPATH/src/k8s.io + mkdir -p $KPATH/src/k8s.io + cd $KPATH/src/k8s.io git clone git@github.com:kubernetes/kubernetes.git } function godepRestore { - cd $GOPATH/src/k8s.io/kubernetes + cd $KPATH/src/k8s.io/kubernetes # restore dependencies - hack/godep-restore.sh + hack/run-in-gopath.sh hack/godep-restore.sh } function getKustomizeDeps { # get Kustomize and Kustomize dependencies - godep get sigs.k8s.io/kustomize/pkg/commands - godep get github.com/bgentry/go-netrc/netrc - godep get github.com/hashicorp/go-cleanhttp - godep get github.com/hashicorp/go-getter - godep get github.com/hashicorp/go-safetemp - godep get github.com/hashicorp/go-version + hack/run-in-gopath.sh godep get sigs.k8s.io/kustomize/pkg/commands + hack/run-in-gopath.sh godep get github.com/bgentry/go-netrc/netrc + hack/run-in-gopath.sh godep get github.com/hashicorp/go-cleanhttp + hack/run-in-gopath.sh godep get github.com/hashicorp/go-getter + hack/run-in-gopath.sh godep get github.com/hashicorp/go-safetemp + hack/run-in-gopath.sh godep get github.com/hashicorp/go-version # The hashes below passed bin/pre-commit.sh with kustomize HEAD at time of merger. DEPS=( @@ -61,7 +63,7 @@ function getKustomizeDeps { ) function foo { - cd $GOPATH/src/github.com/$1 + cd $KPATH/src/k8s.io/kubernetes/_output/local/go/src/github.com/$1 git checkout $2 } for i in "${DEPS[@]}"; do @@ -71,139 +73,34 @@ function getKustomizeDeps { function updateK8s { # Copy k8sdeps from Kustomize to cli-runtime in staging - mkdir -p $GOPATH/src/k8s.io/kubernetes/staging/src/k8s.io/cli-runtime/pkg/kustomize - cp -r $GOPATH/src/sigs.k8s.io/kustomize/k8sdeps \ - $GOPATH/src/k8s.io/kubernetes/staging/src/k8s.io/cli-runtime/pkg/kustomize/k8sdeps + mkdir -p $KPATH/src/k8s.io/kubernetes/staging/src/k8s.io/cli-runtime/pkg/kustomize + cp -r $KPATH/src/k8s.io/kubernetes/_output/local/go/src/sigs.k8s.io/kustomize/k8sdeps \ + $KPATH/src/k8s.io/kubernetes/staging/src/k8s.io/cli-runtime/pkg/kustomize/k8sdeps # Change import path of k8sdeps - find $GOPATH/src/k8s.io/kubernetes/staging/src/k8s.io/cli-runtime/pkg/kustomize/k8sdeps \ + find $KPATH/src/k8s.io/kubernetes/staging/src/k8s.io/cli-runtime/pkg/kustomize/k8sdeps \ -type f -name "*.go" | \ xargs sed -i \ 's!sigs.k8s.io/kustomize/k8sdeps!k8s.io/cli-runtime/pkg/kustomize/k8sdeps!' # Add kustomize command to kubectl - cat > $GOPATH/visitor.diff << EOF -diff --git a/staging/src/k8s.io/cli-runtime/pkg/genericclioptions/resource/visitor.go b/staging/src/k8s.io/cli-runtime/pkg/genericclioptions/resource/visitor.go -index 32c1a691a5..d7a37e1cde 100644 ---- a/staging/src/k8s.io/cli-runtime/pkg/genericclioptions/resource/visitor.go -+++ b/staging/src/k8s.io/cli-runtime/pkg/genericclioptions/resource/visitor.go -@@ -20,10 +20,12 @@ import ( - "bytes" - "fmt" - "io" -+ "io/ioutil" - "net/http" - "net/url" - "os" - "path/filepath" -+ "strings" - "time" - - "golang.org/x/text/encoding/unicode" -@@ -38,6 +40,9 @@ import ( - utilerrors "k8s.io/apimachinery/pkg/util/errors" - "k8s.io/apimachinery/pkg/util/yaml" - "k8s.io/apimachinery/pkg/watch" -+ "k8s.io/cli-runtime/pkg/kustomize/k8sdeps" -+ "sigs.k8s.io/kustomize/pkg/commands/build" -+ "sigs.k8s.io/kustomize/pkg/fs" - ) - - const ( -@@ -452,7 +457,10 @@ func ExpandPathsToFileVisitors(mapper *mapper, paths string, recursive bool, ext - if err != nil { - return err - } -- -+ if isKustomizationDir(path) { -+ visitors = append(visitors, NewKustomizationVisitor(mapper, path, schema)) -+ return filepath.SkipDir -+ } - if fi.IsDir() { - if path != paths && !recursive { - return filepath.SkipDir -@@ -463,7 +471,10 @@ func ExpandPathsToFileVisitors(mapper *mapper, paths string, recursive bool, ext - if path != paths && ignoreFile(path, extensions) { - return nil - } -- -+ if strings.HasSuffix(path, "kustomization.yaml") { -+ visitors = append(visitors, NewKustomizationVisitor(mapper, filepath.Dir(path), schema)) -+ return nil -+ } - visitor := &FileVisitor{ - Path: path, - StreamVisitor: NewStreamVisitor(nil, mapper, path, schema), -@@ -479,6 +490,14 @@ func ExpandPathsToFileVisitors(mapper *mapper, paths string, recursive bool, ext - return visitors, nil - } - -+func isKustomizationDir(path string) bool { -+ if _, err := os.Stat(filepath.Join(path, "kustomization.yaml")); err == nil { -+ return true -+ } -+ return false -+} -+ -+ - // FileVisitor is wrapping around a StreamVisitor, to handle open/close files - type FileVisitor struct { - Path string -@@ -507,6 +526,37 @@ func (v *FileVisitor) Visit(fn VisitorFunc) error { - return v.StreamVisitor.Visit(fn) - } - -+// KustomizationVisitor prorvides the output of kustomization build -+type KustomizationVisitor struct { -+ Path string -+ *StreamVisitor -+} -+ -+// Visit in a KustomizationVisitor build the kustomization output -+func (v *KustomizationVisitor) Visit(fn VisitorFunc) error { -+ fSys := fs.MakeRealFS() -+ f := k8sdeps.NewFactory() -+ var out bytes.Buffer -+ cmd := build.NewCmdBuild(&out, fSys, f.ResmapF, f.TransformerF) -+ cmd.SetArgs([]string{v.Path}) -+ // we want to silence usage, error output, and any future output from cobra -+ // we will get error output as a golang error from execute -+ cmd.SetOutput(ioutil.Discard) -+ _, err := cmd.ExecuteC() -+ if err != nil { -+ return err -+ } -+ v.StreamVisitor.Reader = bytes.NewReader(out.Bytes()) -+ return v.StreamVisitor.Visit(fn) -+} -+ -+func NewKustomizationVisitor(mapper *mapper, path string, schema ContentValidator) *KustomizationVisitor { -+ return &KustomizationVisitor{ -+ Path: path, -+ StreamVisitor: NewStreamVisitor(nil, mapper, path, schema), -+ } -+} -+ - // StreamVisitor reads objects from an io.Reader and walks them. A stream visitor can only be - // visited once. - // TODO: depends on objects being in JSON format before being passed to decode - need to implement -EOF + cp $DIR/vendor_kustomize.diff $KPATH/vendor_kustomize.diff cd $GOPATH/src/k8s.io/kubernetes - git apply --ignore-space-change --ignore-whitespace $GOPATH/visitor.diff + git apply --ignore-space-change --ignore-whitespace $KPATH/vendor_kustomize.diff } function godepSave { # Save all dependencies into k8s.io/kubernetes/vendor by running # hack/godep-save.sh - ./hack/godep-save.sh + hack/run-in-gopath.sh hack/godep-save.sh } function verify { # make sure in k8s.io/kubernetes/vendor/sigs.k8s.io/kustomize # there is no internal package - test 0 == $(ls $GOPATH/src/k8s.io/kubernetes/vendor/sigs.k8s.io/kustomize | grep “internal” | wc -l) + test 0 == $(ls $KPATH/src/k8s.io/kubernetes/vendor/sigs.k8s.io/kustomize | grep “internal” | wc -l) # Make sure it compiles. test 0 == $(bazel build cmd/kubectl:kubectl) @@ -212,10 +109,6 @@ function verify { echo "The change for vendoring kustomize is ready in $GOPATH/src/k8s.io/kubernetes.\n Next step, open a PR for it.\n" } -function updateDocs { - ./hack/update-generated-docs.sh -} - setUpWorkspace cloneK8s godepRestore @@ -223,4 +116,3 @@ getKustomizeDeps updateK8s godepSave verify -updateDocs