From 2bcf82c6a4cedfa23e922d3b1d28bc3ee3cf4c14 Mon Sep 17 00:00:00 2001 From: Phani Teja Marupaka Date: Wed, 15 Jan 2020 16:58:45 -0800 Subject: [PATCH 01/46] Exclude invalid resources from status --- cmd/resource/status/cmd/util.go | 22 +++++++++------ cmd/resource/status/cmd/util_test.go | 40 ++++++++++++++++++++++++++++ 2 files changed, 54 insertions(+), 8 deletions(-) create mode 100644 cmd/resource/status/cmd/util_test.go diff --git a/cmd/resource/status/cmd/util.go b/cmd/resource/status/cmd/util.go index bd745191f..6071f145d 100644 --- a/cmd/resource/status/cmd/util.go +++ b/cmd/resource/status/cmd/util.go @@ -64,14 +64,20 @@ func (f *CaptureIdentifiersFilter) Filter(slice []*yaml.RNode) ([]*yaml.RNode, e if err != nil { return nil, err } - f.Identifiers = append(f.Identifiers, wait.ResourceIdentifier{ - Name: id.Name, - Namespace: id.Namespace, - GroupKind: schema.GroupKind{ - Group: gv.Group, - Kind: id.Kind, - }, - }) + if IsValidKubernetesResource(id) { + f.Identifiers = append(f.Identifiers, wait.ResourceIdentifier{ + Name: id.Name, + Namespace: id.Namespace, + GroupKind: schema.GroupKind{ + Group: gv.Group, + Kind: id.Kind, + }, + }) + } } return slice, nil } + +func IsValidKubernetesResource(id yaml.ResourceIdentifier) bool { + return id.GetKind() != "" && id.GetAPIVersion() != "" && id.GetName() != "" +} diff --git a/cmd/resource/status/cmd/util_test.go b/cmd/resource/status/cmd/util_test.go new file mode 100644 index 000000000..da081aa14 --- /dev/null +++ b/cmd/resource/status/cmd/util_test.go @@ -0,0 +1,40 @@ +package cmd + +import ( + "github.com/stretchr/testify/assert" + "sigs.k8s.io/kustomize/kyaml/yaml" + "testing" +) + +func TestIsValidKubernetesResource(t *testing.T) { + + testCases := map[string]struct { + data yaml.ResourceIdentifier + expected bool + }{ + "invalid resource": { + data: yaml.ResourceIdentifier { + Name : "", + APIVersion : "", + Kind : "", + Namespace : "", + }, + expected: false, + }, + "valid resource": { + data: yaml.ResourceIdentifier { + Name : "SomeName", + APIVersion : "SomeVersion", + Kind : "SomeKind", + Namespace : "", + }, + expected: true, + }, + } + + for tn, tc := range testCases { + t.Run(tn, func(t *testing.T) { + assert.Equal(t, IsValidKubernetesResource(tc.data), tc.expected) + }) + } +} From 4d07004977e66c7fbed72e32f9ca31835a59b821 Mon Sep 17 00:00:00 2001 From: Phani Teja Marupaka Date: Thu, 16 Jan 2020 14:04:27 -0800 Subject: [PATCH 02/46] Hello app e2e tests --- examples/alphaTestExamples/helloapp.md | 205 ++++++++++++++++++ .../alphaTestExamples/helloworld/README.md | 113 ---------- .../helloworld/configMap.yaml | 7 - .../helloworld/deployment.yaml | 30 --- .../helloworld/grouping.yaml | 6 - .../helloworld/kustomization.yaml | 10 - .../alphaTestExamples/helloworld/service.yaml | 12 - kustomize/go.sum | 114 ++++++++++ 8 files changed, 319 insertions(+), 178 deletions(-) create mode 100644 examples/alphaTestExamples/helloapp.md delete mode 100644 examples/alphaTestExamples/helloworld/README.md delete mode 100644 examples/alphaTestExamples/helloworld/configMap.yaml delete mode 100644 examples/alphaTestExamples/helloworld/deployment.yaml delete mode 100644 examples/alphaTestExamples/helloworld/grouping.yaml delete mode 100644 examples/alphaTestExamples/helloworld/kustomization.yaml delete mode 100644 examples/alphaTestExamples/helloworld/service.yaml diff --git a/examples/alphaTestExamples/helloapp.md b/examples/alphaTestExamples/helloapp.md new file mode 100644 index 000000000..902309e69 --- /dev/null +++ b/examples/alphaTestExamples/helloapp.md @@ -0,0 +1,205 @@ +[hello]: https://github.com/monopole/hello +[kind]: https://github.com/kubernetes-sigs/kind +[helloWorld]: https://github.com/kubernetes-sigs/kustomize/tree/master/examples/helloWorld + +# Demo: hello app + +This demo helps you to deploy an example hello app end-to-end using kustomize. + +Steps: +1. Create the resources files. +2. Kustomize them. +3. Spin-up kubernetes cluster on local using [kind]. +4. Deploy the app using kustomize and verify the status. + +First define a place to work: + + +``` +DEMO_HOME=$(mktemp -d) +``` + +Alternatively, use + +> ``` +> DEMO_HOME=~/hello +> ``` + +## Establish the base + +Let's run the [hello] service. + + +``` +BASE=$DEMO_HOME/base +mkdir -p $BASE +``` + +Now lets add a simple config map resource to the `base` + + +``` +cat <$BASE/configMap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: the-map +data: + altGreeting: "Good Morning!" + enableRisky: "false" +EOF +``` + +Create `deployment.yaml` with any image and with desired number of replicas + + +``` +cat <$BASE/deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: the-deployment +spec: + replicas: 3 + template: + metadata: + labels: + deployment: hello + spec: + containers: + - name: the-container + image: monopole/hello:1 + command: ["/hello", + "--port=8080", + "--enableRiskyFeature=\$(ENABLE_RISKY)"] + ports: + - containerPort: 8080 + env: + - name: ALT_GREETING + valueFrom: + configMapKeyRef: + name: the-map + key: altGreeting + - name: ENABLE_RISKY + valueFrom: + configMapKeyRef: + name: the-map + key: enableRisky +EOF +``` + +Create `service.yaml` pointing to the deployment created above + + +``` +cat <$BASE/service.yaml +kind: Service +apiVersion: v1 +metadata: + name: the-service +spec: + selector: + deployment: hello + type: LoadBalancer + ports: + - protocol: TCP + port: 8666 + targetPort: 8080 +EOF +``` + +Create a `grouping.yaml` resource. By this, you are defining the grouping of the current directory, `base`. Kustomize uses the unique label in this file to track any future state changes made to this directory. Make sure the label key is `kustomize.config.k8s.io/inventory-id` and give any unique label value and DO NOT change it in future. + + +``` +cat <$BASE/grouping.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: inventory-map + labels: + kustomize.config.k8s.io/inventory-id: hello-app +EOF +``` + +Now, create `kustomization.yaml` add all your resources. + + +``` +cat <$BASE/kustomization.yaml +commonLabels: + app: hello + +resources: +- deployment.yaml +- service.yaml +- configMap.yaml +- grouping.yaml +EOF +``` + +### The Base Kustomization + +The `base` directory has a kustomization file: + + +``` +more $BASE/kustomization.yaml +``` + +### Customize the base + +A simple customization step could be to change the _app +label_ applied to all resources: + + +``` +sed -i.bak 's/app: hello/app: my-hello/' \ + $BASE/kustomization.yaml +``` + +To do end to end tests using kustomize on local, go through the following section. You should have GOPATH set up and [kind] installed. + + +``` +MYGOBIN=$GOPATH/bin +``` + +Delete any existing kind cluster and create a new one. By default the name of the cluster is "kind" + +``` +kind delete cluster; +kind create cluster; +``` + +Use the kustomize binary in MYGOBIN to apply a deployment, fetch the status and verify the status. + +``` +export KUSTOMIZE_ENABLE_ALPHA_COMMANDS=true + +$MYGOBIN/kustomize resources apply $BASE --status; + +status=$(mktemp); +$MYGOBIN/resource status fetch $BASE > $status + +test 1 == \ + $(grep "the-deployment" $status | grep "Deployment is available. Replicas: 3" | wc -l); \ + echo $? + +test 1 == \ + $(grep "the-map" $status | grep "Resource is always ready" | wc -l); \ + echo $? + +test 1 == \ + $(grep "the-service" $status | grep "Service is ready" | wc -l); \ + echo $? +``` + +Clean-up the cluster + +``` +kind delete cluster; +``` + +###Next Exercise +Create overlays as described in the [helloWorld] section and verify the results. Good luck! \ No newline at end of file diff --git a/examples/alphaTestExamples/helloworld/README.md b/examples/alphaTestExamples/helloworld/README.md deleted file mode 100644 index 96817fa67..000000000 --- a/examples/alphaTestExamples/helloworld/README.md +++ /dev/null @@ -1,113 +0,0 @@ -[base]: ../../docs/glossary.md#base -[config]: https://github.com/kinflate/example-hello -[gitops]: ../../docs/glossary.md#gitops -[hello]: https://github.com/monopole/hello -[kustomization]: ../../docs/glossary.md#kustomization -[original]: https://github.com/kinflate/example-hello -[overlay]: ../../docs/glossary.md#overlay -[overlays]: ../../docs/glossary.md#overlay -[patch]: ../../docs/glossary.md#patch -[variant]: ../../docs/glossary.md#variant -[variants]: ../../docs/glossary.md#variant - -# Demo: hello world - -Steps: - - 1. Clone an existing configuration as a [base]. - 1. Customize it. - -First define a place to work: - - -``` -DEMO_HOME=$(mktemp -d) -``` - -Alternatively, use - -> ``` -> DEMO_HOME=~/hello -> ``` - -## Establish the base - -Let's run the [hello] service. - -To keep this document shorter, the base resources are -off in a supplemental data directory rather than -declared here as HERE documents. Download them: - - -``` -BASE=$DEMO_HOME/base -mkdir -p $BASE - -curl -s -o "$BASE/#1.yaml" "https://raw.githubusercontent.com\ -/kubernetes-sigs/kustomize\ -/master/examples/alphaTestExamples/helloWorld\ -/{configMap,deployment,grouping,kustomization,service}.yaml" -``` - -### The Base Kustomization - -The `base` directory has a [kustomization] file: - - -``` -more $BASE/kustomization.yaml -``` - -### Customize the base - -A first customization step could be to change the _app -label_ applied to all resources: - - -``` -sed -i.bak 's/app: hello/app: my-hello/' \ - $BASE/kustomization.yaml -``` - -To do end to end tests using kustomize, go through the following section. You should have GOPATH set up and "kind" installed(https://github.com/kubernetes-sigs/kind). - - -``` -MYGOBIN=$GOPATH/bin -``` - -Delete any existing kind cluster and create a new one. By default the name of the cluster is "kind" - -``` -kind delete cluster; -kind create cluster; -``` - -Use the kustomize binary in MYGOBIN to apply a deployment, fetch the status and verify the status. - -``` -export KUSTOMIZE_ENABLE_ALPHA_COMMANDS=true - -$MYGOBIN/kustomize resources apply $BASE --status; - -status=$(mktemp); -$MYGOBIN/resource status fetch $BASE > $status - -test 1 == \ - $(grep "the-deployment" $status | grep "Deployment is available. Replicas: 3" | wc -l); \ - echo $? - -test 1 == \ - $(grep "the-map" $status | grep "Resource is always ready" | wc -l); \ - echo $? - -test 1 == \ - $(grep "the-service" $status | grep "Service is ready" | wc -l); \ - echo $? -``` - -Clean-up the cluster - -``` -kind delete cluster; -``` \ No newline at end of file diff --git a/examples/alphaTestExamples/helloworld/configMap.yaml b/examples/alphaTestExamples/helloworld/configMap.yaml deleted file mode 100644 index e335ab8cc..000000000 --- a/examples/alphaTestExamples/helloworld/configMap.yaml +++ /dev/null @@ -1,7 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: the-map -data: - altGreeting: "Good Morning!" - enableRisky: "false" diff --git a/examples/alphaTestExamples/helloworld/deployment.yaml b/examples/alphaTestExamples/helloworld/deployment.yaml deleted file mode 100644 index 6e7940908..000000000 --- a/examples/alphaTestExamples/helloworld/deployment.yaml +++ /dev/null @@ -1,30 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: the-deployment -spec: - replicas: 3 - template: - metadata: - labels: - deployment: hello - spec: - containers: - - name: the-container - image: monopole/hello:1 - command: ["/hello", - "--port=8080", - "--enableRiskyFeature=$(ENABLE_RISKY)"] - ports: - - containerPort: 8080 - env: - - name: ALT_GREETING - valueFrom: - configMapKeyRef: - name: the-map - key: altGreeting - - name: ENABLE_RISKY - valueFrom: - configMapKeyRef: - name: the-map - key: enableRisky diff --git a/examples/alphaTestExamples/helloworld/grouping.yaml b/examples/alphaTestExamples/helloworld/grouping.yaml deleted file mode 100644 index ea723dda6..000000000 --- a/examples/alphaTestExamples/helloworld/grouping.yaml +++ /dev/null @@ -1,6 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: inventory-map - labels: - "kustomize.config.k8s.io/inventory-id": "hello-world-app" \ No newline at end of file diff --git a/examples/alphaTestExamples/helloworld/kustomization.yaml b/examples/alphaTestExamples/helloworld/kustomization.yaml deleted file mode 100644 index 73e197c8b..000000000 --- a/examples/alphaTestExamples/helloworld/kustomization.yaml +++ /dev/null @@ -1,10 +0,0 @@ -# Example configuration for the webserver -# at https://github.com/monopole/hello -commonLabels: - app: hello - -resources: -- deployment.yaml -- service.yaml -- configMap.yaml -- grouping.yaml diff --git a/examples/alphaTestExamples/helloworld/service.yaml b/examples/alphaTestExamples/helloworld/service.yaml deleted file mode 100644 index e238f7002..000000000 --- a/examples/alphaTestExamples/helloworld/service.yaml +++ /dev/null @@ -1,12 +0,0 @@ -kind: Service -apiVersion: v1 -metadata: - name: the-service -spec: - selector: - deployment: hello - type: LoadBalancer - ports: - - protocol: TCP - port: 8666 - targetPort: 8080 diff --git a/kustomize/go.sum b/kustomize/go.sum index 94255fb13..0d907bbdd 100644 --- a/kustomize/go.sum +++ b/kustomize/go.sum @@ -27,6 +27,7 @@ github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAE github.com/OpenPeeDeeP/depguard v1.0.1 h1:VlW4R6jmBIv3/u1JNlawEvJMM4J+dPORPaZasQee8Us= github.com/OpenPeeDeeP/depguard v1.0.1/go.mod h1:xsIw86fROiiwelg+jB2uM9PiKihMMmUx/1V+TNhjQvM= github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= +github.com/PuerkitoBio/purell v1.1.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI= github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= @@ -36,6 +37,8 @@ github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrU github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= +github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= +github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= @@ -45,11 +48,17 @@ github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghf github.com/chai2010/gettext-go v0.0.0-20160711120539-c6fed771bfd5 h1:7aWHqerlJ41y6FOsEUvknqgXnGmJyJSbjhAWq5pO4F8= github.com/chai2010/gettext-go v0.0.0-20160711120539-c6fed771bfd5/go.mod h1:/iP1qXHoty45bqomnu2LM+VVyAEdWN+vtSHGlQgyxbw= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/coreos/bbolt v1.3.1-coreos.6/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/etcd v3.3.15+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= +github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/pkg v0.0.0-20180108230652-97fdf19511ea/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= @@ -64,8 +73,10 @@ github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8 github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/docker v0.7.3-0.20190327010347-be7ac8be2ae0 h1:w3NnFcKR5241cfmQU5ZZAsf0xcpId6mWOupTvJlUX2U= github.com/docker/docker v0.7.3-0.20190327010347-be7ac8be2ae0/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96 h1:cenwrSVm+Z7QLSV/BsnenAOcDXdX4cMv4wP0B/5QbPg= github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= +github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/elazarl/goproxy v0.0.0-20170405201442-c4fc26588b6e h1:p1yVGRW3nmb85p1Sh1ZJSDm4A4iKLS5QNbvUHMgGu/M= github.com/elazarl/goproxy v0.0.0-20170405201442-c4fc26588b6e/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= @@ -84,6 +95,8 @@ github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMo github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/globalsign/mgo v0.0.0-20180905125535-1ca0a4f7cbcb/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= +github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= github.com/go-critic/go-critic v0.3.5-0.20190904082202-d79a9f0c64db h1:GYXWx7Vr3+zv833u+8IoXbNnQY0AdXsxAgI0kX7xcwA= github.com/go-critic/go-critic v0.3.5-0.20190904082202-d79a9f0c64db/go.mod h1:+sE8vrLDS2M0pZkBk0wy6+nLdKexVDrl/jBqQOTDThA= github.com/go-errors/errors v1.0.1 h1:LUHzmkK3GUKUrL/1gfBUxAHzcev3apQlezX/+O7ma6w= @@ -94,25 +107,53 @@ github.com/go-lintpack/lintpack v0.5.2/go.mod h1:NwZuYi2nUHho8XEIZ6SIxihrnPoqBTD github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= +github.com/go-logr/zapr v0.1.0/go.mod h1:tabnROwaDl0UNxkVeFRbY8bwB37GwRv0P8lg6aAiEnk= github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8= +github.com/go-openapi/analysis v0.0.0-20180825180245-b006789cd277/go.mod h1:k70tL6pCuVxPJOHXQ+wIac1FUrvNkHolPie/cLEU6hI= +github.com/go-openapi/analysis v0.17.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik= +github.com/go-openapi/analysis v0.18.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik= +github.com/go-openapi/analysis v0.19.2/go.mod h1:3P1osvZa9jKjb8ed2TPng3f0i/UY9snX6gxi44djMjk= +github.com/go-openapi/errors v0.17.0/go.mod h1:LcZQpmvG4wyF5j4IhA73wkLFQg+QJXOQHVjmcZxhka0= +github.com/go-openapi/errors v0.18.0/go.mod h1:LcZQpmvG4wyF5j4IhA73wkLFQg+QJXOQHVjmcZxhka0= +github.com/go-openapi/errors v0.19.2/go.mod h1:qX0BLWsyaKfvhluLejVpVNwNRdXZhEbTA4kxxpKBC94= github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+35s3my2LFTysnkMfxsJBAMHj/DoqoB9knIWoYG/Vk0= +github.com/go-openapi/jsonpointer v0.17.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M= +github.com/go-openapi/jsonpointer v0.18.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M= github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= github.com/go-openapi/jsonpointer v0.19.3 h1:gihV7YNZK1iK6Tgwwsxo2rJbD1GTbdm72325Bq8FI3w= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/jsonreference v0.0.0-20160704190145-13c6e3589ad9/go.mod h1:W3Z9FmVs9qj+KR4zFKmDPGiLdk1D9Rlm7cyMvf57TTg= +github.com/go-openapi/jsonreference v0.17.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I= +github.com/go-openapi/jsonreference v0.18.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I= github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc= github.com/go-openapi/jsonreference v0.19.3 h1:5cxNfTy0UVC3X8JL5ymxzyoUZmo8iZb+jeTWn7tUa8o= github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= +github.com/go-openapi/loads v0.17.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= +github.com/go-openapi/loads v0.18.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= +github.com/go-openapi/loads v0.19.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= +github.com/go-openapi/loads v0.19.2/go.mod h1:QAskZPMX5V0C2gvfkGZzJlINuP7Hx/4+ix5jWFxsNPs= +github.com/go-openapi/runtime v0.0.0-20180920151709-4f900dc2ade9/go.mod h1:6v9a6LTXWQCdL8k1AO3cvqx5OtZY/Y9wKTgaoP6YRfA= +github.com/go-openapi/runtime v0.19.0/go.mod h1:OwNfisksmmaZse4+gpV3Ne9AyMOlP1lt4sK4FXt0O64= github.com/go-openapi/spec v0.0.0-20160808142527-6aced65f8501/go.mod h1:J8+jY1nAiCcj+friV/PDoE1/3eeccG9LYBs0tYvLOWc= +github.com/go-openapi/spec v0.17.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI= +github.com/go-openapi/spec v0.18.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI= +github.com/go-openapi/spec v0.19.2/go.mod h1:sCxk3jxKgioEJikev4fgkNmwS+3kuYdJtcsZsD5zxMY= github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo= github.com/go-openapi/spec v0.19.4 h1:ixzUSnHTd6hCemgtAJgluaTSGYpLNpJY4mA2DIkdOAo= github.com/go-openapi/spec v0.19.4/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo= github.com/go-openapi/spec v0.19.5 h1:Xm0Ao53uqnk9QE/LlYV5DEU09UAgpliA85QoT9LzqPw= github.com/go-openapi/spec v0.19.5/go.mod h1:Hm2Jr4jv8G1ciIAo+frC/Ft+rR2kQDh8JHKHb3gWUSk= +github.com/go-openapi/strfmt v0.17.0/go.mod h1:P82hnJI0CXkErkXi8IKjPbNBM6lV6+5pLP5l494TcyU= +github.com/go-openapi/strfmt v0.18.0/go.mod h1:P82hnJI0CXkErkXi8IKjPbNBM6lV6+5pLP5l494TcyU= +github.com/go-openapi/strfmt v0.19.0/go.mod h1:+uW+93UVvGGq2qGaZxdDeJqSAqBqBdl+ZPMF/cC8nDY= github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dpr1UfpPtxFw+EFuQ41HhCWZfha5jSVRG7C7I= +github.com/go-openapi/swag v0.17.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg= +github.com/go-openapi/swag v0.18.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg= github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-openapi/swag v0.19.5 h1:lTz6Ys4CmqqCQmZPBlbQENR1/GucA2bzYTE12Pw4tFY= github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-openapi/validate v0.18.0/go.mod h1:Uh4HdOzKt19xGIGm1qHf/ofbX1YQ4Y+MYsct2VUrAJ4= +github.com/go-openapi/validate v0.19.2/go.mod h1:1tRCw7m3jtI8eNWEEliiAqUIcBztB2KDnRCRMUi7GTA= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-toolsmith/astcast v1.0.0 h1:JojxlmI6STnFVG9yOImLeGREv8W2ocNUM+iOhR6jE7g= github.com/go-toolsmith/astcast v1.0.0/go.mod h1:mt2OdQTeAQcY4DQgPSArJjHCcOwlX+Wl/kwN+LbLGQ4= @@ -143,14 +184,18 @@ github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7a github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d h1:3PaI8p3seN09VjbTYC/QWlUZdZ1qS1zGjy7LH2Wt07I= github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= +github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20180513044358-24b0969c4cb7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef h1:veQD95Isof8w9/WXiA+pa3tz3fJXkt5B7QaRBrM62gk= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v0.0.0-20161109072736-4bd1920723d7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.0.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= @@ -194,6 +239,8 @@ github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/gofuzz v0.0.0-20161122191042-44d81051d367/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI= github.com/google/gofuzz v1.0.0 h1:A8PeW59pxE9IoFRqBp37U+mSNaQoZ46F1f0f863XSXw= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -202,21 +249,27 @@ github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OI github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= +github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d h1:7XGaL1e6bYS1yIonGp9761ExpPPV1ui0SAC59Yube9k= github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= +github.com/googleapis/gnostic v0.3.1 h1:WeAefnSUHlBb0iJKwxFDZdbfGwkd7xRNuV+IpXMJhYk= +github.com/googleapis/gnostic v0.3.1/go.mod h1:on+2t9HRStVgn95RSsFWFz+6Q0Snyqv1awfrALZdbtU= github.com/gophercloud/gophercloud v0.1.0 h1:P/nh25+rzXouhytV2pUHBb65fnds26Ghl8/391+sT5o= github.com/gophercloud/gophercloud v0.1.0/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEoIEcSTewFxm1c5g8= github.com/gorilla/websocket v1.4.0 h1:WDFjx/TMzVgy9VdMMQi2K2Emtwi2QcUQsztZ/zLaH/Q= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gostaticanalysis/analysisutil v0.0.0-20190318220348-4088753ea4d3 h1:JVnpOZS+qxli+rgVl98ILOXVNbW+kb5wcxeGx8ShUIw= github.com/gostaticanalysis/analysisutil v0.0.0-20190318220348-4088753ea4d3/go.mod h1:eEOZF4jCKGi+aprrirO9e7WKB3beBRtWgqGunKl6pKE= +github.com/gregjones/httpcache v0.0.0-20170728041850-787624de3eb7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 h1:pdN6V1QBWetyv/0+wjACpqVH+eVULgEjkurDLq3goeM= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= +github.com/grpc-ecosystem/go-grpc-middleware v0.0.0-20190222133341-cfaf5686ec79/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/grpc-gateway v1.3.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw= github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= @@ -231,12 +284,15 @@ github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/imdario/mergo v0.3.5 h1:JboBksRwiiAJWvIYJVo46AfV+IAIKZpfrSzVKj42R4Q= github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28= +github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/jonboulle/clockwork v0.1.0 h1:VKV+ZcuP6l3yW9doeqz6ziZGgcynBVQO+obU0+0hcPo= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/json-iterator/go v0.0.0-20180612202835-f2b4162afba3/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.8 h1:QiWkFLKq0T7mpzwOTu6BzNDbfTE8OLrYhVKYMLF46Ok= github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= @@ -267,6 +323,8 @@ github.com/logrusorgru/aurora v0.0.0-20181002194514-a7b3b318ed4e/go.mod h1:7rIyQ github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDePerRcY= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.7.0 h1:aizVhC/NAAcKWb+5QsU1iNOZb4Yws5UO2I+aIprQITM= @@ -305,13 +363,18 @@ github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn github.com/olekukonko/tablewriter v0.0.4 h1:vHD/YYe1Wolo78koG299f7V/VAS08c6IpCLn+Ejf/w8= github.com/olekukonko/tablewriter v0.0.4/go.mod h1:zq6QwlOf5SlnkVbMSr5EoBv3636FWnp+qbPhuoO21uA= github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.4.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.10.1 h1:q/mM8GF/n0shIN8SaAZ0V+jnLPzen6WIVZdiwrRlMlo= github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= +github.com/onsi/gomega v1.3.0/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= +github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME= github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= +github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI= @@ -326,15 +389,19 @@ github.com/posener/complete/v2 v2.0.1-alpha.12 h1:0wvkuDfHb5vSZlNBYgpEH4XQHpF46M github.com/posener/complete/v2 v2.0.1-alpha.12/go.mod h1://JlL91cS2JV7rOl6LVHrRqBXoBUecJu3ILQPgbJiMQ= github.com/posener/script v1.0.4 h1:nSuXW5ZdmFnQIueLB2s0qvs4oNsUloM1Zydzh75v42w= github.com/posener/script v1.0.4/go.mod h1:Rg3ijooqulo05aGLyGsHoLmIOUzHUVK19WVgrYBPU/E= +github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM= github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= @@ -357,6 +424,7 @@ github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041/go.mod h1:N5mDOms github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/soheilhy/cmux v0.1.3/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/sourcegraph/go-diff v0.5.1 h1:gO6i5zugwzo1RVTvgvfwCOSVegNuvnNi6bAD1QCmkHs= github.com/sourcegraph/go-diff v0.5.1/go.mod h1:j2dHj3m8aZgQO8lMTcTnBcXkRRRqi34cd2MNlA9u1mE= @@ -388,6 +456,7 @@ github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJy github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/timakin/bodyclose v0.0.0-20190930140734-f7f2e9bca95e h1:RumXZ56IrCj4CL+g1b9OL/oH0QnsF976bC8xQFYUD5Q= github.com/timakin/bodyclose v0.0.0-20190930140734-f7f2e9bca95e/go.mod h1:Qimiffbc6q9tBWlVV6x0P9sat/ao1xEkREYPPj9hphk= +github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= @@ -401,6 +470,7 @@ github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyC github.com/valyala/fasthttp v1.2.0/go.mod h1:4vX61m6KN+xDduDNwXrhIAVZaZaZiQ1luJk8LWSxF3s= github.com/valyala/quicktemplate v1.2.0/go.mod h1:EH+4AkTd43SvgIbQHYu59/cJyxDoOVRUAfrukLPuGJ4= github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio= +github.com/xiang90/probing v0.0.0-20160813154853-07dd2e8dfe18/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xlab/handysort v0.0.0-20150421192137-fb3537ed64a1/go.mod h1:QcJo0QPSfTONNIgpN5RA8prR7fF8nkF6cTWTcNerRO8= github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca h1:1CFlNzQhALwjS9mBAUkycX616GzgsuYUOCHA5+HSlXI= @@ -408,16 +478,23 @@ github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca/go.mod h1:ce1O1j6Ut github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.uber.org/atomic v0.0.0-20181018215023-8dc6146f7569/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/multierr v0.0.0-20180122172545-ddea229ff1df/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/zap v0.0.0-20180814183419-67bc79d13d15/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190320223903-b7391e95e576/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392 h1:ACG4HJsFiNMf47Y4PeRoebLNy/2lXT9EtprMuTFWt1M= golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -431,19 +508,24 @@ golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180112015858-5ccada7d0a7b/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180911220305-26e67e76b6c3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181005035420-146acd28ed58/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190320064053-1272bf9dcd53/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190812203447-cdfb69ac37fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191004110552-13f9640d40b9 h1:rjwSpXsdiK0dV8/Naq3kAw9ymfAeJIyd0upUIElB+lI= @@ -457,7 +539,9 @@ golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180117170059-2c42eef0765b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -468,20 +552,25 @@ golang.org/x/sys v0.0.0-20190209173611-3b5209105503/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190321052220-f7bb7a8bee54/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190911201528-7ad0cfa0b7b5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190922100055-0a153f010e69 h1:rOhMmluY6kLMhdnrivzec6lLgaVbMHMn2ISQXJeJ5EM= golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20171227012246-e19ae1496984/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0 h1:/5xXl8Y5W96D+TtHSlonuFqGHIWVuyCkGJLwGh9JJFs= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -498,6 +587,7 @@ golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3 golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190322203728-c1a832b0ad89/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190521203540-521d6ed310dd/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190719005602-e377ae9d6386/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI= @@ -506,7 +596,9 @@ golang.org/x/tools v0.0.0-20190920225731-5eefd052ad72/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20190930201159-7c411dea38b0/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191010075000-0337d82405ff h1:XdBG6es/oFDr1HwaxkxgVve7NB281QhxgK/i4voubFs= golang.org/x/tools v0.0.0-20191010075000-0337d82405ff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7 h1:9zdDQZ7Thm29KFXgAX/+yaf3eVbP7djjWp/dXAppNCc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gomodules.xyz/jsonpatch/v2 v2.0.1/go.mod h1:IhYNNY4jnS53ZnfE4PAmpKtDpTCj1JFXc+3mwe7XcUU= gonum.org/v1/gonum v0.0.0-20190331200053-3d26580ed485/go.mod h1:2ltnJ7xHfj0zHS40VVPYEAAMTa3ZGguvHGBSJeRWqE0= gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= gonum.org/v1/netlib v0.0.0-20190331212654-76723241ea4e/go.mod h1:kS+toOQn6AQKjmKJ7gzohV1XkqsFehRA2FbsbkopSuQ= @@ -518,8 +610,10 @@ google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -528,12 +622,16 @@ gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/inf.v0 v0.9.0/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= +gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= +gopkg.in/yaml.v2 v2.0.0/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I= @@ -544,23 +642,31 @@ gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3 h1:3JgtbtFHMiCmsznwGVTUWbgGov+pVqnlf1dEJTNAXeM= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +k8s.io/api v0.0.0-20190918155943-95b840bb6a1f/go.mod h1:uWuOHnjmNrtQomJrvEBg0c0HRNyQ+8KTEERVsK0PW48= k8s.io/api v0.0.0-20191214185829-ca1d04f8b0d3/go.mod h1:itOjKREfmUTvcjantxOsyYU5mbFsU7qUnyUuRfF5+5M= k8s.io/api v0.17.0 h1:H9d/lw+VkZKEVIUc8F3wgiQ+FUXTTr21M87jXLU7yqM= k8s.io/api v0.17.0/go.mod h1:npsyOePkeP0CPwyGfXDHxvypiYMJxBWAMpQxCaJ4ZxI= +k8s.io/apiextensions-apiserver v0.0.0-20190918161926-8f644eb6e783/go.mod h1:xvae1SZB3E17UpV59AWc271W/Ph25N+bjPyR63X6tPY= +k8s.io/apimachinery v0.0.0-20190913080033-27d36303b655/go.mod h1:nL6pwRT8NgfF8TT68DBI8uEePRt89cSvoXUVqbkWHq4= k8s.io/apimachinery v0.0.0-20191214185652-442f8fb2f03a/go.mod h1:Ng1IY8TS7sC44KJxT/WUR6qFRfWwahYYYpNXyYRKOCY= k8s.io/apimachinery v0.0.0-20191216025728-0ee8b4573e3a/go.mod h1:Ng1IY8TS7sC44KJxT/WUR6qFRfWwahYYYpNXyYRKOCY= k8s.io/apimachinery v0.17.0 h1:xRBnuie9rXcPxUkDizUsGvPf1cnlZCFu210op7J7LJo= k8s.io/apimachinery v0.17.0/go.mod h1:b9qmWdKlLuU9EBh+06BtLcSf/Mu89rWL33naRxs1uZg= +k8s.io/apiserver v0.0.0-20190918160949-bfa5e2e684ad/go.mod h1:XPCXEwhjaFN29a8NldXA901ElnKeKLrLtREO9ZhFyhg= k8s.io/cli-runtime v0.0.0-20191214191754-e6dc6d5c8724/go.mod h1:wzlq80lvjgHW9if6MlE4OIGC86MDKsy5jtl9nxz/IYY= k8s.io/cli-runtime v0.17.0 h1:XEuStbJBHCQlEKFyTQmceDKEWOSYHZkcYWKp3SsQ9Hk= k8s.io/cli-runtime v0.17.0/go.mod h1:1E5iQpMODZq2lMWLUJELwRu2MLWIzwvMgDBpn3Y81Qo= +k8s.io/client-go v0.0.0-20190918160344-1fbdaa4c8d90/go.mod h1:J69/JveO6XESwVgG53q3Uz5OSfgsv4uxpScmmyYOOlk= k8s.io/client-go v0.0.0-20191214190045-a32a6f7a3052/go.mod h1:tAaoc/sYuIL0+njJefSAmE28CIcxyaFV4kbIujBlY2s= k8s.io/client-go v0.0.0-20191219150334-0b8da7416048/go.mod h1:ZEe8ZASDUAuqVGJ+UN0ka0PfaR+b6a6E1PGsSNZRui8= k8s.io/client-go v0.17.0 h1:8QOGvUGdqDMFrm9sD6IUFl256BcffynGoe80sxgTEDg= k8s.io/client-go v0.17.0/go.mod h1:TYgR6EUHs6k45hb6KWjVD6jFZvJV4gHDikv/It0xz+k= +k8s.io/code-generator v0.0.0-20190912054826-cd179ad6a269/go.mod h1:V5BD6M4CyaN5m+VthcclXWsVcT1Hu+glwa1bi3MIsyE= k8s.io/code-generator v0.0.0-20191214185510-0b9b3c99f9f2/go.mod h1:BjGKcoq1MRUmcssvHiSxodCco1T6nVIt4YeCT5CMSao= +k8s.io/component-base v0.0.0-20190918160511-547f6c5d7090/go.mod h1:933PBGtQFJky3TEwYx4aEPZ4IxqhWh3R6DCmzqIn1hA= k8s.io/component-base v0.0.0-20191214190519-d868452632e2/go.mod h1:wupxkh1T/oUDqyTtcIjiEfpbmIHGm8By/vqpSKC6z8c= k8s.io/component-base v0.17.0 h1:BnDFcmBDq+RPpxXjmuYnZXb59XNN9CaFrX8ba9+3xrA= k8s.io/component-base v0.17.0/go.mod h1:rKuRAokNMY2nn2A6LP/MiwpoaMRHpfRnrPaUJJj1Yoc= @@ -568,13 +674,16 @@ k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8 k8s.io/gengo v0.0.0-20190822140433-26a664648505/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= +k8s.io/klog v0.4.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8= k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= +k8s.io/kube-openapi v0.0.0-20190816220812-743ec37842bf/go.mod h1:1TqjTSzOxsLGIKfj0lK8EeCP7K1iUG65v09OM0/WG5E= k8s.io/kube-openapi v0.0.0-20191107075043-30be4d16710a h1:UcxjrRMyNx/i/y8G7kPvLyy7rfbeuf1PYyBf973pgyU= k8s.io/kube-openapi v0.0.0-20191107075043-30be4d16710a/go.mod h1:1TqjTSzOxsLGIKfj0lK8EeCP7K1iUG65v09OM0/WG5E= k8s.io/kubectl v0.0.0-20191219154910-1528d4eea6dd h1:nZX5+wEqTu/EBIYjrZlFOA63z4+Zcy96lDkCZPU9a9c= k8s.io/kubectl v0.0.0-20191219154910-1528d4eea6dd/go.mod h1:9ehGcuUGjXVZh0qbYSB0vvofQw2JQe6c6cO0k4wu/Oo= k8s.io/metrics v0.0.0-20191214191643-6b1944c9f765/go.mod h1:5V7rewilItwK0cz4nomU0b3XCcees2Ka5EBYWS1HBeM= +k8s.io/utils v0.0.0-20190801114015-581e00157fb1/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= k8s.io/utils v0.0.0-20191114184206-e782cd3c129f h1:GiPwtSzdP43eI1hpPCbROQCCIgCuiMMNF8YUVLF3vJo= k8s.io/utils v0.0.0-20191114184206-e782cd3c129f/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= modernc.org/cc v1.0.0/go.mod h1:1Sk4//wdnYJiUIxnW8ddKpaOJCF37yAdqYnkxUpaYxw= @@ -588,6 +697,8 @@ mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b h1:DxJ5nJdkhDlLok9K6qO+5290kphD mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b/go.mod h1:2odslEg/xrtNQqCYg2/jCoyKnw3vv5biOc3JnIcYfL4= mvdan.cc/unparam v0.0.0-20190720180237-d51796306d8f h1:Cq7MalBHYACRd6EesksG1Q8EoIAKOsiZviGKbOLIej4= mvdan.cc/unparam v0.0.0-20190720180237-d51796306d8f/go.mod h1:4G1h5nDURzA3bwVMZIVpwbkw+04kSxk3rAtzlimaUJw= +sigs.k8s.io/controller-runtime v0.4.0 h1:wATM6/m+3w8lj8FXNaO6Fs/rq/vqoOjO1Q116Z9NPsg= +sigs.k8s.io/controller-runtime v0.4.0/go.mod h1:ApC79lpY3PHW9xj/w9pj+lYkLgwAAUZwfXkME1Lajns= sigs.k8s.io/kustomize v2.0.3+incompatible h1:JUufWFNlI44MdtnjUqVnvh29rR37PQFzPbLXqhyOyX0= sigs.k8s.io/kustomize v2.0.3+incompatible/go.mod h1:MkjgH3RdOWrievjo6c9T245dYlB5QeXV4WCbnt/PEpU= sigs.k8s.io/kustomize/api v0.3.2 h1:64gvYVAvqe2fNfcTevtXh/GmLwVwHIcJ2Z5HBMfjncs= @@ -596,10 +707,13 @@ sigs.k8s.io/kustomize/cmd/config v0.0.5 h1:mFJowsk9IGvwm5dUpVB+ZM63on2JjgaCy+YcV sigs.k8s.io/kustomize/cmd/config v0.0.5/go.mod h1:L47nDnZDfGFQG3gnPJLG2UABn0nVb9v+ndceyMH0jjU= sigs.k8s.io/kustomize/cmd/kubectl v0.0.3 h1:cXn6GqRnOQtp4EC1+NiJKdUHE/aQ+5HhtAB28R4sVXA= sigs.k8s.io/kustomize/cmd/kubectl v0.0.3/go.mod h1:JnS9HnTjUUMOE44WNboy/wi89J/K/XbAoU7O/iPXqqE= +sigs.k8s.io/kustomize/kyaml v0.0.2/go.mod h1:rywm/rcR5LmCBghz9956tE45OdUPChFoXVVs+WmhMTI= sigs.k8s.io/kustomize/kyaml v0.0.5/go.mod h1:waxTrzQRK9i6/5fR5HNo8xa4YwvWn8t85vMnOGFEZik= sigs.k8s.io/kustomize/kyaml v0.0.6 h1:KhQr7JwpCseFTSWCwqp4CJ4mY6Kx+i34tF4e0eNkcXw= sigs.k8s.io/kustomize/kyaml v0.0.6/go.mod h1:tDOfJjL6slQVBLHJ76XfXAFgAOEdfm04AW2HehYOp8k= sigs.k8s.io/structured-merge-diff v0.0.0-20190525122527-15d366b2352e/go.mod h1:wWxsB5ozmmv/SG7nM11ayaAW51xMvak/t1r0CSlcokI= +sigs.k8s.io/structured-merge-diff v0.0.0-20190817042607-6149e4549fca/go.mod h1:IIgPezJWb76P0hotTxzDbWsMYB8APh18qZnxkomBpxA= +sigs.k8s.io/testing_frameworks v0.1.2/go.mod h1:ToQrwSC3s8Xf/lADdZp3Mktcql9CG0UAmdJG9th5i0w= sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= sourcegraph.com/sqs/pbtypes v0.0.0-20180604144634-d3ebe8f20ae4 h1:JPJh2pk3+X4lXAkZIk2RuE/7/FoK9maXw+TNPJhVS/c= From 13c891f54a2de245d3dba73cd3aae76f015464e3 Mon Sep 17 00:00:00 2001 From: Phani Teja Marupaka Date: Thu, 16 Jan 2020 15:49:11 -0800 Subject: [PATCH 03/46] Install kind as part of e2e tests --- Makefile | 12 +++++++++++- examples/alphaTestExamples/helloapp.md | 6 +++--- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 04530a8c3..95a9b8711 100644 --- a/Makefile +++ b/Makefile @@ -202,7 +202,7 @@ test-examples-kustomize-against-HEAD: $(MYGOBIN)/kustomize $(MYGOBIN)/mdrip ./hack/testExamplesAgainstKustomize.sh HEAD .PHONY: -test-examples-e2e-kustomize: $(MYGOBIN)/mdrip +test-examples-e2e-kustomize: $(MYGOBIN)/mdrip $(MYGOBIN)/kind ( \ set -e; \ /bin/rm -f $(MYGOBIN)/kustomize; \ @@ -257,6 +257,16 @@ $(MYGOBIN)/helm: rm -rf $$d \ ) +$(MYGOBIN)/kind: + ( \ + set -e; \ + d=$(shell mktemp -d); cd $$d; \ + wget -O ./kind https://github.com/kubernetes-sigs/kind/releases/download/v0.7.0/kind-$(shell uname)-amd64; \ + chmod +x ./kind; \ + mv ./kind $(MYGOBIN); \ + rm -rf $$d; \ + ) + .PHONY: clean clean: kustomize-external-go-plugin-clean go clean --cache diff --git a/examples/alphaTestExamples/helloapp.md b/examples/alphaTestExamples/helloapp.md index 902309e69..f5b65922e 100644 --- a/examples/alphaTestExamples/helloapp.md +++ b/examples/alphaTestExamples/helloapp.md @@ -168,8 +168,8 @@ MYGOBIN=$GOPATH/bin Delete any existing kind cluster and create a new one. By default the name of the cluster is "kind" ``` -kind delete cluster; -kind create cluster; +$MYGOBIN/kind delete cluster; +$MYGOBIN/kind create cluster; ``` Use the kustomize binary in MYGOBIN to apply a deployment, fetch the status and verify the status. @@ -198,7 +198,7 @@ test 1 == \ Clean-up the cluster ``` -kind delete cluster; +$MYGOBIN/kind delete cluster; ``` ###Next Exercise From 331bab494df98d3ad337a2d05a658516c483f6dd Mon Sep 17 00:00:00 2001 From: Phillip Wittrock Date: Thu, 16 Jan 2020 16:25:16 -0800 Subject: [PATCH 04/46] Release kyaml and cmd/config libs --- releasing/VERSIONS | 4 ++-- releasing/releasemodule.sh | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/releasing/VERSIONS b/releasing/VERSIONS index f079c963f..08e9f2cf9 100644 --- a/releasing/VERSIONS +++ b/releasing/VERSIONS @@ -6,7 +6,7 @@ # kyaml version export kyaml_major=0 export kyaml_minor=0 -export kyaml_patch=8 +export kyaml_patch=9 # kstatus version export kstatus_major=0 @@ -21,7 +21,7 @@ export api_patch=2 # cmd/config version export cmd_config_major=0 export cmd_config_minor=0 -export cmd_config_patch=9 +export cmd_config_patch=10 # cmd/kubectl version export cmd_kubectl_major=0 diff --git a/releasing/releasemodule.sh b/releasing/releasemodule.sh index 5361361eb..7b47230e3 100755 --- a/releasing/releasemodule.sh +++ b/releasing/releasemodule.sh @@ -50,6 +50,7 @@ function releaseModule { git commit -m "update go.mod for release" || echo "no changes made to go.mod" fi + go test ./... if [ "$NO_DRY_RUN" == "true" ]; then git push upstream $branch git tag -a $tag -m "Release $tag on branch $branch" From f4636f85551079c03d584475928f3ffb4dc28219 Mon Sep 17 00:00:00 2001 From: Haiyan Meng Date: Thu, 16 Jan 2020 11:28:41 -0800 Subject: [PATCH 05/46] Add a `fileType` field into the index --- api/internal/crawl/cmd/crawler/crawler.go | 2 +- api/internal/crawl/crawler/crawler.go | 68 ++++++---- api/internal/crawl/crawler/crawler_test.go | 2 + api/internal/crawl/crawler/github/crawler.go | 27 ++-- api/internal/crawl/doc/doc.go | 10 +- api/internal/crawl/doc/doc_test.go | 9 ++ api/internal/crawl/doc/docname.go | 4 + api/internal/crawl/doc/unique_doc.go | 2 +- api/internal/crawl/index/elasticsearch.go | 3 + api/internal/crawl/search_cmds/fileType.md | 123 +++++++++++++++++++ api/internal/crawl/search_cmds/misc.md | 13 ++ api/internal/crawl/utils/utils.go | 13 +- 12 files changed, 224 insertions(+), 52 deletions(-) create mode 100644 api/internal/crawl/search_cmds/fileType.md diff --git a/api/internal/crawl/cmd/crawler/crawler.go b/api/internal/crawl/cmd/crawler/crawler.go index 2d301efe1..54bf3f432 100644 --- a/api/internal/crawl/cmd/crawler/crawler.go +++ b/api/internal/crawl/cmd/crawler/crawler.go @@ -126,7 +126,7 @@ func main() { } } - // seen tracks the IDs of all the documents in the index. + // seen tracks the IDs of all the documents in the index and their corresponding file types. // This helps avoid indexing a given document multiple times. seen := utils.NewSeenMap() diff --git a/api/internal/crawl/crawler/crawler.go b/api/internal/crawl/crawler/crawler.go index 11715616d..7861749f0 100644 --- a/api/internal/crawl/crawler/crawler.go +++ b/api/internal/crawl/crawler/crawler.go @@ -38,6 +38,8 @@ type Crawler interface { // Write to the document what the created time is. SetCreated(context.Context, *doc.Document) error + SetDefaultBranch(*doc.Document) + Match(*doc.Document) bool } @@ -78,7 +80,7 @@ func findMatch(d *doc.Document, crawlers []Crawler) Crawler { func addBranches(cdoc CrawledDocument, match Crawler, indx IndexFunc, seen utils.SeenMap, stack *CrawlSeed) { - seen.Add(cdoc.ID()) + seen.Set(cdoc.ID(), cdoc.GetDocument().FileType) // Insert into index if err := indx(cdoc, index.InsertOrUpdate); err != nil { @@ -87,14 +89,14 @@ func addBranches(cdoc CrawledDocument, match Crawler, indx IndexFunc, return } - deps, err := cdoc.GetResources(true, false, false) + deps, err := cdoc.GetResources(true, true, true) if err != nil { logger.Println(err) return } for _, dep := range deps { - if seen.Seen(dep.ID()) { + if seen.Seen(dep.ID()) && seen.Value(dep.ID()) == dep.FileType { continue } *stack = append(*stack, dep) @@ -102,7 +104,7 @@ func addBranches(cdoc CrawledDocument, match Crawler, indx IndexFunc, } func doCrawl(ctx context.Context, docsPtr *CrawlSeed, crawlers []Crawler, conv Converter, indx IndexFunc, - seen utils.SeenMap, stack *CrawlSeed) { + seen utils.SeenMap, stack *CrawlSeed, refreshDoc bool, updateFileType bool) { UpdatedDocCount := 0 seenDocCount := 0 @@ -126,9 +128,11 @@ func doCrawl(ctx context.Context, docsPtr *CrawlSeed, crawlers []Crawler, conv C logger.Printf("Crawling doc %d: %s", crawledDocCount, tail.Path()) if seen.Seen(tail.ID()) { - logger.Printf("this doc has been seen before") - seenDocCount++ - continue + if !updateFileType || seen.Value(tail.ID()) == tail.FileType { + logger.Printf("this doc has been seen before") + seenDocCount++ + continue + } } if tail.WasCached() { @@ -151,26 +155,34 @@ func doCrawl(ctx context.Context, docsPtr *CrawlSeed, crawlers []Crawler, conv C // calling FetchDocument. Otherwise, the binary may enter into an infinite loop // if a kustomization file points to its kustmozation root in its `resources` or // `bases` field. - seen.Add(tail.ID()) + seen.Set(tail.ID(), tail.FileType) - if err := match.FetchDocument(ctx, tail); err != nil { - logger.Printf("FetchDocument failed on doc(%s): %v", tail.Path(), err) - FetchDocumentErrCount++ - // delete the document from the index - cdoc := &doc.KustomizationDocument{ - Document: *tail, - } - seen.Add(cdoc.ID()) - if err := indx(cdoc, index.Delete); err != nil { - logger.Printf("Failed to delete doc(%s): %v", cdoc.Path(), err) - } - deleteDocCount++ - continue + if refreshDoc || tail.DefaultBranch == "" { + match.SetDefaultBranch(tail) } - if err := match.SetCreated(ctx, tail); err != nil { - logger.Printf("SetCreated failed on doc(%s): %v", tail.Path(), err) - SetCreatedErrCount++ + if refreshDoc || tail.DocumentData == "" { + if err := match.FetchDocument(ctx, tail); err != nil { + logger.Printf("FetchDocument failed on doc(%s): %v", tail.Path(), err) + FetchDocumentErrCount++ + // delete the document from the index + cdoc := &doc.KustomizationDocument{ + Document: *tail, + } + seen.Set(cdoc.ID(), tail.FileType) + if err := indx(cdoc, index.Delete); err != nil { + logger.Printf("Failed to delete doc(%s): %v", cdoc.Path(), err) + } + deleteDocCount++ + continue + } + } + + if refreshDoc || tail.CreationTime == nil { + if err := match.SetCreated(ctx, tail); err != nil { + logger.Printf("SetCreated failed on doc(%s): %v", tail.Path(), err) + SetCreatedErrCount++ + } } cdoc, err := conv(tail) @@ -206,14 +218,14 @@ func CrawlFromSeed(ctx context.Context, seed CrawlSeed, crawlers []Crawler, // Exploit seed to update bulk of corpus. logger.Printf("updating %d documents from seed\n", len(seed)) // each unique document in seed will be crawled once. - doCrawl(ctx, &seed, crawlers, conv, indx, seen, &stack) + doCrawl(ctx, &seed, crawlers, conv, indx, seen, &stack, true, false) // Traverse any new documents added while updating corpus. logger.Printf("crawling %d new documents found in the seed\n", len(stack)) // While crawling each document in stack, the documents directly referred in the document // will be added into stack. // After this statement is done, stack will become empty. - doCrawl(ctx, &stack, crawlers, conv, indx, seen, &stack) + doCrawl(ctx, &stack, crawlers, conv, indx, seen, &stack, false, true) } // CrawlGithubRunner is a blocking function and only returns once all of the @@ -294,6 +306,8 @@ func CrawlGithub(ctx context.Context, crawlers []Crawler, conv Converter, for cdoc := range ch { docCount++ logger.Printf("Processing doc %d found on Github", docCount) + // all the docs here are kustomization files found by querying Github, and + // their `FileType` fields all should be empty. if seen.Seen(cdoc.ID()) { logger.Printf("the doc has been seen before") continue @@ -320,5 +334,5 @@ func CrawlGithub(ctx context.Context, crawlers []Crawler, conv Converter, // Handle deps of newly discovered documents. logger.Printf("crawling the %d new documents referred by other documents", len(stack)) - doCrawl(ctx, &stack, crawlers, conv, indx, seen, &stack) + doCrawl(ctx, &stack, crawlers, conv, indx, seen, &stack, false, true) } diff --git a/api/internal/crawl/crawler/crawler_test.go b/api/internal/crawl/crawler/crawler_test.go index d18a4afce..5e93d5bcb 100644 --- a/api/internal/crawl/crawler/crawler_test.go +++ b/api/internal/crawl/crawler/crawler_test.go @@ -37,6 +37,8 @@ func (c testCrawler) Match(d *doc.Document) bool { return d != nil } +func (c testCrawler) SetDefaultBranch(d *doc.Document) {} + func (c testCrawler) FetchDocument(_ context.Context, d *doc.Document) error { if i, ok := c.lukp[d.ID()]; ok { d.DocumentData = c.docs[i].DocumentData diff --git a/api/internal/crawl/crawler/github/crawler.go b/api/internal/crawl/crawler/github/crawler.go index e1aeff401..70c1a23ca 100644 --- a/api/internal/crawl/crawler/github/crawler.go +++ b/api/internal/crawl/crawler/github/crawler.go @@ -60,8 +60,16 @@ func NewCrawler(accessToken string, retryCount uint64, client *http.Client, } } -func (gc githubCrawler) SetDefaultBranch(repo, branch string) { - gc.branchMap[repo] = branch +func (gc githubCrawler) SetDefaultBranch(d *doc.Document) { + url := gc.client.ReposRequest(d.RepositoryFullName()) + defaultBranch, err := gc.client.GetDefaultBranch(url, d.RepositoryURL, gc.branchMap) + if err != nil { + logger.Printf( + "(error: %v) setting default_branch to master\n", err) + defaultBranch = "master" + } + d.DefaultBranch = defaultBranch + gc.branchMap[d.RepositoryURL] = d.DefaultBranch } func (gc githubCrawler) DefaultBranch(repo string) string { @@ -114,19 +122,6 @@ func (gc githubCrawler) Crawl(ctx context.Context, // it will try to add each string in konfig.RecognizedKustomizationFileNames() to // d.FilePath, and try to fetch the document again. func (gc githubCrawler) FetchDocument(_ context.Context, d *doc.Document) error { - // set the default branch if it is empty - if d.DefaultBranch == "" { - url := gc.client.ReposRequest(d.RepositoryFullName()) - defaultBranch, err := gc.client.GetDefaultBranch(url, d.RepositoryURL, gc.branchMap) - if err != nil { - logger.Printf( - "(error: %v) setting default_branch to master\n", err) - defaultBranch = "master" - } - d.DefaultBranch = defaultBranch - } - gc.SetDefaultBranch(d.RepositoryURL, d.DefaultBranch) - repoURL := d.RepositoryURL + "/" + d.FilePath + "?ref=" + d.DefaultBranch repoSpec, err := git.NewRepoSpecFromUrl(repoURL) if err != nil { @@ -283,6 +278,8 @@ func kustomizationResultAdapter(gcl GhClient, k GhFileSpec, seen utils.SeenMap, defaultBranch = "master" } + // document here is a kustomization file found by querying Github, whose + // `FileType` field should be empty. document := doc.Document{ FilePath: k.Path, DefaultBranch: defaultBranch, diff --git a/api/internal/crawl/doc/doc.go b/api/internal/crawl/doc/doc.go index 87cd5a3e2..0e5965ac1 100644 --- a/api/internal/crawl/doc/doc.go +++ b/api/internal/crawl/doc/doc.go @@ -87,17 +87,17 @@ func (doc *KustomizationDocument) GetResources( res := make([]*Document, 0) if includeResources { - resourceDocs := doc.CollectDocuments(k.Resources) + resourceDocs := doc.CollectDocuments(k.Resources, "resource") res = append(res, resourceDocs...) } if includeGenerators { - generatorDocs := doc.CollectDocuments(k.Generators) + generatorDocs := doc.CollectDocuments(k.Generators, "generator") res = append(res, generatorDocs...) } if includeTransformers { - transformerDocs := doc.CollectDocuments(k.Transformers) + transformerDocs := doc.CollectDocuments(k.Transformers, "transformer") res = append(res, transformerDocs...) } @@ -106,7 +106,8 @@ func (doc *KustomizationDocument) GetResources( // CollectDocuments construct a Document for each path in paths, and return // a slice of Document pointers. -func (doc *KustomizationDocument) CollectDocuments(paths []string) []*Document { +func (doc *KustomizationDocument) CollectDocuments( + paths []string, fileType string) []*Document { docs := make([]*Document, 0, len(paths)) for _, r := range paths { if strings.TrimSpace(r) == "" { @@ -117,6 +118,7 @@ func (doc *KustomizationDocument) CollectDocuments(paths []string) []*Document { log.Printf("CollectDocuments error: %v\n", err) continue } + next.FileType = fileType docs = append(docs, &next) } return docs diff --git a/api/internal/crawl/doc/doc_test.go b/api/internal/crawl/doc/doc_test.go index c193809a6..e66241287 100644 --- a/api/internal/crawl/doc/doc_test.go +++ b/api/internal/crawl/doc/doc_test.go @@ -215,19 +215,23 @@ resources: { RepositoryURL: "sigs.k8s.io/kustomize", FilePath: "some/path/to/base", + FileType: "resource", }, { RepositoryURL: "sigs.k8s.io/kustomize", FilePath: "some/path/to/otherbase", + FileType: "resource", }, { RepositoryURL: "sigs.k8s.io/kustomize", FilePath: "some/path/to/kdir/file.yaml", + FileType: "resource", }, { RepositoryURL: "https://github.com/kubernetes-sigs/kustomize", FilePath: "examples/helloWorld", DefaultBranch: "v3.1.0", + FileType: "resource", }, }, }, @@ -312,10 +316,12 @@ transformers: { RepositoryURL: "sigs.k8s.io/kustomize", FilePath: "some/path/to/kdir/gen.yaml", + FileType: "generator", }, { RepositoryURL: "sigs.k8s.io/kustomize", FilePath: "some/path/to/kdir/file.yaml", + FileType: "resource", }, }, }, @@ -345,14 +351,17 @@ transformers: { RepositoryURL: "sigs.k8s.io/kustomize", FilePath: "some/path/to/kdir/tr.yaml", + FileType: "transformer", }, { RepositoryURL: "sigs.k8s.io/kustomize", FilePath: "some/path/to/kdir/gen.yaml", + FileType: "generator", }, { RepositoryURL: "sigs.k8s.io/kustomize", FilePath: "some/path/to/kdir/file.yaml", + FileType: "resource", }, }, }, diff --git a/api/internal/crawl/doc/docname.go b/api/internal/crawl/doc/docname.go index e295e4620..a3cab942a 100644 --- a/api/internal/crawl/doc/docname.go +++ b/api/internal/crawl/doc/docname.go @@ -17,6 +17,9 @@ type Document struct { DocumentData string `json:"document,omitempty"` CreationTime *time.Time `json:"creationTime,omitempty"` IsSame bool `json:"-"` + // FileType can be one of the following: + // "generator", "transformer", "resource", "". + FileType string `json:"fileType,omitempty"` } // Implements the CrawlerDocument interface. @@ -32,6 +35,7 @@ func (doc *Document) Copy() *Document { DocumentData: doc.DocumentData, CreationTime: doc.CreationTime, IsSame: doc.IsSame, + FileType: doc.FileType, } } diff --git a/api/internal/crawl/doc/unique_doc.go b/api/internal/crawl/doc/unique_doc.go index 026b345a5..da5294b1f 100644 --- a/api/internal/crawl/doc/unique_doc.go +++ b/api/internal/crawl/doc/unique_doc.go @@ -22,7 +22,7 @@ func (uds *UniqueDocuments) Add(d *Document) { return } uds.docs = append(uds.docs, d) - uds.docIDs.Add(d.ID()) + uds.docIDs.Set(d.ID(), "") } func (uds *UniqueDocuments) AddDocuments(docs []*Document) { diff --git a/api/internal/crawl/index/elasticsearch.go b/api/internal/crawl/index/elasticsearch.go index 7d7ce2b9e..ebd6b70c0 100644 --- a/api/internal/crawl/index/elasticsearch.go +++ b/api/internal/crawl/index/elasticsearch.go @@ -26,6 +26,9 @@ const IndexConfig = ` "defaultBranch": { "type": "keyword" }, + "fileType": { + "type": "keyword" + }, "document": { "type": "text" }, diff --git a/api/internal/crawl/search_cmds/fileType.md b/api/internal/crawl/search_cmds/fileType.md new file mode 100644 index 000000000..ced952f3e --- /dev/null +++ b/api/internal/crawl/search_cmds/fileType.md @@ -0,0 +1,123 @@ +Find all the documents having the `fileType` field set: +``` +curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d' +{ + "query": { + "exists": { + "field": "fileType" + } + } +} +' +``` + +Find all the documents whose `fileType` field is not set: +``` +curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d' +{ + "size": 10000, + "query": { + "bool": { + "must_not": { + "exists": { + "field": "fileType" + } + } + } + } +} +' +``` + +Search for all the documents whose `fileType` field is `resource`: +``` +curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d' +{ + "query": { + "bool": { + "filter": [ + { "regexp": { "fileType": "resource" }} + ] + } + } +} +' +``` + +Count distinct values of the `fileType` field: +``` +curl -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' +{ + "aggs" : { + "fileType_count" : { + "cardinality" : { + "field" : "fileType", + "precision_threshold": 40000 + } + } + } +} +' +``` + +List all the values of the `fileType` field and the frequency of each value: +``` +curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' +{ + "aggs" : { + "fileType" : { + "terms" : { + "field" : "fileType" + } + } + } +} +' +``` + + +For all the kustomization files in the index, list all the values of the +`fileType` field and the frequency of each value: +``` +curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' +{ + "query": { + "bool": { + "filter": [ + { "regexp": { "filePath": ".*/kustomization((.yaml)?|(.yml)?)/*" }} + ] + } + }, + "aggs" : { + "fileType" : { + "terms" : { + "field" : "fileType" + } + } + } +} +' +``` + +For all the non-kustomization files in the index, list all the values of the +`fileType` field and the frequency of each value: +``` +curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' +{ + "query": { + "bool": { + "must_not": { + "regexp": { "filePath": ".*/kustomization((.yaml)?|(.yml)?)/*" } + } + } + }, + "aggs" : { + "fileType" : { + "terms" : { + "field" : "fileType" + } + } + } +} +' +``` diff --git a/api/internal/crawl/search_cmds/misc.md b/api/internal/crawl/search_cmds/misc.md index 303ae5d3b..d68736ea2 100644 --- a/api/internal/crawl/search_cmds/misc.md +++ b/api/internal/crawl/search_cmds/misc.md @@ -16,4 +16,17 @@ curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_mapping?pretty" Delete the kustomize index from the ElasticSearch cluster (**Use this command with caution**): ``` curl -X DELETE "${ElasticSearchURL}:9200/${INDEXNAME}?pretty" +``` + +Add a new field into an existing index. +``` +curl -X PUT "${ElasticSearchURL}:9200/${INDEXNAME}/_mapping/_doc?pretty" -H 'Content-Type: application/json' -d' +{ + "properties": { + "fileType": { + "type": "keyword" + } + } +} +' ``` \ No newline at end of file diff --git a/api/internal/crawl/utils/utils.go b/api/internal/crawl/utils/utils.go index a397b2d52..d6b6fab68 100644 --- a/api/internal/crawl/utils/utils.go +++ b/api/internal/crawl/utils/utils.go @@ -1,16 +1,21 @@ package utils -type SeenMap map[string]struct{} +type SeenMap map[string]string func (seen SeenMap) Seen(item string) bool { _, ok := seen[item] return ok } -func (seen SeenMap) Add(item string) { - seen[item] = struct{}{} +func (seen SeenMap) Set(k, v string) { + seen[k] = v +} + +// The caller should make sure that key is in the map. +func (seen SeenMap) Value(k string) string { + return seen[k] } func NewSeenMap() SeenMap { - return make(map[string]struct{}) + return make(map[string]string) } From 377eb5b66d3717a5e25c705fa70601cb516fe50a Mon Sep 17 00:00:00 2001 From: Haiyan Meng Date: Fri, 17 Jan 2020 14:53:40 -0800 Subject: [PATCH 06/46] Fix the regexp for determining kustomization file --- api/internal/crawl/search_cmds/creationTime.md | 12 ++++++------ api/internal/crawl/search_cmds/fileType.md | 4 ++-- api/internal/crawl/search_cmds/keyword_search.md | 4 ++-- api/internal/crawl/search_cmds/repositoryUrl.md | 8 ++++---- 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/api/internal/crawl/search_cmds/creationTime.md b/api/internal/crawl/search_cmds/creationTime.md index 3ebfaf157..e83165efe 100644 --- a/api/internal/crawl/search_cmds/creationTime.md +++ b/api/internal/crawl/search_cmds/creationTime.md @@ -27,7 +27,7 @@ curl -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'C "query": { "bool": { "filter": [ - { "regexp": { "filePath": ".*/kustomization((.yaml)?|(.yml)?)/*" }} + { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" }} ] } }, @@ -45,7 +45,7 @@ curl -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'C "query": { "bool": { "must_not": { - "regexp": { "filePath": ".*/kustomization((.yaml)?|(.yml)?)/*" } + "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" } } } }, @@ -94,7 +94,7 @@ curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Co "query": { "bool": { "filter": [ - { "regexp": { "filePath": ".*/kustomization((.yaml)?|(.yml)?)/*" }} + { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" }} ] } }, @@ -117,7 +117,7 @@ curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Co "query": { "bool": { "must_not": [ - { "regexp": { "filePath": ".*/kustomization((.yaml)?|(.yml)?)/*" }} + { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" }} ] } }, @@ -140,7 +140,7 @@ curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Co "query": { "bool": { "filter": [ - { "regexp": { "filePath": ".*/kustomization((.yaml)?|(.yml)?)/*" }} + { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" }} ] } }, @@ -163,7 +163,7 @@ curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Co "query": { "bool": { "must_not": [ - { "regexp": { "filePath": ".*/kustomization((.yaml)?|(.yml)?)/*" }} + { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" }} ] } }, diff --git a/api/internal/crawl/search_cmds/fileType.md b/api/internal/crawl/search_cmds/fileType.md index ced952f3e..c5c58357a 100644 --- a/api/internal/crawl/search_cmds/fileType.md +++ b/api/internal/crawl/search_cmds/fileType.md @@ -84,7 +84,7 @@ curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Co "query": { "bool": { "filter": [ - { "regexp": { "filePath": ".*/kustomization((.yaml)?|(.yml)?)/*" }} + { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" }} ] } }, @@ -107,7 +107,7 @@ curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Co "query": { "bool": { "must_not": { - "regexp": { "filePath": ".*/kustomization((.yaml)?|(.yml)?)/*" } + "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" } } } }, diff --git a/api/internal/crawl/search_cmds/keyword_search.md b/api/internal/crawl/search_cmds/keyword_search.md index 588f938fb..99a677813 100644 --- a/api/internal/crawl/search_cmds/keyword_search.md +++ b/api/internal/crawl/search_cmds/keyword_search.md @@ -57,7 +57,7 @@ curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-T "query": { "bool": { "filter": [ - { "regexp": { "filePath": ".*/kustomization((.yaml)?|(.yml)?)/*" }} + { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" }} ] } } @@ -73,7 +73,7 @@ curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-T "query": { "bool": { "must_not": [ - { "regexp": { "filePath": ".*/kustomization((.yaml)?|(.yml)?)/*" }} + { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" }} ] } } diff --git a/api/internal/crawl/search_cmds/repositoryUrl.md b/api/internal/crawl/search_cmds/repositoryUrl.md index 291aa1c69..9ce169929 100644 --- a/api/internal/crawl/search_cmds/repositoryUrl.md +++ b/api/internal/crawl/search_cmds/repositoryUrl.md @@ -21,7 +21,7 @@ curl -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'C "query": { "bool": { "filter": [ - { "regexp": { "filePath": ".*/kustomization((.yaml)?|(.yml)?)/*" }} + { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" }} ] } }, @@ -44,7 +44,7 @@ curl -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'C "query": { "bool": { "must_not": { - "regexp": { "filePath": ".*/kustomization((.yaml)?|(.yml)?)/*" } + "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" } } } }, @@ -85,7 +85,7 @@ curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Co "query": { "bool": { "filter": [ - { "regexp": { "filePath": ".*/kustomization((.yaml)?|(.yml)?)/*" }} + { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" }} ] } }, @@ -108,7 +108,7 @@ curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Co "query": { "bool": { "must_not": { - "regexp": { "filePath": ".*/kustomization((.yaml)?|(.yml)?)/*" } + "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" } } } }, From 96ee9e91469cefdc71febf557dff9986d229db8e Mon Sep 17 00:00:00 2001 From: Haiyan Meng Date: Fri, 17 Jan 2020 15:49:14 -0800 Subject: [PATCH 07/46] Add curl ElasticSearch cmd for using `filter` and `range` together --- .../crawl/search_cmds/creationTime.md | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/api/internal/crawl/search_cmds/creationTime.md b/api/internal/crawl/search_cmds/creationTime.md index e83165efe..d7eafdc45 100644 --- a/api/internal/crawl/search_cmds/creationTime.md +++ b/api/internal/crawl/search_cmds/creationTime.md @@ -87,6 +87,30 @@ curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-T ' ``` +Query all the kustomization files whose `creationTime` falls within the specific range: +``` +curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d' +{ + "size": 20, + "query": { + "bool": { + "filter": { + "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" } + }, + "must": { + "range": { + "creationTime": { + "gte": "2017-09-24T15:49:57.000Z", + "lte": "2017-09-24T15:49:57.000Z" + } + } + } + } + } +} +' +``` + Aggregate how many new kustomization files were added into Github each month: ``` curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' From 0bd872e6d53abd3e7dfa22bdf2852c6a4fc04908 Mon Sep 17 00:00:00 2001 From: Phani Teja Marupaka Date: Mon, 20 Jan 2020 11:42:39 -0800 Subject: [PATCH 08/46] Do not remove empty lines in configmap/secret --- api/krusty/baseandoverlaymedium_test.go | 6 ++++-- api/kv/kv.go | 2 +- api/kv/kv_test.go | 4 ++-- kustomize/go.mod | 1 + 4 files changed, 8 insertions(+), 5 deletions(-) diff --git a/api/krusty/baseandoverlaymedium_test.go b/api/krusty/baseandoverlaymedium_test.go index c207a18d2..569dc6c27 100644 --- a/api/krusty/baseandoverlaymedium_test.go +++ b/api/krusty/baseandoverlaymedium_test.go @@ -153,7 +153,8 @@ FRUIT=banana LEGUME=chickpea `) th.WriteF("/app/overlay/configmap/dummy.txt", - `Lorem ipsum dolor sit amet, consectetur + `Lorem ipsum dolor sit amet, consectetur + adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. `) @@ -294,6 +295,7 @@ apiVersion: v1 data: nonsense: | Lorem ipsum dolor sit amet, consectetur + adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. kind: ConfigMap @@ -304,6 +306,6 @@ metadata: app: mungebot org: kubernetes repo: test-infra - name: test-infra-app-config-4mt28b5bg2 + name: test-infra-app-config-hh272bg5d4 `) } diff --git a/api/kv/kv.go b/api/kv/kv.go index 25a2e8476..7bda10fd7 100644 --- a/api/kv/kv.go +++ b/api/kv/kv.go @@ -93,7 +93,7 @@ func (kvl *loader) keyValuesFromFileSources(sources []string) ([]types.Pair, err // trimTrailingSpacesInLines takes string with multiple lines and trims the trailing white spaces and tabs from each line. func trimTrailingSpacesInLines(str string) string { - re := regexp.MustCompile(`\s*\n`) + re := regexp.MustCompile(`[ \t]*\n`) return re.ReplaceAllString(str, "\n") } diff --git a/api/kv/kv_test.go b/api/kv/kv_test.go index 25b628197..7241fcd98 100644 --- a/api/kv/kv_test.go +++ b/api/kv/kv_test.go @@ -97,8 +97,8 @@ func TestKeyValuesFromFileSources(t *testing.T) { } func TestTrimTrailingSpacesInLines(t *testing.T) { - input := "\"fooKey\": \"fooValue\" \t\n\t\"barKey\": \"barValue\"" - expected := "\"fooKey\": \"fooValue\"\n\t\"barKey\": \"barValue\"" + input := "\"fooKey\": \"fooValue\" \t\n \t\t \n\t\"barKey\": \"barValue\"" + expected := "\"fooKey\": \"fooValue\"\n\n\t\"barKey\": \"barValue\"" res := trimTrailingSpacesInLines(input) if !reflect.DeepEqual(res, expected) { t.Errorf("Trim trailing spaces in lines should succeed, got: %s exptected: %s", res, expected) diff --git a/kustomize/go.mod b/kustomize/go.mod index d8abd9115..f1aed0041 100644 --- a/kustomize/go.mod +++ b/kustomize/go.mod @@ -20,6 +20,7 @@ exclude ( ) replace ( + sigs.k8s.io/kustomize/api v0.3.2 => ../api sigs.k8s.io/kustomize/cmd/kubectl v0.0.3 => ../cmd/kubectl sigs.k8s.io/kustomize/kstatus v0.0.1 => ../kstatus ) From e851e5eb94aff7158b4f4cea1b4e2ab2ffb317db Mon Sep 17 00:00:00 2001 From: Phani Teja Marupaka Date: Mon, 20 Jan 2020 23:02:50 -0800 Subject: [PATCH 09/46] E2E Tests with multiple apps --- .../alphaTestExamples/MultipleDeployments.md | 155 ++++++++++++++++++ examples/alphaTestExamples/helloapp.md | 56 ++++++- 2 files changed, 203 insertions(+), 8 deletions(-) create mode 100644 examples/alphaTestExamples/MultipleDeployments.md diff --git a/examples/alphaTestExamples/MultipleDeployments.md b/examples/alphaTestExamples/MultipleDeployments.md new file mode 100644 index 000000000..47f7593ab --- /dev/null +++ b/examples/alphaTestExamples/MultipleDeployments.md @@ -0,0 +1,155 @@ +[kind]: https://github.com/kubernetes-sigs/kind + +# Demo: Multiple Deployments + +This demo helps you to multiple services on same kubenetes cluster using kustomize. + +Steps: +1. Create the resources files for wordpress service. +2. Create the resources files for mysql service. +3. Spin-up kubernetes cluster on local using [kind]. +4. Deploy the wordpress app using kustomize and verify the status. +5. Deploy the mysql app using kustomize on same "kind" cluster and verify the status. +6. Add and remove a resource to mysql service and verify prune. + +First define a place to work: + + +``` +DEMO_HOME=$(mktemp -d) +``` + +Alternatively, use + +> ``` +> DEMO_HOME=~/hello +> ``` + +## Establish the base + + +``` +BASE=$DEMO_HOME/base +mkdir -p $BASE + +mkdir $BASE/wordpress +mkdir $BASE/mysql + +curl -s -o "$BASE/wordpress/#1.yaml" "https://raw.githubusercontent.com\ +/kubernetes-sigs/kustomize\ +/master/examples/wordpress/wordpress\ +/{deployment,kustomization,service}.yaml" + +curl -s -o "$BASE/mysql/#1.yaml" "https://raw.githubusercontent.com\ +/kubernetes-sigs/kustomize\ +/master/examples/wordpress/mysql\ +/{secret,deployment,kustomization,service}.yaml" +``` + +Create a `grouping.yaml` resource. By this, you are defining the grouping of the current directory, `mysql`. Kustomize uses the unique label in this file to track any future state changes made to this directory. Make sure the label key is `kustomize.config.k8s.io/inventory-id` and give any unique label value and DO NOT change it in future. + +``` +cat <$BASE/mysql/grouping.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: inventory-map + labels: + kustomize.config.k8s.io/inventory-id: mysql-app +EOF +``` + + +``` +MYGOBIN=$GOPATH/bin +``` + +Delete any existing kind cluster and create a new one. By default the name of the cluster is "kind" + +``` +$MYGOBIN/kind delete cluster; +$MYGOBIN/kind create cluster; +``` + +Let's run the wordpress and mysql services. + +``` +export KUSTOMIZE_ENABLE_ALPHA_COMMANDS=true + +$MYGOBIN/kustomize resources apply $BASE/mysql --status; + +status=$(mktemp); +$MYGOBIN/resource status fetch $BASE/mysql > $status + +test 1 == \ + $(grep "mysql" $status | grep "Deployment is available. Replicas: 1" | wc -l); \ + echo $? + +test 1 == \ + $(grep "mysql-pass" $status | grep "Resource is always ready" | wc -l); \ + echo $? + +test 1 == \ + $(grep "mysql" $status | grep "Service is ready" | wc -l); \ + echo $? + +$MYGOBIN/kustomize resources apply $BASE/wordpress --status; + +status=$(mktemp); +$MYGOBIN/resource status fetch $BASE/wordpress > $status + +test 1 == \ + $(grep "wordpress" $status | grep "Deployment is available. Replicas: 1" | wc -l); \ + echo $? + +test 1 == \ + $(grep "wordpress" $status | grep "Service is ready" | wc -l); \ + echo $? +``` + +Let's replace the secret resource from mysql service and verify prune and addition of resource. + + +``` +cat <$BASE/mysql/secret2.yaml +apiVersion: v1 +kind: Secret +metadata: + name: mysql-pass2 +type: Opaque +data: + # Default password is "admin". + password: YWRtaW5= +EOF + +rm $BASE/mysql/secret.yaml + +sed -i.bak 's/secret/secret2/' \ + $BASE/mysql/kustomization.yaml + +sed -i.bak 's/mysql-pass/mysql-pass2/' \ + $BASE/mysql/deployment.yaml + +$MYGOBIN/kustomize resources apply $BASE/mysql --status; + +status=$(mktemp); +$MYGOBIN/resource status fetch $BASE/mysql > $status + +test 1 == \ + $(grep "mysql" $status | grep "Deployment is available. Replicas: 1" | wc -l); \ + echo $? + +test 1 == \ + $(grep "mysql-pass2" $status | grep "Resource is always ready" | wc -l); \ + echo $? + +test 1 == \ + $(grep "mysql" $status | grep "Service is ready" | wc -l); \ + echo $? +``` + +Clean-up the cluster + +``` +$MYGOBIN/kind delete cluster; +``` \ No newline at end of file diff --git a/examples/alphaTestExamples/helloapp.md b/examples/alphaTestExamples/helloapp.md index f5b65922e..fbe681178 100644 --- a/examples/alphaTestExamples/helloapp.md +++ b/examples/alphaTestExamples/helloapp.md @@ -37,7 +37,7 @@ mkdir -p $BASE Now lets add a simple config map resource to the `base` - + ``` cat <$BASE/configMap.yaml apiVersion: v1 @@ -52,7 +52,7 @@ EOF Create `deployment.yaml` with any image and with desired number of replicas - + ``` cat <$BASE/deployment.yaml apiVersion: apps/v1 @@ -90,7 +90,7 @@ EOF Create `service.yaml` pointing to the deployment created above - + ``` cat <$BASE/service.yaml kind: Service @@ -110,7 +110,7 @@ EOF Create a `grouping.yaml` resource. By this, you are defining the grouping of the current directory, `base`. Kustomize uses the unique label in this file to track any future state changes made to this directory. Make sure the label key is `kustomize.config.k8s.io/inventory-id` and give any unique label value and DO NOT change it in future. - + ``` cat <$BASE/grouping.yaml apiVersion: v1 @@ -124,7 +124,7 @@ EOF Now, create `kustomization.yaml` add all your resources. - + ``` cat <$BASE/kustomization.yaml commonLabels: @@ -173,7 +173,7 @@ $MYGOBIN/kind create cluster; ``` Use the kustomize binary in MYGOBIN to apply a deployment, fetch the status and verify the status. - + ``` export KUSTOMIZE_ENABLE_ALPHA_COMMANDS=true @@ -195,11 +195,51 @@ test 1 == \ echo $? ``` +Now let's replace the configMap with configMap2 apply the config, fetch and verify the status. This should delete the-map from deployment and add the-map2. + +``` +cat <$BASE/configMap2.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: the-map2 +data: + altGreeting: "Good Evening!" + enableRisky: "false" +EOF + +rm $BASE/configMap.yaml + +sed -i.bak 's/configMap/configMap2/' \ + $BASE/kustomization.yaml + +sed -i.bak 's/the-map/the-map2/' \ + $BASE/deployment.yaml + +$MYGOBIN/kustomize resources apply $BASE --status; + +status=$(mktemp); +$MYGOBIN/resource status fetch $BASE > $status +$MYGOBIN/resource status fetch $BASE > /tmp/teste2eout/out.txt + +test 1 == \ + $(grep "the-deployment" $status | grep "Deployment is available. Replicas: 3" | wc -l); \ + echo $? + +test 1 == \ + $(grep "the-map2" $status | grep "Resource is always ready" | wc -l); \ + echo $? + +test 1 == \ + $(grep "the-service" $status | grep "Service is ready" | wc -l); \ + echo $? +``` + Clean-up the cluster - + ``` $MYGOBIN/kind delete cluster; ``` -###Next Exercise +### Next Exercise Create overlays as described in the [helloWorld] section and verify the results. Good luck! \ No newline at end of file From 1120c6bc7a6d13483750dfa0eaa6d85005bceec9 Mon Sep 17 00:00:00 2001 From: Haiyan Meng Date: Fri, 17 Jan 2020 16:14:09 -0800 Subject: [PATCH 10/46] Add a `User` field into Document to make it easy to aggregate on github user level. --- api/internal/crawl/crawler/crawler.go | 4 + api/internal/crawl/crawler/github/crawler.go | 2 + api/internal/crawl/doc/doc_test.go | 9 ++ api/internal/crawl/doc/docname.go | 37 ++++-- api/internal/crawl/doc/docname_test.go | 39 ++++++ api/internal/crawl/index/elasticsearch.go | 3 + api/internal/crawl/search_cmds/id.md | 12 ++ api/internal/crawl/search_cmds/user.md | 123 +++++++++++++++++++ 8 files changed, 221 insertions(+), 8 deletions(-) create mode 100644 api/internal/crawl/search_cmds/id.md create mode 100644 api/internal/crawl/search_cmds/user.md diff --git a/api/internal/crawl/crawler/crawler.go b/api/internal/crawl/crawler/crawler.go index 7861749f0..9a6a401d6 100644 --- a/api/internal/crawl/crawler/crawler.go +++ b/api/internal/crawl/crawler/crawler.go @@ -148,6 +148,10 @@ func doCrawl(ctx context.Context, docsPtr *CrawlSeed, crawlers []Crawler, conv C continue } + if tail.User == "" { + tail.User = doc.UserName(tail.RepositoryURL) + } + // If the Document represents a kustomization root, FetchDcoument will change // the `filePath` field of the Document by adding `kustomization.yaml` or // `kustomization.yml` or `kustomization` into the the field. diff --git a/api/internal/crawl/crawler/github/crawler.go b/api/internal/crawl/crawler/github/crawler.go index 70c1a23ca..f1b723c5b 100644 --- a/api/internal/crawl/crawler/github/crawler.go +++ b/api/internal/crawl/crawler/github/crawler.go @@ -284,6 +284,7 @@ func kustomizationResultAdapter(gcl GhClient, k GhFileSpec, seen utils.SeenMap, FilePath: k.Path, DefaultBranch: defaultBranch, RepositoryURL: k.Repository.URL, + User: doc.UserName(k.Repository.URL), } if seen.Seen(document.ID()) { @@ -301,6 +302,7 @@ func kustomizationResultAdapter(gcl GhClient, k GhFileSpec, seen utils.SeenMap, FilePath: k.Path, DefaultBranch: defaultBranch, RepositoryURL: k.Repository.URL, + User: doc.UserName(k.Repository.URL), }, } creationTime, err := gcl.GetFileCreationTime(k) diff --git a/api/internal/crawl/doc/doc_test.go b/api/internal/crawl/doc/doc_test.go index e66241287..0bc7bf3b1 100644 --- a/api/internal/crawl/doc/doc_test.go +++ b/api/internal/crawl/doc/doc_test.go @@ -216,22 +216,26 @@ resources: RepositoryURL: "sigs.k8s.io/kustomize", FilePath: "some/path/to/base", FileType: "resource", + User: "sigs.k8s.io", }, { RepositoryURL: "sigs.k8s.io/kustomize", FilePath: "some/path/to/otherbase", FileType: "resource", + User: "sigs.k8s.io", }, { RepositoryURL: "sigs.k8s.io/kustomize", FilePath: "some/path/to/kdir/file.yaml", FileType: "resource", + User: "sigs.k8s.io", }, { RepositoryURL: "https://github.com/kubernetes-sigs/kustomize", FilePath: "examples/helloWorld", DefaultBranch: "v3.1.0", FileType: "resource", + User: "kubernetes-sigs", }, }, }, @@ -317,11 +321,13 @@ transformers: RepositoryURL: "sigs.k8s.io/kustomize", FilePath: "some/path/to/kdir/gen.yaml", FileType: "generator", + User: "sigs.k8s.io", }, { RepositoryURL: "sigs.k8s.io/kustomize", FilePath: "some/path/to/kdir/file.yaml", FileType: "resource", + User: "sigs.k8s.io", }, }, }, @@ -352,16 +358,19 @@ transformers: RepositoryURL: "sigs.k8s.io/kustomize", FilePath: "some/path/to/kdir/tr.yaml", FileType: "transformer", + User: "sigs.k8s.io", }, { RepositoryURL: "sigs.k8s.io/kustomize", FilePath: "some/path/to/kdir/gen.yaml", FileType: "generator", + User: "sigs.k8s.io", }, { RepositoryURL: "sigs.k8s.io/kustomize", FilePath: "some/path/to/kdir/file.yaml", FileType: "resource", + User: "sigs.k8s.io", }, }, }, diff --git a/api/internal/crawl/doc/docname.go b/api/internal/crawl/doc/docname.go index a3cab942a..fe1698943 100644 --- a/api/internal/crawl/doc/docname.go +++ b/api/internal/crawl/doc/docname.go @@ -11,7 +11,10 @@ import ( ) type Document struct { - RepositoryURL string `json:"repositoryUrl,omitempty"` + RepositoryURL string `json:"repositoryUrl,omitempty"` + // User makes it easy to aggregate data in the user level instead + // of the repository level + User string `json:"user,omitempty"` FilePath string `json:"filePath,omitempty"` DefaultBranch string `json:"defaultBranch,omitempty"` DocumentData string `json:"document,omitempty"` @@ -30,6 +33,7 @@ func (doc *Document) GetDocument() *Document { func (doc *Document) Copy() *Document { return &Document{ RepositoryURL: doc.RepositoryURL, + User: doc.User, FilePath: doc.FilePath, DefaultBranch: doc.DefaultBranch, DocumentData: doc.DocumentData, @@ -56,6 +60,7 @@ func (doc *Document) FromRelativePath(newFile string) (Document, error) { RepositoryURL: repoSpec.Host + path.Clean(repoSpec.OrgRepo), FilePath: path.Clean(repoSpec.Path), DefaultBranch: repoSpec.Ref, + User: UserName(repoSpec.Host + path.Clean(repoSpec.OrgRepo)), }, nil } // else document is probably relative path. @@ -63,6 +68,7 @@ func (doc *Document) FromRelativePath(newFile string) (Document, error) { ret := Document{ RepositoryURL: doc.RepositoryURL, DefaultBranch: doc.DefaultBranch, + User: UserName(doc.RepositoryURL), } ogDir, _ := path.Split(doc.FilePath) @@ -87,13 +93,7 @@ func (doc *Document) ID() string { } func (doc *Document) RepositoryFullName() string { - url := strings.TrimRight(doc.RepositoryURL, "/") - - gitPrefix := "git@github.com:" - if strings.HasPrefix(url, gitPrefix) { - url = url[len(gitPrefix):] - } - + url := TrimUrl(doc.RepositoryURL) sections := strings.Split(url, "/") l := len(sections) if l < 2 { @@ -101,3 +101,24 @@ func (doc *Document) RepositoryFullName() string { } return path.Join(sections[l-2], sections[l-1]) } + +// TrimUrl removes all the trailing slashes and the "git@github.com:" prefix (if exists). +func TrimUrl(s string) string { + url := strings.TrimRight(s, "/") + + gitPrefix := "git@github.com:" + if strings.HasPrefix(url, gitPrefix) { + url = url[len(gitPrefix):] + } + return url +} + +func UserName(repositoryURL string) string { + url := TrimUrl(repositoryURL) + sections := strings.Split(url, "/") + l := len(sections) + if l < 2 { + return url + } + return sections[l-2] +} diff --git a/api/internal/crawl/doc/docname_test.go b/api/internal/crawl/doc/docname_test.go index a03beaf06..00ae08cff 100644 --- a/api/internal/crawl/doc/docname_test.go +++ b/api/internal/crawl/doc/docname_test.go @@ -28,6 +28,7 @@ func TestFromRelativePath(t *testing.T) { RepositoryURL: "example.com/repo", FilePath: "path/to/other/file/resource.yaml", DefaultBranch: "master", + User: "example.com", }, }, { @@ -36,6 +37,7 @@ func TestFromRelativePath(t *testing.T) { RepositoryURL: "example.com/repo", FilePath: "path/to/other/file/patch.yaml", DefaultBranch: "master", + User: "example.com", }, }, { @@ -44,6 +46,7 @@ func TestFromRelativePath(t *testing.T) { RepositoryURL: "example.com/repo", FilePath: "path/to/file/service.yaml", DefaultBranch: "master", + User: "example.com", }, }, }, @@ -109,3 +112,39 @@ func TestDocument_RepositoryFullName(t *testing.T) { } } } + +func TestDocument_UserName(t *testing.T) { + testCases := []struct { + repositoryURL string + expectedUserName string + }{ + { + repositoryURL: "https://github.com/user/repo", + expectedUserName: "user", + }, + { + repositoryURL: "https://github.com//user/repo////", + expectedUserName: "user", + }, + { + repositoryURL: "repo/", + expectedUserName: "repo", + }, + { + repositoryURL: "", + expectedUserName: "", + }, + { + repositoryURL: "git@github.com:user/repo", + expectedUserName: "user", + }, + } + + for _, tc := range testCases { + returnedUserName := UserName(tc.repositoryURL) + if returnedUserName != tc.expectedUserName { + t.Errorf("UserName expected %s, got %s", + tc.expectedUserName, returnedUserName) + } + } +} diff --git a/api/internal/crawl/index/elasticsearch.go b/api/internal/crawl/index/elasticsearch.go index ebd6b70c0..a1f465a36 100644 --- a/api/internal/crawl/index/elasticsearch.go +++ b/api/internal/crawl/index/elasticsearch.go @@ -20,6 +20,9 @@ const IndexConfig = ` "repositoryUrl": { "type": "keyword" }, + "user": { + "type": "keyword" + }, "filePath": { "type": "keyword" }, diff --git a/api/internal/crawl/search_cmds/id.md b/api/internal/crawl/search_cmds/id.md new file mode 100644 index 000000000..66d767eae --- /dev/null +++ b/api/internal/crawl/search_cmds/id.md @@ -0,0 +1,12 @@ +Find the document with the given `_id`: +``` +curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d' +{ + "query": { + "terms": { + "_id": [ "b3a03f3327841617db696e2d6abc30e1a1bd653f1a2bbce05637f7dcae1a43f7" ] + } + } +} +' +``` diff --git a/api/internal/crawl/search_cmds/user.md b/api/internal/crawl/search_cmds/user.md new file mode 100644 index 000000000..b8ae3538c --- /dev/null +++ b/api/internal/crawl/search_cmds/user.md @@ -0,0 +1,123 @@ +Find all the documents having the `user` field set: +``` +curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d' +{ + "query": { + "exists": { + "field": "user" + } + } +} +' +``` + +Find all the documents whose `user` field is not set: +``` +curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d' +{ + "size": 10000, + "query": { + "bool": { + "must_not": { + "exists": { + "field": "user" + } + } + } + } +} +' +``` + +Search for all the documents whose `user` field is `kubernetes-sigs`: +``` +curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d' +{ + "query": { + "bool": { + "filter": [ + { "regexp": { "user": "kubernetes-sigs" }} + ] + } + } +} +' +``` + +Count distinct values of the `user` field: +``` +curl -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' +{ + "aggs" : { + "user_count" : { + "cardinality" : { + "field" : "user", + "precision_threshold": 40000 + } + } + } +} +' +``` + +List all the values of the `user` field and the frequency of each value: +``` +curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' +{ + "aggs" : { + "user" : { + "terms" : { + "field" : "user" + } + } + } +} +' +``` + + +For all the kustomization files in the index, list all the values of the +`user` field and the frequency of each value: +``` +curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' +{ + "query": { + "bool": { + "filter": [ + { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" }} + ] + } + }, + "aggs" : { + "user" : { + "terms" : { + "field" : "user" + } + } + } +} +' +``` + +For all the non-kustomization files in the index, list all the values of the +`user` field and the frequency of each value: +``` +curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' +{ + "query": { + "bool": { + "must_not": { + "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" } + } + } + }, + "aggs" : { + "user" : { + "terms" : { + "field" : "user" + } + } + } +} +' +``` From 73d44f9d317f1518c854827210dba2b761fbb759 Mon Sep 17 00:00:00 2001 From: Morten Torkildsen Date: Fri, 17 Jan 2020 10:53:49 -0800 Subject: [PATCH 11/46] Fix printing to make sure always match setting colors with a reset --- cmd/resource/status/cmd/print.go | 43 ++++++++++++++++---------------- 1 file changed, 22 insertions(+), 21 deletions(-) diff --git a/cmd/resource/status/cmd/print.go b/cmd/resource/status/cmd/print.go index fbf872002..8af0aaeb5 100644 --- a/cmd/resource/status/cmd/print.go +++ b/cmd/resource/status/cmd/print.go @@ -19,11 +19,12 @@ const ( statusColumn = "status" messageColumn = "message" - ESC = 27 - RED color = 31 - GREEN color = 32 - YELLOW color = 33 - WHITE color = 37 + RESET = 0 + ESC = 27 + RED color = 31 + GREEN color = 32 + YELLOW color = 33 + DEFAULT color = -1 // This is not a valid ANSI escape code. It is used here to mean that no color should be set. ) type color int @@ -36,10 +37,6 @@ func eraseCurrentLine(w io.Writer) { printOrDie(w, "%c[2K\r", ESC) } -func setColor(w io.Writer, color color) { - printOrDie(w, "%c[%dm", ESC, color) -} - type colorFunc func(s status.Status) color type contentFunc func(resource ResourceStatusData) string @@ -51,7 +48,7 @@ type tableColumnInfo struct { } func defaultColorFunc(_ status.Status) color { - return WHITE + return DEFAULT } var ( @@ -137,7 +134,6 @@ func (s *TablePrinter) Print() { func (s *TablePrinter) PrintUntil(stop <-chan struct{}, interval time.Duration) <-chan struct{} { completed := make(chan struct{}) - setColor(s.out, WHITE) s.printTable(s.statusInfo.CurrentStatus(), false) go func() { defer close(completed) @@ -167,9 +163,7 @@ func (s *TablePrinter) printTable(data StatusData, deleteUp bool) { eraseCurrentLine(s.out) if s.showAggStatus { printOrDie(s.out, "AggregateStatus: ") - setColor(s.out, colorForStatus(data.AggregateStatus)) - printOrDie(s.out, "%s\n", data.AggregateStatus) - setColor(s.out, WHITE) + printWithColorOrDie(s.out, colorForStatus(data.AggregateStatus), "%s\n", data.AggregateStatus) } s.printTableRow(headers()) for _, resource := range data.ResourceStatuses { @@ -179,13 +173,12 @@ func (s *TablePrinter) printTable(data StatusData, deleteUp bool) { func (s *TablePrinter) printTableRow(rowData []RowData) { for i, row := range rowData { - setColor(s.out, row.color) + format := fmt.Sprintf("%%-%ds", row.width) - printOrDie(s.out, format, trimString(row.content, row.width)) + printWithColorOrDie(s.out, row.color, format, trimString(row.content, row.width)) if i != len(rowData)-1 { printOrDie(s.out, " ") } - setColor(s.out, WHITE) } printOrDie(s.out, "\n") } @@ -202,7 +195,7 @@ func headers() []RowData { column := tableColumns[columnName] headers = append(headers, RowData{ content: column.header, - color: WHITE, + color: DEFAULT, width: column.width, }) } @@ -331,18 +324,26 @@ func printOrDie(w io.Writer, format string, a ...interface{}) { } } +func printWithColorOrDie(w io.Writer, color color, format string, a ...interface{}) { + if color == DEFAULT { + printOrDie(w, format, a...) + } else { + printOrDie(w, "%c[%dm", ESC, color) + printOrDie(w, format, a...) + printOrDie(w, "%c[%dm", ESC, RESET) + } +} + func colorForStatus(s status.Status) color { switch s { case status.CurrentStatus: return GREEN - case status.UnknownStatus: - return WHITE case status.InProgressStatus: return YELLOW case status.FailedStatus: return RED } - return WHITE + return DEFAULT } func trimString(str string, maxLength int) string { From 3623d9205efba2d02106e9135f82f7ed8be985d8 Mon Sep 17 00:00:00 2001 From: Sean Sullivan Date: Tue, 21 Jan 2020 15:18:22 -0800 Subject: [PATCH 12/46] Adds new helper function retrieveGroupingLabel() --- cmd/kubectl/kubectlcobra/grouping.go | 30 ++++++++--- cmd/kubectl/kubectlcobra/grouping_test.go | 65 ++++++++++++++++++++++- 2 files changed, 87 insertions(+), 8 deletions(-) diff --git a/cmd/kubectl/kubectlcobra/grouping.go b/cmd/kubectl/kubectlcobra/grouping.go index 2f50881bc..c9e9cd090 100644 --- a/cmd/kubectl/kubectlcobra/grouping.go +++ b/cmd/kubectl/kubectlcobra/grouping.go @@ -22,6 +22,26 @@ const ( GroupingHash = "kustomize.config.k8s.io/inventory-hash" ) +// retrieveGroupingLabel returns the string value of the GroupingLabel +// for the passed object. Returns error if the passed object is nil or +// is not a grouping object. +func retrieveGroupingLabel(obj runtime.Object) (string, error) { + var groupingLabel string + if obj == nil { + return "", fmt.Errorf("Grouping object is nil.\n") + } + accessor, err := meta.Accessor(obj) + if err != nil { + return "", err + } + labels := accessor.GetLabels() + groupingLabel, exists := labels[GroupingLabel] + if !exists { + return "", fmt.Errorf("Grouping label does not exist for grouping object: %s\n", GroupingLabel) + } + return strings.TrimSpace(groupingLabel), nil +} + // isGroupingObject returns true if the passed object has the // grouping label. // TODO(seans3): Check type is ConfigMap. @@ -29,13 +49,9 @@ func isGroupingObject(obj runtime.Object) bool { if obj == nil { return false } - accessor, err := meta.Accessor(obj) - if err == nil { - labels := accessor.GetLabels() - _, exists := labels[GroupingLabel] - if exists { - return true - } + groupingLabel, err := retrieveGroupingLabel(obj) + if err == nil && len(groupingLabel) > 0 { + return true } return false } diff --git a/cmd/kubectl/kubectlcobra/grouping_test.go b/cmd/kubectl/kubectlcobra/grouping_test.go index f2cca5ad8..7b0786f23 100644 --- a/cmd/kubectl/kubectlcobra/grouping_test.go +++ b/cmd/kubectl/kubectlcobra/grouping_test.go @@ -22,6 +22,8 @@ var pod1Name = "pod-1" var pod2Name = "pod-2" var pod3Name = "pod-3" +var testGroupingLabel = "test-app-label" + var groupingObj = unstructured.Unstructured{ Object: map[string]interface{}{ "apiVersion": "v1", @@ -30,7 +32,7 @@ var groupingObj = unstructured.Unstructured{ "name": groupingObjName, "namespace": testNamespace, "labels": map[string]interface{}{ - GroupingLabel: "true", + GroupingLabel: testGroupingLabel, }, }, }, @@ -109,6 +111,67 @@ var nilInfo = &resource.Info{ Object: nil, } +var groupingObjLabelWithSpace = unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "v1", + "kind": "ConfigMap", + "metadata": map[string]interface{}{ + "name": groupingObjName, + "namespace": testNamespace, + "labels": map[string]interface{}{ + GroupingLabel: "\tgrouping-label ", + }, + }, + }, +} + +func TestRetrieveGroupingLabel(t *testing.T) { + tests := []struct { + obj runtime.Object + groupingLabel string + isError bool + }{ + // Nil grouping object throws error. + { + obj: nil, + groupingLabel: "", + isError: true, + }, + // Pod is not a grouping object. + { + obj: &pod2, + groupingLabel: "", + isError: true, + }, + // Retrieves label without preceding/trailing whitespace. + { + obj: &groupingObjLabelWithSpace, + groupingLabel: "grouping-label", + isError: false, + }, + { + obj: &groupingObj, + groupingLabel: testGroupingLabel, + isError: false, + }, + } + + for _, test := range tests { + actual, err := retrieveGroupingLabel(test.obj) + if test.isError && err == nil { + t.Errorf("Did not receive expected error.\n") + } + if !test.isError { + if err != nil { + t.Fatalf("Received unexpected error: %s\n", err) + } + if test.groupingLabel != actual { + t.Errorf("Expected grouping label (%s), got (%s)\n", test.groupingLabel, actual) + } + } + } +} + func TestIsGroupingObject(t *testing.T) { tests := []struct { obj runtime.Object From bf2e398b339173ae6f5e9bc7e99abeed1bfc0b58 Mon Sep 17 00:00:00 2001 From: Sean Sullivan Date: Tue, 21 Jan 2020 15:56:48 -0800 Subject: [PATCH 13/46] Adds InventorySet Equals() function --- cmd/kubectl/kubectlcobra/inventory.go | 10 ++++ cmd/kubectl/kubectlcobra/inventory_test.go | 57 ++++++++++++++++++++++ 2 files changed, 67 insertions(+) diff --git a/cmd/kubectl/kubectlcobra/inventory.go b/cmd/kubectl/kubectlcobra/inventory.go index 336bbb5e4..2c2913578 100644 --- a/cmd/kubectl/kubectlcobra/inventory.go +++ b/cmd/kubectl/kubectlcobra/inventory.go @@ -164,6 +164,16 @@ func (is *InventorySet) Subtract(other *InventorySet) (*InventorySet, error) { return result, nil } +// Equals returns true if the "other" inventory set is the same +// as this current inventory set. Relies on the fact that the +// inventory items are sorted for the String() function. +func (is *InventorySet) Equals(other *InventorySet) bool { + if other == nil { + return false + } + return is.String() == other.String() +} + // String returns a string describing set of Inventory structs. func (is *InventorySet) String() string { strs := []string{} diff --git a/cmd/kubectl/kubectlcobra/inventory_test.go b/cmd/kubectl/kubectlcobra/inventory_test.go index 93fd9a0e8..f5b5006da 100644 --- a/cmd/kubectl/kubectlcobra/inventory_test.go +++ b/cmd/kubectl/kubectlcobra/inventory_test.go @@ -487,3 +487,60 @@ func TestInventorySetSubtract(t *testing.T) { } } } + +func TestInventorySetEquals(t *testing.T) { + tests := []struct { + set1 []*Inventory + set2 []*Inventory + isEqual bool + }{ + { + set1: []*Inventory{}, + set2: []*Inventory{&inventory1}, + isEqual: false, + }, + { + set1: []*Inventory{&inventory1}, + set2: []*Inventory{}, + isEqual: false, + }, + { + set1: []*Inventory{&inventory1, &inventory2}, + set2: []*Inventory{&inventory1}, + isEqual: false, + }, + { + set1: []*Inventory{&inventory1, &inventory2}, + set2: []*Inventory{&inventory3, &inventory4}, + isEqual: false, + }, + // Empty sets are equal. + { + set1: []*Inventory{}, + set2: []*Inventory{}, + isEqual: true, + }, + { + set1: []*Inventory{&inventory1}, + set2: []*Inventory{&inventory1}, + isEqual: true, + }, + // Ordering of the inventory items does not matter for equality. + { + set1: []*Inventory{&inventory1, &inventory2}, + set2: []*Inventory{&inventory2, &inventory1}, + isEqual: true, + }, + } + + for _, test := range tests { + invSet1 := NewInventorySet(test.set1) + invSet2 := NewInventorySet(test.set2) + if !invSet1.Equals(invSet2) && test.isEqual { + t.Errorf("Expected equal inventory sets; got unequal (%s)/(%s)\n", invSet1, invSet2) + } + if invSet1.Equals(invSet2) && !test.isEqual { + t.Errorf("Expected inequal inventory sets; got equal (%s)/(%s)\n", invSet1, invSet2) + } + } +} From 8e5bce17dc8922366243ecda669c914a456f1d1b Mon Sep 17 00:00:00 2001 From: Jeffrey Regan Date: Tue, 21 Jan 2020 16:23:44 -0800 Subject: [PATCH 14/46] Move status command to kustomize. --- .golangci-kustomize.yml | 9 +- Makefile | 2 - cmd/resource/LICENSE_TEMPLATE | 2 - cmd/resource/Makefile | 39 -- cmd/resource/fixgomod.sh | 20 - cmd/resource/go.mod | 21 - cmd/resource/go.sum | 444 ------------------ cmd/resource/main.go | 29 -- cmd/resource/status/status.go | 24 - .../alphaTestExamples/MultipleDeployments.md | 43 +- examples/alphaTestExamples/helloapp.md | 39 +- kustomize/go.mod | 6 + kustomize/go.sum | 20 +- kustomize/internal/commands/commands.go | 2 + kustomize/internal/commands/status/all.go | 31 ++ .../internal/commands}/status/cmd/events.go | 2 +- .../commands}/status/cmd/events_test.go | 0 .../internal/commands}/status/cmd/fetch.go | 2 +- .../commands}/status/cmd/fetch_test.go | 0 .../commands}/status/cmd/helpers_test.go | 0 .../internal/commands}/status/cmd/print.go | 0 .../internal/commands}/status/cmd/util.go | 0 .../commands}/status/cmd/util_test.go | 31 +- .../internal/commands}/status/cmd/wait.go | 2 +- .../commands}/status/cmd/wait_test.go | 0 .../commands}/status/docs/commands/events.md | 0 .../commands}/status/docs/commands/fetch.md | 0 .../commands}/status/docs/commands/wait.md | 0 .../status/generateddocs/commands/docs.go | 0 29 files changed, 119 insertions(+), 649 deletions(-) delete mode 100644 cmd/resource/LICENSE_TEMPLATE delete mode 100644 cmd/resource/Makefile delete mode 100755 cmd/resource/fixgomod.sh delete mode 100644 cmd/resource/go.mod delete mode 100644 cmd/resource/go.sum delete mode 100644 cmd/resource/main.go delete mode 100644 cmd/resource/status/status.go create mode 100644 kustomize/internal/commands/status/all.go rename {cmd/resource => kustomize/internal/commands}/status/cmd/events.go (96%) rename {cmd/resource => kustomize/internal/commands}/status/cmd/events_test.go (100%) rename {cmd/resource => kustomize/internal/commands}/status/cmd/fetch.go (97%) rename {cmd/resource => kustomize/internal/commands}/status/cmd/fetch_test.go (100%) rename {cmd/resource => kustomize/internal/commands}/status/cmd/helpers_test.go (100%) rename {cmd/resource => kustomize/internal/commands}/status/cmd/print.go (100%) rename {cmd/resource => kustomize/internal/commands}/status/cmd/util.go (100%) rename {cmd/resource => kustomize/internal/commands}/status/cmd/util_test.go (54%) rename {cmd/resource => kustomize/internal/commands}/status/cmd/wait.go (98%) rename {cmd/resource => kustomize/internal/commands}/status/cmd/wait_test.go (100%) rename {cmd/resource => kustomize/internal/commands}/status/docs/commands/events.md (100%) rename {cmd/resource => kustomize/internal/commands}/status/docs/commands/fetch.md (100%) rename {cmd/resource => kustomize/internal/commands}/status/docs/commands/wait.md (100%) rename {cmd/resource => kustomize/internal/commands}/status/generateddocs/commands/docs.go (100%) diff --git a/.golangci-kustomize.yml b/.golangci-kustomize.yml index 097af0f10..003110429 100644 --- a/.golangci-kustomize.yml +++ b/.golangci-kustomize.yml @@ -7,8 +7,13 @@ linters: - bodyclose - deadcode - depguard + # - dogsled - dupl + # - errcheck + # - funlen + # - gochecknoinits - goconst + # - gocritic - gocyclo - gofmt - goimports @@ -21,6 +26,7 @@ linters: - lll - misspell - nakedret + # - scopelint - staticcheck - structcheck # stylecheck demands that acronyms not be treated as words @@ -28,9 +34,10 @@ linters: # - stylecheck - typecheck - unconvert - - unused - unparam + - unused - varcheck + # - whitespace linters-settings: dupl: diff --git a/Makefile b/Makefile index 95a9b8711..97cf480fd 100644 --- a/Makefile +++ b/Makefile @@ -208,8 +208,6 @@ test-examples-e2e-kustomize: $(MYGOBIN)/mdrip $(MYGOBIN)/kind /bin/rm -f $(MYGOBIN)/kustomize; \ echo "Installing kustomize from ."; \ cd kustomize; go install .; cd ..; \ - echo "Installing resource from ."; \ - cd cmd/resource; go install .; cd ../..; \ ./hack/testExamplesE2EAgainstKustomize.sh .; \ ) diff --git a/cmd/resource/LICENSE_TEMPLATE b/cmd/resource/LICENSE_TEMPLATE deleted file mode 100644 index 0c2b3b655..000000000 --- a/cmd/resource/LICENSE_TEMPLATE +++ /dev/null @@ -1,2 +0,0 @@ -Copyright {{.Year}} {{.Holder}} -SPDX-License-Identifier: Apache-2.0 diff --git a/cmd/resource/Makefile b/cmd/resource/Makefile deleted file mode 100644 index 29d262a6a..000000000 --- a/cmd/resource/Makefile +++ /dev/null @@ -1,39 +0,0 @@ -# Copyright 2019 The Kubernetes Authors. -# SPDX-License-Identifier: Apache-2.0 - -.PHONY: generate license fix vet fmt test build tidy - -GOBIN := $(shell go env GOPATH)/bin - -build: - go build -v -o $(GOBIN)/resource . - -all: generate build license fix vet fmt test lint tidy - -fix: - go fix ./... - -fmt: - go fmt ./... - -generate: - (which $(GOBIN)/mdtogo || go get sigs.k8s.io/kustomize/cmd/mdtogo) - GOBIN=$(GOBIN) go generate ./... - -license: - (which $(GOBIN)/addlicense || go get github.com/google/addlicense) - $(GOBIN)/addlicense -y 2019 -c "The Kubernetes Authors." -f LICENSE_TEMPLATE . - -tidy: - go mod tidy - -lint: - (which $(GOBIN)/golangci-lint || go get github.com/golangci/golangci-lint/cmd/golangci-lint@v1.19.1) - $(GOBIN)/golangci-lint run ./... - -test: - go test -cover ./... - -vet: - go vet ./... - diff --git a/cmd/resource/fixgomod.sh b/cmd/resource/fixgomod.sh deleted file mode 100755 index fdf07ff61..000000000 --- a/cmd/resource/fixgomod.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash -# Copyright 2019 The Kubernetes Authors. -# SPDX-License-Identifier: Apache-2.0 - -set -e - -: "${kyaml_major?Need to source VERSIONS}" -: "${kyaml_minor?Need to source VERSIONS}" -: "${kyaml_patch?Need to source VERSIONS}" - -: "${kstatus_major?Need to source VERSIONS}" -: "${kstatus_minor?Need to source VERSIONS}" -: "${kstatus_patch?Need to source VERSIONS}" - - -go mod edit -dropreplace=sigs.k8s.io/kustomize/kyaml@v0.0.0 -go mod edit -require=sigs.k8s.io/kustomize/kyaml@v$kyaml_major.$kyaml_minor.$kyaml_patch - -go mod edit -dropreplace=sigs.k8s.io/kustomize/kstatus@v0.0.0 -go mod edit -require=sigs.k8s.io/kustomize/kstatus@v$kstatus_major.$kstatus_minor.$kstatus_patch diff --git a/cmd/resource/go.mod b/cmd/resource/go.mod deleted file mode 100644 index f1ac2cec8..000000000 --- a/cmd/resource/go.mod +++ /dev/null @@ -1,21 +0,0 @@ -module sigs.k8s.io/kustomize/cmd/resource - -go 1.12 - -require ( - github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d - github.com/pkg/errors v0.8.1 - github.com/spf13/cobra v0.0.5 - github.com/stretchr/testify v1.4.0 - k8s.io/api v0.17.0 - k8s.io/apimachinery v0.17.0 - k8s.io/client-go v0.17.0 - sigs.k8s.io/controller-runtime v0.4.0 - sigs.k8s.io/kustomize/kstatus v0.0.0 - sigs.k8s.io/kustomize/kyaml v0.0.0 -) - -replace ( - sigs.k8s.io/kustomize/kstatus v0.0.0 => ../../kstatus - sigs.k8s.io/kustomize/kyaml v0.0.0 => ../../kyaml -) diff --git a/cmd/resource/go.sum b/cmd/resource/go.sum deleted file mode 100644 index e49e8f56e..000000000 --- a/cmd/resource/go.sum +++ /dev/null @@ -1,444 +0,0 @@ -cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.38.0 h1:ROfEUZz+Gh5pa62DJWXSaonyu3StP6EA6lPEXPI6mCo= -cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= -github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= -github.com/Azure/go-autorest/autorest v0.9.0 h1:MRvx8gncNaXJqOoLmhNjUAKh33JJF8LyxPhomEtOsjs= -github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI= -github.com/Azure/go-autorest/autorest/adal v0.5.0 h1:q2gDruN08/guU9vAjuPWff0+QIrpH6ediguzdAzXAUU= -github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0= -github.com/Azure/go-autorest/autorest/date v0.1.0 h1:YGrhWfrgtFs84+h0o46rJrlmsZtyZRg470CqAXTZaGM= -github.com/Azure/go-autorest/autorest/date v0.1.0/go.mod h1:plvfp3oPSKwf2DNjlBjWF/7vwR+cUD/ELuzDCXwHUVA= -github.com/Azure/go-autorest/autorest/mocks v0.1.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= -github.com/Azure/go-autorest/autorest/mocks v0.2.0 h1:Ww5g4zThfD/6cLb4z6xxgeyDa7QDkizMkJKe0ysZXp0= -github.com/Azure/go-autorest/autorest/mocks v0.2.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= -github.com/Azure/go-autorest/logger v0.1.0 h1:ruG4BSDXONFRrZZJ2GUXDiUyVpayPmb1GnWeHDdaNKY= -github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc= -github.com/Azure/go-autorest/tracing v0.5.0 h1:TRn4WjSnkcSy5AEG3pnbtFSwNtwzjr4VYyQflFE619k= -github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk= -github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= -github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= -github.com/PuerkitoBio/purell v1.1.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= -github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI= -github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= -github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= -github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M= -github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= -github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpHMqeKTCYkitsPqHNxTmd4SNR5r94FGM8= -github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo= -github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= -github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= -github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= -github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0= -github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= -github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= -github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/coreos/bbolt v1.3.1-coreos.6/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= -github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= -github.com/coreos/etcd v3.3.15+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= -github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= -github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= -github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/pkg v0.0.0-20180108230652-97fdf19511ea/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= -github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= -github.com/davecgh/go-spew v0.0.0-20151105211317-5215b55f46b2/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= -github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= -github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= -github.com/docker/docker v0.7.3-0.20190327010347-be7ac8be2ae0/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= -github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= -github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= -github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= -github.com/elazarl/goproxy v0.0.0-20170405201442-c4fc26588b6e/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= -github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= -github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= -github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= -github.com/evanphx/json-patch v4.5.0+incompatible h1:ouOWdg56aJriqS0huScTkVXPC5IcNrDCXZ6OoTAWu7M= -github.com/evanphx/json-patch v4.5.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= -github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= -github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= -github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= -github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= -github.com/globalsign/mgo v0.0.0-20180905125535-1ca0a4f7cbcb/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= -github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= -github.com/go-errors/errors v1.0.1 h1:LUHzmkK3GUKUrL/1gfBUxAHzcev3apQlezX/+O7ma6w= -github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= -github.com/go-logr/logr v0.1.0 h1:M1Tv3VzNlEHg6uyACnRdtrploV2P7wZqH8BoQMtz0cg= -github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= -github.com/go-logr/zapr v0.1.0 h1:h+WVe9j6HAA01niTJPA/kKH0i7e0rLZBCwauQFcRE54= -github.com/go-logr/zapr v0.1.0/go.mod h1:tabnROwaDl0UNxkVeFRbY8bwB37GwRv0P8lg6aAiEnk= -github.com/go-openapi/analysis v0.0.0-20180825180245-b006789cd277/go.mod h1:k70tL6pCuVxPJOHXQ+wIac1FUrvNkHolPie/cLEU6hI= -github.com/go-openapi/analysis v0.17.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik= -github.com/go-openapi/analysis v0.18.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik= -github.com/go-openapi/analysis v0.19.2/go.mod h1:3P1osvZa9jKjb8ed2TPng3f0i/UY9snX6gxi44djMjk= -github.com/go-openapi/errors v0.17.0/go.mod h1:LcZQpmvG4wyF5j4IhA73wkLFQg+QJXOQHVjmcZxhka0= -github.com/go-openapi/errors v0.18.0/go.mod h1:LcZQpmvG4wyF5j4IhA73wkLFQg+QJXOQHVjmcZxhka0= -github.com/go-openapi/errors v0.19.2/go.mod h1:qX0BLWsyaKfvhluLejVpVNwNRdXZhEbTA4kxxpKBC94= -github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+35s3my2LFTysnkMfxsJBAMHj/DoqoB9knIWoYG/Vk0= -github.com/go-openapi/jsonpointer v0.17.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M= -github.com/go-openapi/jsonpointer v0.18.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M= -github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= -github.com/go-openapi/jsonpointer v0.19.3 h1:gihV7YNZK1iK6Tgwwsxo2rJbD1GTbdm72325Bq8FI3w= -github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= -github.com/go-openapi/jsonreference v0.0.0-20160704190145-13c6e3589ad9/go.mod h1:W3Z9FmVs9qj+KR4zFKmDPGiLdk1D9Rlm7cyMvf57TTg= -github.com/go-openapi/jsonreference v0.17.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I= -github.com/go-openapi/jsonreference v0.18.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I= -github.com/go-openapi/jsonreference v0.19.2 h1:o20suLFB4Ri0tuzpWtyHlh7E7HnkqTNLq6aR6WVNS1w= -github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc= -github.com/go-openapi/loads v0.17.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= -github.com/go-openapi/loads v0.18.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= -github.com/go-openapi/loads v0.19.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= -github.com/go-openapi/loads v0.19.2/go.mod h1:QAskZPMX5V0C2gvfkGZzJlINuP7Hx/4+ix5jWFxsNPs= -github.com/go-openapi/runtime v0.0.0-20180920151709-4f900dc2ade9/go.mod h1:6v9a6LTXWQCdL8k1AO3cvqx5OtZY/Y9wKTgaoP6YRfA= -github.com/go-openapi/runtime v0.19.0/go.mod h1:OwNfisksmmaZse4+gpV3Ne9AyMOlP1lt4sK4FXt0O64= -github.com/go-openapi/spec v0.0.0-20160808142527-6aced65f8501/go.mod h1:J8+jY1nAiCcj+friV/PDoE1/3eeccG9LYBs0tYvLOWc= -github.com/go-openapi/spec v0.17.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI= -github.com/go-openapi/spec v0.18.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI= -github.com/go-openapi/spec v0.19.2/go.mod h1:sCxk3jxKgioEJikev4fgkNmwS+3kuYdJtcsZsD5zxMY= -github.com/go-openapi/spec v0.19.5 h1:Xm0Ao53uqnk9QE/LlYV5DEU09UAgpliA85QoT9LzqPw= -github.com/go-openapi/spec v0.19.5/go.mod h1:Hm2Jr4jv8G1ciIAo+frC/Ft+rR2kQDh8JHKHb3gWUSk= -github.com/go-openapi/strfmt v0.17.0/go.mod h1:P82hnJI0CXkErkXi8IKjPbNBM6lV6+5pLP5l494TcyU= -github.com/go-openapi/strfmt v0.18.0/go.mod h1:P82hnJI0CXkErkXi8IKjPbNBM6lV6+5pLP5l494TcyU= -github.com/go-openapi/strfmt v0.19.0/go.mod h1:+uW+93UVvGGq2qGaZxdDeJqSAqBqBdl+ZPMF/cC8nDY= -github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dpr1UfpPtxFw+EFuQ41HhCWZfha5jSVRG7C7I= -github.com/go-openapi/swag v0.17.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg= -github.com/go-openapi/swag v0.18.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg= -github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= -github.com/go-openapi/swag v0.19.5 h1:lTz6Ys4CmqqCQmZPBlbQENR1/GucA2bzYTE12Pw4tFY= -github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= -github.com/go-openapi/validate v0.18.0/go.mod h1:Uh4HdOzKt19xGIGm1qHf/ofbX1YQ4Y+MYsct2VUrAJ4= -github.com/go-openapi/validate v0.19.2/go.mod h1:1tRCw7m3jtI8eNWEEliiAqUIcBztB2KDnRCRMUi7GTA= -github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= -github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= -github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20180513044358-24b0969c4cb7 h1:u4bArs140e9+AfE52mFHOXVFnOSBJBRlzTHrOPLOIhE= -github.com/golang/groupcache v0.0.0-20180513044358-24b0969c4cb7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/protobuf v0.0.0-20161109072736-4bd1920723d7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.0.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= -github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= -github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg= -github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/gofuzz v0.0.0-20161122191042-44d81051d367/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI= -github.com/google/gofuzz v1.0.0 h1:A8PeW59pxE9IoFRqBp37U+mSNaQoZ46F1f0f863XSXw= -github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= -github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= -github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= -github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= -github.com/googleapis/gnostic v0.3.1 h1:WeAefnSUHlBb0iJKwxFDZdbfGwkd7xRNuV+IpXMJhYk= -github.com/googleapis/gnostic v0.3.1/go.mod h1:on+2t9HRStVgn95RSsFWFz+6Q0Snyqv1awfrALZdbtU= -github.com/gophercloud/gophercloud v0.1.0 h1:P/nh25+rzXouhytV2pUHBb65fnds26Ghl8/391+sT5o= -github.com/gophercloud/gophercloud v0.1.0/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEoIEcSTewFxm1c5g8= -github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= -github.com/gregjones/httpcache v0.0.0-20170728041850-787624de3eb7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= -github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= -github.com/grpc-ecosystem/go-grpc-middleware v0.0.0-20190222133341-cfaf5686ec79/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= -github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= -github.com/grpc-ecosystem/grpc-gateway v1.3.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw= -github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU= -github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= -github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= -github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= -github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28= -github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= -github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= -github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= -github.com/json-iterator/go v0.0.0-20180612202835-f2b4162afba3/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= -github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= -github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.8 h1:QiWkFLKq0T7mpzwOTu6BzNDbfTE8OLrYhVKYMLF46Ok= -github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= -github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= -github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= -github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= -github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= -github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e h1:hB2xlXdHp/pmPZq0y3QnmWAArdw9PqbmotexnWx/FU8= -github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= -github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v0.0.0-20180320133207-05fbef0ca5da/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= -github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= -github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.4.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.10.1 h1:q/mM8GF/n0shIN8SaAZ0V+jnLPzen6WIVZdiwrRlMlo= -github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= -github.com/onsi/gomega v1.3.0/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= -github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME= -github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= -github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= -github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= -github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= -github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= -github.com/prometheus/client_golang v0.9.2 h1:awm861/B8OKDd2I/6o1dy3ra4BamzKhYOiGItCeZ740= -github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM= -github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910 h1:idejC8f05m9MGOsuEi1ATq9shN03HrxNkD/luQvxCv8= -github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= -github.com/prometheus/common v0.0.0-20181126121408-4724e9255275 h1:PnBWHBf+6L0jOqq0gIVUe6Yk0/QMZ640k6NvkxcBf+8= -github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= -github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a h1:9a8MnZMP0X2nLJdBg+pBmGgkJlSaKC2KaQmTCk1XDtE= -github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446/go.mod h1:uYEyJGbgTkfkS4+E/PavXkNJcbFIpEtjt2B0KDQ5+9M= -github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= -github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= -github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= -github.com/soheilhy/cmux v0.1.3/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= -github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= -github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= -github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cobra v0.0.5 h1:f0B+LkLX6DtmRH1isoNA9VTtNUK9K8xYd28JNNfOv/s= -github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= -github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= -github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= -github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= -github.com/stretchr/testify v0.0.0-20151208002404-e3a8ff8ce365/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= -github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= -github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= -github.com/xiang90/probing v0.0.0-20160813154853-07dd2e8dfe18/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= -github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca h1:1CFlNzQhALwjS9mBAUkycX616GzgsuYUOCHA5+HSlXI= -github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg= -github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= -go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= -go.uber.org/atomic v0.0.0-20181018215023-8dc6146f7569/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= -go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= -go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU= -go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= -go.uber.org/multierr v0.0.0-20180122172545-ddea229ff1df/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= -go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI= -go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= -go.uber.org/zap v0.0.0-20180814183419-67bc79d13d15/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= -go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= -go.uber.org/zap v1.10.0 h1:ORx85nbTijNz8ljznvCMR1ZBIPKFn3jQrag10X2AsuM= -go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= -golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190320223903-b7391e95e576/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7 h1:0hQKqeLdqlt5iIwVOBErRisrHJAN57yOiPRQItI20fU= -golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190312203227-4b39c73a6495/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= -golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= -golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= -golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= -golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180112015858-5ccada7d0a7b/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181005035420-146acd28ed58/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190320064053-1272bf9dcd53/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190812203447-cdfb69ac37fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191004110552-13f9640d40b9 h1:rjwSpXsdiK0dV8/Naq3kAw9ymfAeJIyd0upUIElB+lI= -golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 h1:SVwTIAaPC2U/AvvLNZ2a7OVsmBpC8L5BlwK1whH3hm0= -golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180117170059-2c42eef0765b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190209173611-3b5209105503/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190321052220-f7bb7a8bee54/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190911201528-7ad0cfa0b7b5 h1:SW/0nsKCUaozCUtZTakri5laocGx/5bkDSSLrFUsa5s= -golang.org/x/sys v0.0.0-20190911201528-7ad0cfa0b7b5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.1-0.20171227012246-e19ae1496984/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= -golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= -golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20191024005414-555d28b269f0 h1:/5xXl8Y5W96D+TtHSlonuFqGHIWVuyCkGJLwGh9JJFs= -golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20181011042414-1f849cf54d09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190206041539-40960b6deb8e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= -golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7 h1:9zdDQZ7Thm29KFXgAX/+yaf3eVbP7djjWp/dXAppNCc= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -gomodules.xyz/jsonpatch/v2 v2.0.1 h1:xyiBuvkD2g5n7cYzx6u2sxQvsAy4QJsZFCzGVdzOXZ0= -gomodules.xyz/jsonpatch/v2 v2.0.1/go.mod h1:IhYNNY4jnS53ZnfE4PAmpKtDpTCj1JFXc+3mwe7XcUU= -gonum.org/v1/gonum v0.0.0-20190331200053-3d26580ed485/go.mod h1:2ltnJ7xHfj0zHS40VVPYEAAMTa3ZGguvHGBSJeRWqE0= -gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= -gonum.org/v1/netlib v0.0.0-20190331212654-76723241ea4e/go.mod h1:kS+toOQn6AQKjmKJ7gzohV1XkqsFehRA2FbsbkopSuQ= -google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= -google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= -google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.5.0 h1:KxkO13IPW4Lslp2bz+KHP2E3gtFlrIGNThxkZQ3g+4c= -google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873 h1:nfPFGzJkUDX6uBmpN/pSw7MbOAWegH5QDQuoXFHedLg= -google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/grpc v1.23.0 h1:AzbTB6ux+okLTzP8Ru1Xs41C303zdcfEht7MQnYJt5A= -google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= -gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= -gopkg.in/inf.v0 v0.9.0/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= -gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= -gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= -gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= -gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= -gopkg.in/yaml.v2 v2.0.0/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= -gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I= -gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v3 v3.0.0-20191026110619-0b21df46bc1d h1:LCPbGQ34PMrwad11aMZ+dbz5SAsq/0ySjRwQ8I9Qwd8= -gopkg.in/yaml.v3 v3.0.0-20191026110619-0b21df46bc1d/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= -honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -k8s.io/api v0.0.0-20190918155943-95b840bb6a1f h1:8FRUST8oUkEI45WYKyD8ed7Ad0Kg5v11zHyPkEVb2xo= -k8s.io/api v0.0.0-20190918155943-95b840bb6a1f/go.mod h1:uWuOHnjmNrtQomJrvEBg0c0HRNyQ+8KTEERVsK0PW48= -k8s.io/api v0.17.0 h1:H9d/lw+VkZKEVIUc8F3wgiQ+FUXTTr21M87jXLU7yqM= -k8s.io/api v0.17.0/go.mod h1:npsyOePkeP0CPwyGfXDHxvypiYMJxBWAMpQxCaJ4ZxI= -k8s.io/apiextensions-apiserver v0.0.0-20190918161926-8f644eb6e783 h1:V6ndwCPoao1yZ52agqOKaUAl7DYWVGiXjV7ePA2i610= -k8s.io/apiextensions-apiserver v0.0.0-20190918161926-8f644eb6e783/go.mod h1:xvae1SZB3E17UpV59AWc271W/Ph25N+bjPyR63X6tPY= -k8s.io/apimachinery v0.0.0-20190913080033-27d36303b655 h1:CS1tBQz3HOXiseWZu6ZicKX361CZLT97UFnnPx0aqBw= -k8s.io/apimachinery v0.0.0-20190913080033-27d36303b655/go.mod h1:nL6pwRT8NgfF8TT68DBI8uEePRt89cSvoXUVqbkWHq4= -k8s.io/apimachinery v0.17.0 h1:xRBnuie9rXcPxUkDizUsGvPf1cnlZCFu210op7J7LJo= -k8s.io/apimachinery v0.17.0/go.mod h1:b9qmWdKlLuU9EBh+06BtLcSf/Mu89rWL33naRxs1uZg= -k8s.io/apiserver v0.0.0-20190918160949-bfa5e2e684ad/go.mod h1:XPCXEwhjaFN29a8NldXA901ElnKeKLrLtREO9ZhFyhg= -k8s.io/client-go v0.0.0-20190918160344-1fbdaa4c8d90 h1:mLmhKUm1X+pXu0zXMEzNsOF5E2kKFGe5o6BZBIIqA6A= -k8s.io/client-go v0.0.0-20190918160344-1fbdaa4c8d90/go.mod h1:J69/JveO6XESwVgG53q3Uz5OSfgsv4uxpScmmyYOOlk= -k8s.io/client-go v0.17.0 h1:8QOGvUGdqDMFrm9sD6IUFl256BcffynGoe80sxgTEDg= -k8s.io/client-go v0.17.0/go.mod h1:TYgR6EUHs6k45hb6KWjVD6jFZvJV4gHDikv/It0xz+k= -k8s.io/code-generator v0.0.0-20190912054826-cd179ad6a269/go.mod h1:V5BD6M4CyaN5m+VthcclXWsVcT1Hu+glwa1bi3MIsyE= -k8s.io/component-base v0.0.0-20190918160511-547f6c5d7090/go.mod h1:933PBGtQFJky3TEwYx4aEPZ4IxqhWh3R6DCmzqIn1hA= -k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= -k8s.io/gengo v0.0.0-20190822140433-26a664648505/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= -k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= -k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= -k8s.io/klog v0.4.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= -k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8= -k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= -k8s.io/kube-openapi v0.0.0-20190816220812-743ec37842bf/go.mod h1:1TqjTSzOxsLGIKfj0lK8EeCP7K1iUG65v09OM0/WG5E= -k8s.io/kube-openapi v0.0.0-20191107075043-30be4d16710a h1:UcxjrRMyNx/i/y8G7kPvLyy7rfbeuf1PYyBf973pgyU= -k8s.io/kube-openapi v0.0.0-20191107075043-30be4d16710a/go.mod h1:1TqjTSzOxsLGIKfj0lK8EeCP7K1iUG65v09OM0/WG5E= -k8s.io/utils v0.0.0-20190801114015-581e00157fb1/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= -k8s.io/utils v0.0.0-20191114184206-e782cd3c129f h1:GiPwtSzdP43eI1hpPCbROQCCIgCuiMMNF8YUVLF3vJo= -k8s.io/utils v0.0.0-20191114184206-e782cd3c129f/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= -modernc.org/cc v1.0.0/go.mod h1:1Sk4//wdnYJiUIxnW8ddKpaOJCF37yAdqYnkxUpaYxw= -modernc.org/golex v1.0.0/go.mod h1:b/QX9oBD/LhixY6NDh+IdGv17hgB+51fET1i2kPSmvk= -modernc.org/mathutil v1.0.0/go.mod h1:wU0vUrJsVWBZ4P6e7xtFJEhFSNsfRLJ8H458uRjg03k= -modernc.org/strutil v1.0.0/go.mod h1:lstksw84oURvj9y3tn8lGvRxyRC1S2+g5uuIzNfIOBs= -modernc.org/xc v1.0.0/go.mod h1:mRNCo0bvLjGhHO9WsyuKVU4q0ceiDDDoEeWDJHrNx8I= -sigs.k8s.io/controller-runtime v0.4.0 h1:wATM6/m+3w8lj8FXNaO6Fs/rq/vqoOjO1Q116Z9NPsg= -sigs.k8s.io/controller-runtime v0.4.0/go.mod h1:ApC79lpY3PHW9xj/w9pj+lYkLgwAAUZwfXkME1Lajns= -sigs.k8s.io/structured-merge-diff v0.0.0-20190525122527-15d366b2352e/go.mod h1:wWxsB5ozmmv/SG7nM11ayaAW51xMvak/t1r0CSlcokI= -sigs.k8s.io/structured-merge-diff v0.0.0-20190817042607-6149e4549fca/go.mod h1:IIgPezJWb76P0hotTxzDbWsMYB8APh18qZnxkomBpxA= -sigs.k8s.io/testing_frameworks v0.1.2 h1:vK0+tvjF0BZ/RYFeZ1E6BYBwHJJXhjuZ3TdsEKH+UQM= -sigs.k8s.io/testing_frameworks v0.1.2/go.mod h1:ToQrwSC3s8Xf/lADdZp3Mktcql9CG0UAmdJG9th5i0w= -sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs= -sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= diff --git a/cmd/resource/main.go b/cmd/resource/main.go deleted file mode 100644 index 4df301e04..000000000 --- a/cmd/resource/main.go +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright 2019 The Kubernetes Authors. -// SPDX-License-Identifier: Apache-2.0 - -package main - -import ( - "os" - - "github.com/spf13/cobra" - - "sigs.k8s.io/kustomize/cmd/resource/status" - - // This is here rather than in the libraries because of - // https://github.com/kubernetes-sigs/kustomize/issues/2060 - _ "k8s.io/client-go/plugin/pkg/client/auth" -) - -var root = &cobra.Command{ - Use: "resource", - Short: "resource reference command", -} - -func main() { - root.AddCommand(status.StatusCommand()) - - if err := root.Execute(); err != nil { - os.Exit(1) - } -} diff --git a/cmd/resource/status/status.go b/cmd/resource/status/status.go deleted file mode 100644 index ff729a2a9..000000000 --- a/cmd/resource/status/status.go +++ /dev/null @@ -1,24 +0,0 @@ -// Copyright 2019 The Kubernetes Authors. -// SPDX-License-Identifier: Apache-2.0 - -//go:generate $GOBIN/mdtogo docs/commands generateddocs/commands --license=none - -package status - -import ( - "github.com/spf13/cobra" - "sigs.k8s.io/kustomize/cmd/resource/status/cmd" -) - -func StatusCommand() *cobra.Command { - var status = &cobra.Command{ - Use: "status", - Short: "[Alpha] Commands for working with resource status.", - } - - status.AddCommand(cmd.FetchCommand()) - status.AddCommand(cmd.WaitCommand()) - status.AddCommand(cmd.EventsCommand()) - - return status -} diff --git a/examples/alphaTestExamples/MultipleDeployments.md b/examples/alphaTestExamples/MultipleDeployments.md index 47f7593ab..75495f48e 100644 --- a/examples/alphaTestExamples/MultipleDeployments.md +++ b/examples/alphaTestExamples/MultipleDeployments.md @@ -31,6 +31,8 @@ Alternatively, use ``` BASE=$DEMO_HOME/base mkdir -p $BASE +OUTPUT=$DEMO_HOME/output +mkdir -p $OUTPUT mkdir $BASE/wordpress mkdir $BASE/mysql @@ -59,16 +61,11 @@ metadata: EOF ``` - -``` -MYGOBIN=$GOPATH/bin -``` - Delete any existing kind cluster and create a new one. By default the name of the cluster is "kind" ``` -$MYGOBIN/kind delete cluster; -$MYGOBIN/kind create cluster; +kind delete cluster +kind create cluster ``` Let's run the wordpress and mysql services. @@ -76,34 +73,34 @@ Let's run the wordpress and mysql services. ``` export KUSTOMIZE_ENABLE_ALPHA_COMMANDS=true -$MYGOBIN/kustomize resources apply $BASE/mysql --status; +kustomize resources apply $BASE/mysql --status; status=$(mktemp); -$MYGOBIN/resource status fetch $BASE/mysql > $status +kustomize status fetch $BASE/mysql > $OUTPUT/status test 1 == \ - $(grep "mysql" $status | grep "Deployment is available. Replicas: 1" | wc -l); \ + $(grep "mysql" $OUTPUT/status | grep "Deployment is available. Replicas: 1" | wc -l); \ echo $? test 1 == \ - $(grep "mysql-pass" $status | grep "Resource is always ready" | wc -l); \ + $(grep "mysql-pass" $OUTPUT/status | grep "Resource is always ready" | wc -l); \ echo $? test 1 == \ - $(grep "mysql" $status | grep "Service is ready" | wc -l); \ + $(grep "mysql" $OUTPUT/status | grep "Service is ready" | wc -l); \ echo $? -$MYGOBIN/kustomize resources apply $BASE/wordpress --status; +kustomize resources apply $BASE/wordpress --status; status=$(mktemp); -$MYGOBIN/resource status fetch $BASE/wordpress > $status +kustomize status fetch $BASE/wordpress > $OUTPUT/status test 1 == \ - $(grep "wordpress" $status | grep "Deployment is available. Replicas: 1" | wc -l); \ + $(grep "wordpress" $OUTPUT/status | grep "Deployment is available. Replicas: 1" | wc -l); \ echo $? test 1 == \ - $(grep "wordpress" $status | grep "Service is ready" | wc -l); \ + $(grep "wordpress" $OUTPUT/status | grep "Service is ready" | wc -l); \ echo $? ``` @@ -130,26 +127,26 @@ sed -i.bak 's/secret/secret2/' \ sed -i.bak 's/mysql-pass/mysql-pass2/' \ $BASE/mysql/deployment.yaml -$MYGOBIN/kustomize resources apply $BASE/mysql --status; +kustomize resources apply $BASE/mysql --status; status=$(mktemp); -$MYGOBIN/resource status fetch $BASE/mysql > $status +kustomize status fetch $BASE/mysql > $OUTPUT/status test 1 == \ - $(grep "mysql" $status | grep "Deployment is available. Replicas: 1" | wc -l); \ + $(grep "mysql" $OUTPUT/status | grep "Deployment is available. Replicas: 1" | wc -l); \ echo $? test 1 == \ - $(grep "mysql-pass2" $status | grep "Resource is always ready" | wc -l); \ + $(grep "mysql-pass2" $OUTPUT/status | grep "Resource is always ready" | wc -l); \ echo $? test 1 == \ - $(grep "mysql" $status | grep "Service is ready" | wc -l); \ + $(grep "mysql" $OUTPUT/status | grep "Service is ready" | wc -l); \ echo $? ``` Clean-up the cluster ``` -$MYGOBIN/kind delete cluster; -``` \ No newline at end of file +kind delete cluster +``` diff --git a/examples/alphaTestExamples/helloapp.md b/examples/alphaTestExamples/helloapp.md index fbe681178..e20f1e7e7 100644 --- a/examples/alphaTestExamples/helloapp.md +++ b/examples/alphaTestExamples/helloapp.md @@ -33,6 +33,8 @@ Let's run the [hello] service. ``` BASE=$DEMO_HOME/base mkdir -p $BASE +OUTPUT=$DEMO_HOME/output +mkdir -p $OUTPUT ``` Now lets add a simple config map resource to the `base` @@ -158,18 +160,13 @@ sed -i.bak 's/app: hello/app: my-hello/' \ $BASE/kustomization.yaml ``` -To do end to end tests using kustomize on local, go through the following section. You should have GOPATH set up and [kind] installed. - - -``` -MYGOBIN=$GOPATH/bin -``` +The following requires installation of [kind]. Delete any existing kind cluster and create a new one. By default the name of the cluster is "kind" ``` -$MYGOBIN/kind delete cluster; -$MYGOBIN/kind create cluster; +kind delete cluster +kind create cluster ``` Use the kustomize binary in MYGOBIN to apply a deployment, fetch the status and verify the status. @@ -177,21 +174,20 @@ Use the kustomize binary in MYGOBIN to apply a deployment, fetch the status and ``` export KUSTOMIZE_ENABLE_ALPHA_COMMANDS=true -$MYGOBIN/kustomize resources apply $BASE --status; +kustomize resources apply $BASE --status; -status=$(mktemp); -$MYGOBIN/resource status fetch $BASE > $status +kustomize status fetch $BASE > $OUTPUT/status test 1 == \ - $(grep "the-deployment" $status | grep "Deployment is available. Replicas: 3" | wc -l); \ + $(grep "the-deployment" $OUTPUT/status | grep "Deployment is available. Replicas: 3" | wc -l); \ echo $? test 1 == \ - $(grep "the-map" $status | grep "Resource is always ready" | wc -l); \ + $(grep "the-map" $OUTPUT/status | grep "Resource is always ready" | wc -l); \ echo $? test 1 == \ - $(grep "the-service" $status | grep "Service is ready" | wc -l); \ + $(grep "the-service" $OUTPUT/status | grep "Service is ready" | wc -l); \ echo $? ``` @@ -216,30 +212,29 @@ sed -i.bak 's/configMap/configMap2/' \ sed -i.bak 's/the-map/the-map2/' \ $BASE/deployment.yaml -$MYGOBIN/kustomize resources apply $BASE --status; +kustomize resources apply $BASE --status; status=$(mktemp); -$MYGOBIN/resource status fetch $BASE > $status -$MYGOBIN/resource status fetch $BASE > /tmp/teste2eout/out.txt +kustomize status fetch $BASE > $OUTPUT/status test 1 == \ - $(grep "the-deployment" $status | grep "Deployment is available. Replicas: 3" | wc -l); \ + $(grep "the-deployment" $OUTPUT/status | grep "Deployment is available. Replicas: 3" | wc -l); \ echo $? test 1 == \ - $(grep "the-map2" $status | grep "Resource is always ready" | wc -l); \ + $(grep "the-map2" $OUTPUT/status | grep "Resource is always ready" | wc -l); \ echo $? test 1 == \ - $(grep "the-service" $status | grep "Service is ready" | wc -l); \ + $(grep "the-service" $OUTPUT/status | grep "Service is ready" | wc -l); \ echo $? ``` Clean-up the cluster ``` -$MYGOBIN/kind delete cluster; +kind delete cluster ``` ### Next Exercise -Create overlays as described in the [helloWorld] section and verify the results. Good luck! \ No newline at end of file +Create overlays as described in the [helloWorld] section and verify the results. diff --git a/kustomize/go.mod b/kustomize/go.mod index f1aed0041..1912aec9e 100644 --- a/kustomize/go.mod +++ b/kustomize/go.mod @@ -3,13 +3,19 @@ module sigs.k8s.io/kustomize/kustomize/v3 go 1.13 require ( + github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d github.com/pkg/errors v0.8.1 github.com/spf13/cobra v0.0.5 github.com/spf13/pflag v1.0.5 + github.com/stretchr/testify v1.4.0 + k8s.io/api v0.17.0 + k8s.io/apimachinery v0.17.0 k8s.io/client-go v0.17.0 + sigs.k8s.io/controller-runtime v0.4.0 sigs.k8s.io/kustomize/api v0.3.2 sigs.k8s.io/kustomize/cmd/config v0.0.5 sigs.k8s.io/kustomize/cmd/kubectl v0.0.3 + sigs.k8s.io/kustomize/kstatus v0.0.1 sigs.k8s.io/kustomize/kyaml v0.0.6 sigs.k8s.io/yaml v1.1.0 ) diff --git a/kustomize/go.sum b/kustomize/go.sum index 0d907bbdd..2c7916b9f 100644 --- a/kustomize/go.sum +++ b/kustomize/go.sum @@ -34,12 +34,15 @@ github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdko github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= +github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpHMqeKTCYkitsPqHNxTmd4SNR5r94FGM8= +github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/bombsimon/wsl v1.2.5 h1:9gTOkIwVtoDZywvX802SDHokeX4kW1cKnV8ZTVAPkRs= @@ -106,7 +109,9 @@ github.com/go-lintpack/lintpack v0.5.2 h1:DI5mA3+eKdWeJ40nU4d6Wc26qmdG8RCi/btYq0 github.com/go-lintpack/lintpack v0.5.2/go.mod h1:NwZuYi2nUHho8XEIZ6SIxihrnPoqBTDqfpXvXAN0sXM= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-logr/logr v0.1.0 h1:M1Tv3VzNlEHg6uyACnRdtrploV2P7wZqH8BoQMtz0cg= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= +github.com/go-logr/zapr v0.1.0 h1:h+WVe9j6HAA01niTJPA/kKH0i7e0rLZBCwauQFcRE54= github.com/go-logr/zapr v0.1.0/go.mod h1:tabnROwaDl0UNxkVeFRbY8bwB37GwRv0P8lg6aAiEnk= github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8= github.com/go-openapi/analysis v0.0.0-20180825180245-b006789cd277/go.mod h1:k70tL6pCuVxPJOHXQ+wIac1FUrvNkHolPie/cLEU6hI= @@ -338,6 +343,7 @@ github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hd github.com/mattn/go-runewidth v0.0.7 h1:Ei8KR0497xHyKJPAv59M1dkC+rOZCMBJ+t3fZ+twI54= github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw= +github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= @@ -393,16 +399,20 @@ github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prY github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM= github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= +github.com/prometheus/client_golang v1.0.0 h1:vrDKnkGzuGvhNAL56c7DBz29ZL+KxnoR0x7enabFceM= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90 h1:S/YWwWx/RA8rT8tKFRuGUZhuA90OyIBpPCXkcbwU8DE= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.4.1 h1:K0MGApIoQvMw27RTdJkPbr3JZ7DNbtxQNyi5STVM6Kw= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.0.2 h1:6LJUbpNm42llc4HRCuvApCSWB/WfhuNo9K98Q9sNGfs= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/quasilyte/go-consistent v0.0.0-20190521200055-c6f3937de18c/go.mod h1:5STLWrekHfjyYwxBRVRXNOSewLJ3PWfDJd1VyTS21fI= @@ -480,11 +490,14 @@ go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.uber.org/atomic v0.0.0-20181018215023-8dc6146f7569/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/multierr v0.0.0-20180122172545-ddea229ff1df/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/zap v0.0.0-20180814183419-67bc79d13d15/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.10.0 h1:ORx85nbTijNz8ljznvCMR1ZBIPKFn3jQrag10X2AsuM= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -598,6 +611,7 @@ golang.org/x/tools v0.0.0-20191010075000-0337d82405ff h1:XdBG6es/oFDr1HwaxkxgVve golang.org/x/tools v0.0.0-20191010075000-0337d82405ff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7 h1:9zdDQZ7Thm29KFXgAX/+yaf3eVbP7djjWp/dXAppNCc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gomodules.xyz/jsonpatch/v2 v2.0.1 h1:xyiBuvkD2g5n7cYzx6u2sxQvsAy4QJsZFCzGVdzOXZ0= gomodules.xyz/jsonpatch/v2 v2.0.1/go.mod h1:IhYNNY4jnS53ZnfE4PAmpKtDpTCj1JFXc+3mwe7XcUU= gonum.org/v1/gonum v0.0.0-20190331200053-3d26580ed485/go.mod h1:2ltnJ7xHfj0zHS40VVPYEAAMTa3ZGguvHGBSJeRWqE0= gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= @@ -649,6 +663,7 @@ k8s.io/api v0.0.0-20190918155943-95b840bb6a1f/go.mod h1:uWuOHnjmNrtQomJrvEBg0c0H k8s.io/api v0.0.0-20191214185829-ca1d04f8b0d3/go.mod h1:itOjKREfmUTvcjantxOsyYU5mbFsU7qUnyUuRfF5+5M= k8s.io/api v0.17.0 h1:H9d/lw+VkZKEVIUc8F3wgiQ+FUXTTr21M87jXLU7yqM= k8s.io/api v0.17.0/go.mod h1:npsyOePkeP0CPwyGfXDHxvypiYMJxBWAMpQxCaJ4ZxI= +k8s.io/apiextensions-apiserver v0.0.0-20190918161926-8f644eb6e783 h1:V6ndwCPoao1yZ52agqOKaUAl7DYWVGiXjV7ePA2i610= k8s.io/apiextensions-apiserver v0.0.0-20190918161926-8f644eb6e783/go.mod h1:xvae1SZB3E17UpV59AWc271W/Ph25N+bjPyR63X6tPY= k8s.io/apimachinery v0.0.0-20190913080033-27d36303b655/go.mod h1:nL6pwRT8NgfF8TT68DBI8uEePRt89cSvoXUVqbkWHq4= k8s.io/apimachinery v0.0.0-20191214185652-442f8fb2f03a/go.mod h1:Ng1IY8TS7sC44KJxT/WUR6qFRfWwahYYYpNXyYRKOCY= @@ -701,18 +716,15 @@ sigs.k8s.io/controller-runtime v0.4.0 h1:wATM6/m+3w8lj8FXNaO6Fs/rq/vqoOjO1Q116Z9 sigs.k8s.io/controller-runtime v0.4.0/go.mod h1:ApC79lpY3PHW9xj/w9pj+lYkLgwAAUZwfXkME1Lajns= sigs.k8s.io/kustomize v2.0.3+incompatible h1:JUufWFNlI44MdtnjUqVnvh29rR37PQFzPbLXqhyOyX0= sigs.k8s.io/kustomize v2.0.3+incompatible/go.mod h1:MkjgH3RdOWrievjo6c9T245dYlB5QeXV4WCbnt/PEpU= -sigs.k8s.io/kustomize/api v0.3.2 h1:64gvYVAvqe2fNfcTevtXh/GmLwVwHIcJ2Z5HBMfjncs= -sigs.k8s.io/kustomize/api v0.3.2/go.mod h1:A+ATnlHqzictQfQC1q3KB/T6MSr0UWQsrrLxMWkge2E= sigs.k8s.io/kustomize/cmd/config v0.0.5 h1:mFJowsk9IGvwm5dUpVB+ZM63on2JjgaCy+YcVsFaVxU= sigs.k8s.io/kustomize/cmd/config v0.0.5/go.mod h1:L47nDnZDfGFQG3gnPJLG2UABn0nVb9v+ndceyMH0jjU= -sigs.k8s.io/kustomize/cmd/kubectl v0.0.3 h1:cXn6GqRnOQtp4EC1+NiJKdUHE/aQ+5HhtAB28R4sVXA= -sigs.k8s.io/kustomize/cmd/kubectl v0.0.3/go.mod h1:JnS9HnTjUUMOE44WNboy/wi89J/K/XbAoU7O/iPXqqE= sigs.k8s.io/kustomize/kyaml v0.0.2/go.mod h1:rywm/rcR5LmCBghz9956tE45OdUPChFoXVVs+WmhMTI= sigs.k8s.io/kustomize/kyaml v0.0.5/go.mod h1:waxTrzQRK9i6/5fR5HNo8xa4YwvWn8t85vMnOGFEZik= sigs.k8s.io/kustomize/kyaml v0.0.6 h1:KhQr7JwpCseFTSWCwqp4CJ4mY6Kx+i34tF4e0eNkcXw= sigs.k8s.io/kustomize/kyaml v0.0.6/go.mod h1:tDOfJjL6slQVBLHJ76XfXAFgAOEdfm04AW2HehYOp8k= sigs.k8s.io/structured-merge-diff v0.0.0-20190525122527-15d366b2352e/go.mod h1:wWxsB5ozmmv/SG7nM11ayaAW51xMvak/t1r0CSlcokI= sigs.k8s.io/structured-merge-diff v0.0.0-20190817042607-6149e4549fca/go.mod h1:IIgPezJWb76P0hotTxzDbWsMYB8APh18qZnxkomBpxA= +sigs.k8s.io/testing_frameworks v0.1.2 h1:vK0+tvjF0BZ/RYFeZ1E6BYBwHJJXhjuZ3TdsEKH+UQM= sigs.k8s.io/testing_frameworks v0.1.2/go.mod h1:ToQrwSC3s8Xf/lADdZp3Mktcql9CG0UAmdJG9th5i0w= sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= diff --git a/kustomize/internal/commands/commands.go b/kustomize/internal/commands/commands.go index fcd73b6ba..ffe3bb5c7 100644 --- a/kustomize/internal/commands/commands.go +++ b/kustomize/internal/commands/commands.go @@ -19,6 +19,7 @@ import ( "sigs.k8s.io/kustomize/kustomize/v3/internal/commands/config" "sigs.k8s.io/kustomize/kustomize/v3/internal/commands/create" "sigs.k8s.io/kustomize/kustomize/v3/internal/commands/edit" + "sigs.k8s.io/kustomize/kustomize/v3/internal/commands/status" "sigs.k8s.io/kustomize/kustomize/v3/internal/commands/version" ) @@ -43,6 +44,7 @@ See https://sigs.k8s.io/kustomize edit.NewCmdEdit(fSys, v, uf), create.NewCmdCreate(fSys, uf), version.NewCmdVersion(stdOut), + status.NewCmdStatus(), ) if cc := config.NewCmdConfig(fSys); cc != nil { c.AddCommand(cc) diff --git a/kustomize/internal/commands/status/all.go b/kustomize/internal/commands/status/all.go new file mode 100644 index 000000000..7a6d59227 --- /dev/null +++ b/kustomize/internal/commands/status/all.go @@ -0,0 +1,31 @@ +// Copyright 2019 The Kubernetes Authors. +// SPDX-License-Identifier: Apache-2.0 + +//go:generate $GOBIN/mdtogo docs/commands generateddocs/commands --license=none + +package status + +import ( + "github.com/spf13/cobra" + "sigs.k8s.io/kustomize/kustomize/v3/internal/commands/status/cmd" + "sigs.k8s.io/kustomize/kyaml/commandutil" +) + +func NewCmdStatus() *cobra.Command { + var c = &cobra.Command{ + Use: "status", + Short: "[Alpha] Commands for working with resource status.", + Hidden: commandutil.GetAlphaEnabled(), + } + + if !commandutil.GetAlphaEnabled() { + c.Short = "[Alpha] To enable set KUSTOMIZE_ENABLE_ALPHA_COMMANDS=true" + c.Long = "[Alpha] To enable set KUSTOMIZE_ENABLE_ALPHA_COMMANDS=true" + return c + } + + c.AddCommand(cmd.FetchCommand()) + c.AddCommand(cmd.WaitCommand()) + c.AddCommand(cmd.EventsCommand()) + return c +} diff --git a/cmd/resource/status/cmd/events.go b/kustomize/internal/commands/status/cmd/events.go similarity index 96% rename from cmd/resource/status/cmd/events.go rename to kustomize/internal/commands/status/cmd/events.go index 3e981a2d2..f89a3fde0 100644 --- a/cmd/resource/status/cmd/events.go +++ b/kustomize/internal/commands/status/cmd/events.go @@ -9,7 +9,7 @@ import ( "github.com/pkg/errors" "github.com/spf13/cobra" - "sigs.k8s.io/kustomize/cmd/resource/status/generateddocs/commands" + "sigs.k8s.io/kustomize/kustomize/v3/internal/commands/status/generateddocs/commands" "sigs.k8s.io/kustomize/kyaml/kio" ) diff --git a/cmd/resource/status/cmd/events_test.go b/kustomize/internal/commands/status/cmd/events_test.go similarity index 100% rename from cmd/resource/status/cmd/events_test.go rename to kustomize/internal/commands/status/cmd/events_test.go diff --git a/cmd/resource/status/cmd/fetch.go b/kustomize/internal/commands/status/cmd/fetch.go similarity index 97% rename from cmd/resource/status/cmd/fetch.go rename to kustomize/internal/commands/status/cmd/fetch.go index e164fc8b4..5680a7b69 100644 --- a/cmd/resource/status/cmd/fetch.go +++ b/kustomize/internal/commands/status/cmd/fetch.go @@ -9,9 +9,9 @@ import ( "github.com/pkg/errors" "github.com/spf13/cobra" - "sigs.k8s.io/kustomize/cmd/resource/status/generateddocs/commands" "sigs.k8s.io/kustomize/kstatus/status" "sigs.k8s.io/kustomize/kstatus/wait" + "sigs.k8s.io/kustomize/kustomize/v3/internal/commands/status/generateddocs/commands" "sigs.k8s.io/kustomize/kyaml/kio" ) diff --git a/cmd/resource/status/cmd/fetch_test.go b/kustomize/internal/commands/status/cmd/fetch_test.go similarity index 100% rename from cmd/resource/status/cmd/fetch_test.go rename to kustomize/internal/commands/status/cmd/fetch_test.go diff --git a/cmd/resource/status/cmd/helpers_test.go b/kustomize/internal/commands/status/cmd/helpers_test.go similarity index 100% rename from cmd/resource/status/cmd/helpers_test.go rename to kustomize/internal/commands/status/cmd/helpers_test.go diff --git a/cmd/resource/status/cmd/print.go b/kustomize/internal/commands/status/cmd/print.go similarity index 100% rename from cmd/resource/status/cmd/print.go rename to kustomize/internal/commands/status/cmd/print.go diff --git a/cmd/resource/status/cmd/util.go b/kustomize/internal/commands/status/cmd/util.go similarity index 100% rename from cmd/resource/status/cmd/util.go rename to kustomize/internal/commands/status/cmd/util.go diff --git a/cmd/resource/status/cmd/util_test.go b/kustomize/internal/commands/status/cmd/util_test.go similarity index 54% rename from cmd/resource/status/cmd/util_test.go rename to kustomize/internal/commands/status/cmd/util_test.go index da081aa14..796b4b1a9 100644 --- a/cmd/resource/status/cmd/util_test.go +++ b/kustomize/internal/commands/status/cmd/util_test.go @@ -1,34 +1,35 @@ package cmd import ( + "testing" + "github.com/stretchr/testify/assert" "sigs.k8s.io/kustomize/kyaml/yaml" - "testing" ) func TestIsValidKubernetesResource(t *testing.T) { testCases := map[string]struct { - data yaml.ResourceIdentifier - expected bool + data yaml.ResourceIdentifier + expected bool }{ "invalid resource": { - data: yaml.ResourceIdentifier { - Name : "", - APIVersion : "", - Kind : "", - Namespace : "", + data: yaml.ResourceIdentifier{ + Name: "", + APIVersion: "", + Kind: "", + Namespace: "", }, - expected: false, + expected: false, }, "valid resource": { - data: yaml.ResourceIdentifier { - Name : "SomeName", - APIVersion : "SomeVersion", - Kind : "SomeKind", - Namespace : "", + data: yaml.ResourceIdentifier{ + Name: "SomeName", + APIVersion: "SomeVersion", + Kind: "SomeKind", + Namespace: "", }, - expected: true, + expected: true, }, } diff --git a/cmd/resource/status/cmd/wait.go b/kustomize/internal/commands/status/cmd/wait.go similarity index 98% rename from cmd/resource/status/cmd/wait.go rename to kustomize/internal/commands/status/cmd/wait.go index ddd0e5425..60a319696 100644 --- a/cmd/resource/status/cmd/wait.go +++ b/kustomize/internal/commands/status/cmd/wait.go @@ -10,9 +10,9 @@ import ( "github.com/pkg/errors" "github.com/spf13/cobra" - "sigs.k8s.io/kustomize/cmd/resource/status/generateddocs/commands" "sigs.k8s.io/kustomize/kstatus/status" "sigs.k8s.io/kustomize/kstatus/wait" + "sigs.k8s.io/kustomize/kustomize/v3/internal/commands/status/generateddocs/commands" "sigs.k8s.io/kustomize/kyaml/kio" ) diff --git a/cmd/resource/status/cmd/wait_test.go b/kustomize/internal/commands/status/cmd/wait_test.go similarity index 100% rename from cmd/resource/status/cmd/wait_test.go rename to kustomize/internal/commands/status/cmd/wait_test.go diff --git a/cmd/resource/status/docs/commands/events.md b/kustomize/internal/commands/status/docs/commands/events.md similarity index 100% rename from cmd/resource/status/docs/commands/events.md rename to kustomize/internal/commands/status/docs/commands/events.md diff --git a/cmd/resource/status/docs/commands/fetch.md b/kustomize/internal/commands/status/docs/commands/fetch.md similarity index 100% rename from cmd/resource/status/docs/commands/fetch.md rename to kustomize/internal/commands/status/docs/commands/fetch.md diff --git a/cmd/resource/status/docs/commands/wait.md b/kustomize/internal/commands/status/docs/commands/wait.md similarity index 100% rename from cmd/resource/status/docs/commands/wait.md rename to kustomize/internal/commands/status/docs/commands/wait.md diff --git a/cmd/resource/status/generateddocs/commands/docs.go b/kustomize/internal/commands/status/generateddocs/commands/docs.go similarity index 100% rename from cmd/resource/status/generateddocs/commands/docs.go rename to kustomize/internal/commands/status/generateddocs/commands/docs.go From 1a7e2561ff704a137528890ae33b432f8832eeaa Mon Sep 17 00:00:00 2001 From: Sean Sullivan Date: Tue, 21 Jan 2020 17:14:08 -0800 Subject: [PATCH 15/46] Fixed inventory Equal(), checking nil as passed parameter --- cmd/kubectl/kubectlcobra/inventory.go | 3 +++ cmd/kubectl/kubectlcobra/inventory_test.go | 12 ++++++++++++ 2 files changed, 15 insertions(+) diff --git a/cmd/kubectl/kubectlcobra/inventory.go b/cmd/kubectl/kubectlcobra/inventory.go index 2c2913578..9a50cb076 100644 --- a/cmd/kubectl/kubectlcobra/inventory.go +++ b/cmd/kubectl/kubectlcobra/inventory.go @@ -71,6 +71,9 @@ func parseInventory(inv string) (*Inventory, error) { // Equals returns true if the Inventory structs are identical; // false otherwise. func (i *Inventory) Equals(other *Inventory) bool { + if other == nil { + return false + } return i.String() == other.String() } diff --git a/cmd/kubectl/kubectlcobra/inventory_test.go b/cmd/kubectl/kubectlcobra/inventory_test.go index f5b5006da..d4725d680 100644 --- a/cmd/kubectl/kubectlcobra/inventory_test.go +++ b/cmd/kubectl/kubectlcobra/inventory_test.go @@ -80,6 +80,18 @@ func TestInventoryEqual(t *testing.T) { inventory2 *Inventory isEqual bool }{ + // "Other" inventory is nil, then not equal. + { + inventory1: &Inventory{ + Name: "test-inv", + GroupKind: schema.GroupKind{ + Group: "apps", + Kind: "Deployment", + }, + }, + inventory2: nil, + isEqual: false, + }, // Two equal inventories without a namespace { inventory1: &Inventory{ From e2f4339ec628751a88d548ad724c7e0e6fbd8e0b Mon Sep 17 00:00:00 2001 From: Yujun Zhang Date: Wed, 22 Jan 2020 13:30:37 +0800 Subject: [PATCH 16/46] Add document about how `kustomize build` works --- docs/howItWorks.md | 51 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 docs/howItWorks.md diff --git a/docs/howItWorks.md b/docs/howItWorks.md new file mode 100644 index 000000000..090ade175 --- /dev/null +++ b/docs/howItWorks.md @@ -0,0 +1,51 @@ +# How does `kustomize build` work + +Call stack when running `kustomize build`, with links to code. + +## Run build + +* [RunBuild](https://github.com/kubernetes-sigs/kustomize/blob/c7d78970fb86782dbdded3a93944b774f826071f/kustomize/internal/commands/build/build.go#L121) + * [MakeKustomizer](https://github.com/kubernetes-sigs/kustomize/blob/c7d78970fb86782dbdded3a93944b774f826071f/api/krusty/kustomizer.go#L32) + * [Run](https://github.com/kubernetes-sigs/kustomize/blob/c7d78970fb86782dbdded3a93944b774f826071f/api/krusty/kustomizer.go#L47): performs a kustomization. It uses its internal filesystem reference to read the file at the given path argument, interpret it as a kustomization.yaml file, perform the kustomization it represents, and return the resulting resources. + * Create factories + * [tranformer.NewFactoryImpl](https://github.com/kubernetes-sigs/kustomize/blob/c7d78970fb86782dbdded3a93944b774f826071f/api/internal/k8sdeps/transformer/factory.go#L17) + * [resmap.NewFactory](https://github.com/kubernetes-sigs/kustomize/blob/c7d78970fb86782dbdded3a93944b774f826071f/api/resmap/factory.go#L21) + * [resource.NewFactory](https://github.com/kubernetes-sigs/kustomize/blob/c7d78970fb86782dbdded3a93944b774f826071f/api/resource/factory.go#L23) + * [kustruct.NewKunstructuredFactoryImpl](https://github.com/kubernetes-sigs/kustomize/blob/c7d78970fb86782dbdded3a93944b774f826071f/api/k8sdeps/kunstruct/factory.go#L28) + * [loader.NewLoader](https://github.com/kubernetes-sigs/kustomize/blob/c7d78970fb86782dbdded3a93944b774f826071f/api/loader/loader.go#L19) + * [validator.NewKustValidator](https://github.com/kubernetes-sigs/kustomize/blob/c7d78970fb86782dbdded3a93944b774f826071f/api/k8sdeps/validator/validators.go#L23) + * [NewKustTarget](https://github.com/kubernetes-sigs/kustomize/blob/c7d78970fb86782dbdded3a93944b774f826071f/api/internal/target/kusttarget.go#L38) + * [Load](https://github.com/kubernetes-sigs/kustomize/blob/c7d78970fb86782dbdded3a93944b774f826071f/api/internal/target/kusttarget.go#L54) + * [MakeCustomizeResMap](https://github.com/kubernetes-sigs/kustomize/blob/c7d78970fb86782dbdded3a93944b774f826071f/api/internal/target/kusttarget.go#L109): details in next section + * [emitResources](https://github.com/kubernetes-sigs/kustomize/blob/c7d78970fb86782dbdded3a93944b774f826071f/kustomize/internal/commands/build/build.go#L143) + +## Make resource map + +* [makeCustomizeResMap](https://github.com/kubernetes-sigs/kustomize/blob/c7d78970fb86782dbdded3a93944b774f826071f/api/internal/target/kusttarget.go#L117) + * [AccumulateTarget](https://github.com/kubernetes-sigs/kustomize/blob/c7d78970fb86782dbdded3a93944b774f826071f/api/internal/target/kusttarget.go#L196): returns a new ResAccumulator, holding customized resources and the data/rules used to do so. The name back references and vars are not yet fixed. + * [accummulateResources](https://github.com/kubernetes-sigs/kustomize/blob/c7d78970fb86782dbdded3a93944b774f826071f/api/internal/target/kusttarget.go#L302): fills the given resourceAccumulator with resources read from the given list of paths. + * Merge config from builtin and CRDs + * [runGenerators](https://github.com/kubernetes-sigs/kustomize/blob/c7d78970fb86782dbdded3a93944b774f826071f/api/internal/target/kusttarget.go#L239) + * [configureBuiltinGenerators](https://github.com/kubernetes-sigs/kustomize/blob/c7d78970fb86782dbdded3a93944b774f826071f/api/internal/target/kusttarget_configplugin.go#L28) + * ConfigMapGenerator + * SecretGenerator + * [configureExternalGenerators]() + * Iterate all generators + * [runTransfomers](https://github.com/kubernetes-sigs/kustomize/blob/c7d78970fb86782dbdded3a93944b774f826071f/api/internal/target/kusttarget.go#L274) + * [configureBuiltinTransformers](https://github.com/kubernetes-sigs/kustomize/blob/c7d78970fb86782dbdded3a93944b774f826071f/api/internal/target/kusttarget_configplugin.go#L44) + * PatchStrategicMergeTransformer + * PatchTransformer + * NamespaceTransformer + * PrefixSuffixTransformer + * LabelTransformer + * AnnotationsTransformer + * PatchJson6902Transformer + * ReplicaCountTransformer + * ImageTagTransformer + * [configureExternalTransformers](https://github.com/kubernetes-sigs/kustomize/blob/c7d78970fb86782dbdded3a93944b774f826071f/api/internal/target/kusttarget.go#L291) + * [MergeVars](https://github.com/kubernetes-sigs/kustomize/blob/c7d78970fb86782dbdded3a93944b774f826071f/api/internal/accumulator/resaccumulator.go#L64) + * The following steps must be done last, not as part of the recursion implicit in AccumulateTarget. + * [addHashesToNames](https://github.com/kubernetes-sigs/kustomize/blob/c7d78970fb86782dbdded3a93944b774f826071f/api/internal/target/kusttarget.go#L153) + * [FixBackReferences](https://github.com/kubernetes-sigs/kustomize/blob/c7d78970fb86782dbdded3a93944b774f826071f/api/internal/accumulator/resaccumulator.go#L160): Given that names have changed (prefixs/suffixes added), fix all the back references to those names. + * [ResolveVars](https://github.com/kubernetes-sigs/kustomize/blob/c7d78970fb86782dbdded3a93944b774f826071f/api/internal/accumulator/resaccumulator.go#L141) + * [computeInventory](https://github.com/kubernetes-sigs/kustomize/blob/c7d78970fb86782dbdded3a93944b774f826071f/api/internal/target/kusttarget.go#L163) From 9f9a1d4159a244c611a793ac77ce9149cb9fd9f6 Mon Sep 17 00:00:00 2001 From: beantaxi Date: Wed, 22 Jan 2020 12:04:49 -0600 Subject: [PATCH 17/46] Rename authoriing.md to authoring.md --- docs/{authoriing.md => authoring.md} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename docs/{authoriing.md => authoring.md} (99%) diff --git a/docs/authoriing.md b/docs/authoring.md similarity index 99% rename from docs/authoriing.md rename to docs/authoring.md index 40c3d6f11..68ea1ec5e 100644 --- a/docs/authoriing.md +++ b/docs/authoring.md @@ -34,4 +34,4 @@ With an existing kustomization file the `kustomize edit` command * add * set * remove -* fix \ No newline at end of file +* fix From 0820865e1d6e756ed452e1af12abe3881529b023 Mon Sep 17 00:00:00 2001 From: Haiyan Meng Date: Wed, 22 Jan 2020 10:13:57 -0800 Subject: [PATCH 18/46] Retry FindRangesForRepoSearch --- api/internal/crawl/crawler/github/crawler.go | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/api/internal/crawl/crawler/github/crawler.go b/api/internal/crawl/crawler/github/crawler.go index f1b723c5b..21cfd6f81 100644 --- a/api/internal/crawl/crawler/github/crawler.go +++ b/api/internal/crawl/crawler/github/crawler.go @@ -87,10 +87,20 @@ func (gc githubCrawler) Crawl(ctx context.Context, accessToken: gc.client.accessToken, } + var ranges []string + var err error // Since Github returns a max of 1000 results per query, we can use // multiple queries that split the search space into chunks of at most // 1000 files to get all of the data. - ranges, err := FindRangesForRepoSearch(newCache(noETagClient, gc.query)) + for i := 0; i < 5; i++ { + ranges, err = FindRangesForRepoSearch(newCache(noETagClient, gc.query)) + if err == nil { + logger.Printf("FindRangesForRepoSearch succeeded after %d retries", i) + break + } else { + time.Sleep(time.Minute) + } + } if err != nil { return fmt.Errorf("could not split %v into ranges, %v\n", gc.query, err) From 00f68c12a81690693442f3fda24c53b192595cff Mon Sep 17 00:00:00 2001 From: HowJMay Date: Thu, 23 Jan 2020 23:35:38 +0800 Subject: [PATCH 19/46] fix typos Fix typos --- api/internal/crawl/crawler/github/queries.go | 2 +- api/internal/crawl/crawler/github/split_search_ranges.go | 4 ++-- api/internal/crawl/index/kustomize.go | 2 +- api/internal/tools/tools.go | 2 +- cmd/config/docs/api-conventions/config-io.md | 2 +- cmd/config/docs/api-conventions/functions-impl.md | 2 +- cmd/config/internal/generateddocs/api/docs.go | 4 ++-- 7 files changed, 9 insertions(+), 9 deletions(-) diff --git a/api/internal/crawl/crawler/github/queries.go b/api/internal/crawl/crawler/github/queries.go index ba05af38f..444ee3d10 100644 --- a/api/internal/crawl/crawler/github/queries.go +++ b/api/internal/crawl/crawler/github/queries.go @@ -166,7 +166,7 @@ type request struct { query Query } -// CopyWith copies the requests and adds the extra query parameters. Usefull +// CopyWith copies the requests and adds the extra query parameters. It is useful // for dynamically adding sizes to a filename only query without modifying it. func (r request) CopyWith(queryParams ...queryField) request { cpy := r diff --git a/api/internal/crawl/crawler/github/split_search_ranges.go b/api/internal/crawl/crawler/github/split_search_ranges.go index 1322932f4..8d9037478 100644 --- a/api/internal/crawl/crawler/github/split_search_ranges.go +++ b/api/internal/crawl/crawler/github/split_search_ranges.go @@ -93,7 +93,7 @@ package github // apiCallsPerResult * 10(pages) * 100(resultsPerPage) * totalResults / 1000 // = apiCallsPerResult * totalResults. // -// So it could very well take apiCallsPerResult * 50 times longer to acutally +// So it could very well take apiCallsPerResult * 50 times longer to actually // fetch the results (assuming the quotas for the API calls are the same as the // search API), than it does to perform these range searches. @@ -139,7 +139,7 @@ type cachedSearch interface { // problematic). The current cache implementation looks at the // predecessor entry to find out if the current value is monotonic. // This is where the bit trick is used, since each step in the binary -// search is adding or ommiting to add a decreasing power of 2 to the query +// search is adding or omitting to add a decreasing power of 2 to the query // value, we can remove the least significant set bit to find the // predecessor in constant time. Ultimately since the search is rate // limited, we could also easily afford to compute this in linear time diff --git a/api/internal/crawl/index/kustomize.go b/api/internal/crawl/index/kustomize.go index 7e2e7b104..f7b20f13f 100644 --- a/api/internal/crawl/index/kustomize.go +++ b/api/internal/crawl/index/kustomize.go @@ -270,7 +270,7 @@ func (it *KustomizeIterator) Value() KustomizeResult { return it.scrollImpl } -// Check if any errors have occured. +// Check if any errors have occurred. func (it *KustomizeIterator) Err() error { return it.err } diff --git a/api/internal/tools/tools.go b/api/internal/tools/tools.go index ac7ae28e9..7be4ef948 100644 --- a/api/internal/tools/tools.go +++ b/api/internal/tools/tools.go @@ -4,7 +4,7 @@ // SPDX-License-Identifier: Apache-2.0 // This file exists to automatically trigger installs -// of the given tools, and is the offical 'unofficial' +// of the given tools, and is the official 'unofficial' // way to declare a dependence on a Go binary until // some better technique comes along. diff --git a/cmd/config/docs/api-conventions/config-io.md b/cmd/config/docs/api-conventions/config-io.md index 44337b4f4..35b013608 100644 --- a/cmd/config/docs/api-conventions/config-io.md +++ b/cmd/config/docs/api-conventions/config-io.md @@ -29,7 +29,7 @@ metadata: ### `config.kubernetes.io/index` Records the index of a Resource in file. In a multi-object YAML file, Resources are separated -by three dashes (`---`), and the index represents the positon of the Resource starting from zero. +by three dashes (`---`), and the index represents the position of the Resource starting from zero. This annotation **SHOULD** be set when reading Resources from files. It **SHOULD** be unset when writing Resources to files. diff --git a/cmd/config/docs/api-conventions/functions-impl.md b/cmd/config/docs/api-conventions/functions-impl.md index c56ab52a8..c0e8548c3 100644 --- a/cmd/config/docs/api-conventions/functions-impl.md +++ b/cmd/config/docs/api-conventions/functions-impl.md @@ -129,7 +129,7 @@ spec: ### Output -The function is invoked using by runing `kustomize config run dir/`. +The function is invoked using byrunning `kustomize config run dir/`. `dir/my-instance_deployment.yaml` contains the Deployment: diff --git a/cmd/config/internal/generateddocs/api/docs.go b/cmd/config/internal/generateddocs/api/docs.go index 54849a963..b7a3a512d 100644 --- a/cmd/config/internal/generateddocs/api/docs.go +++ b/cmd/config/internal/generateddocs/api/docs.go @@ -33,7 +33,7 @@ Example: ### ` + "`" + `config.kubernetes.io/index` + "`" + ` Records the index of a Resource in file. In a multi-object YAML file, Resources are separated -by three dashes (` + "`" + `---` + "`" + `), and the index represents the positon of the Resource starting from zero. +by three dashes (` + "`" + `---` + "`" + `), and the index represents the position of the Resource starting from zero. This annotation **SHOULD** be set when reading Resources from files. It **SHOULD** be unset when writing Resources to files. @@ -187,7 +187,7 @@ are passed to the Function through the ` + "`" + `ResourceList.functionConfig` + ### Output -The function is invoked using by runing ` + "`" + `kustomize config run dir/` + "`" + `. +The function is invoked using byrunning ` + "`" + `kustomize config run dir/` + "`" + `. ` + "`" + `dir/my-instance_deployment.yaml` + "`" + ` contains the Deployment: From 0f5256d9529d3eeba2f1282b2d3494fbc79c0b00 Mon Sep 17 00:00:00 2001 From: Morten Torkildsen Date: Wed, 22 Jan 2020 19:49:57 -0800 Subject: [PATCH 20/46] Clean up output format for status events command --- .../internal/commands/status/cmd/events.go | 6 +- .../commands/status/cmd/events_test.go | 92 +++++++++++-------- .../internal/commands/status/cmd/fetch.go | 6 +- .../commands/status/cmd/helpers_test.go | 4 +- .../internal/commands/status/cmd/print.go | 40 ++++---- .../internal/commands/status/cmd/util.go | 32 +++++-- .../internal/commands/status/cmd/wait.go | 6 +- 7 files changed, 113 insertions(+), 73 deletions(-) diff --git a/kustomize/internal/commands/status/cmd/events.go b/kustomize/internal/commands/status/cmd/events.go index f89a3fde0..0bf24d2dc 100644 --- a/kustomize/internal/commands/status/cmd/events.go +++ b/kustomize/internal/commands/status/cmd/events.go @@ -54,7 +54,7 @@ type EventsRunner struct { func (r *EventsRunner) runE(c *cobra.Command, args []string) error { ctx := context.Background() - resolver, err := r.newResolverFunc(r.Interval) + resolver, mapper, err := r.newResolverFunc(r.Interval) if err != nil { return errors.Wrap(err, "error creating resolver") } @@ -62,7 +62,9 @@ func (r *EventsRunner) runE(c *cobra.Command, args []string) error { // Set up a CaptureIdentifierFilter and run all inputs through the // filter with the pipeline to capture the inventory of resources // which we are interested in. - captureFilter := &CaptureIdentifiersFilter{} + captureFilter := &CaptureIdentifiersFilter{ + Mapper: mapper, + } filters := []kio.Filter{captureFilter} var inputs []kio.Reader diff --git a/kustomize/internal/commands/status/cmd/events_test.go b/kustomize/internal/commands/status/cmd/events_test.go index b570302df..8752966e1 100644 --- a/kustomize/internal/commands/status/cmd/events_test.go +++ b/kustomize/internal/commands/status/cmd/events_test.go @@ -200,7 +200,11 @@ func (e *EventOutput) allResources() []ResourceIdentifier { if !event.isResourceUpdateEvent() { continue } - r := event.identifier + r := ResourceIdentifier{ + namespace: event.namespace, + name: event.name, + kind: event.kind, + } if _, found := seenResources[r]; !found { resources = append(resources, r) seenResources[r] = true @@ -215,7 +219,12 @@ func (e *EventOutput) statusesForResource(resource ResourceIdentifier) []status. if !event.isResourceUpdateEvent() { continue } - if event.identifier.Equals(resource) { + identifier := ResourceIdentifier{ + namespace: event.namespace, + name: event.name, + kind: event.kind, + } + if identifier.Equals(resource) { statuses = append(statuses, event.status) } } @@ -223,32 +232,36 @@ func (e *EventOutput) statusesForResource(resource ResourceIdentifier) []status. } type EventOutputLine struct { - eventType string - aggStatus status.Status - identifier ResourceIdentifier - status status.Status - message string + eventType wait.EventType + aggStatus status.Status + kind string + namespace string + name string + status status.Status + message string } func (e *EventOutputLine) isResourceUpdateEvent() bool { - return e.eventType == string(wait.ResourceUpdate) + return e.eventType == wait.ResourceUpdate } var ( eventRegex = regexp.MustCompile(`^\s*` + - `(?P\S+)\s+` + - `(?P\S+)\s+` + - `((?P\S+)\s+` + + `(?P(?:Current|InProgress|Failed|Terminating|Unknown))\s+` + + `(?P.*\S)` + + `\s*$`) + resourceEventRegex = regexp.MustCompile(`^\s*` + `(?P\S+)\s+` + + `(?P(?:Current|InProgress|Failed|Terminating|Unknown))\s+` + + `(?P\S+)\s+` + `(?P\S+)\s+` + - `(?P\S+)\s+` + - `(?P.*\S)){0,1}` + + `(?P(?:Current|InProgress|Failed|Terminating|Unknown))\s+` + + `(?P.*\S)` + `\s*$`) eventHeaderRegex = regexp.MustCompile(`^\s*` + - `EVENT TYPE\s+` + + `NAMESPACE\s+` + `AGG STATUS\s+` + `TYPE\s+` + - `NAMESPACE\s+` + `NAME\s+` + `STATUS\s+` + `MESSAGE` + @@ -262,40 +275,43 @@ func parseEventOutput(_ *testing.T, output string) EventOutput { if len(line) == 0 { continue // Ignore empty lines } + match := eventHeaderRegex.FindStringSubmatch(line) if match != nil { continue // Ignore headers } + match = eventRegex.FindStringSubmatch(line) + if match != nil { + aggStatus := status.FromStringOrDie(match[1]) + var eventType wait.EventType + if aggStatus == status.CurrentStatus { + eventType = wait.Completed + } else { + eventType = wait.Aborted + } + eventOutput.events = append(eventOutput.events, EventOutputLine{ + eventType: eventType, + aggStatus: aggStatus, + message: match[2], + }) + } + + match = resourceEventRegex.FindStringSubmatch(line) if match == nil { eventOutput.unknownLines = append(eventOutput.unknownLines, line) continue } - eventOutputLine := EventOutputLine{ - eventType: match[1], + eventOutput.events = append(eventOutput.events, EventOutputLine{ + eventType: wait.ResourceUpdate, aggStatus: status.FromStringOrDie(match[2]), - } - - if eventOutputLine.eventType == string(wait.ResourceUpdate) { - resourceType := match[4] - parts := strings.Split(resourceType, "/") - var identifier ResourceIdentifier - if len(parts) == 2 { - identifier.apiVersion = parts[0] - identifier.kind = parts[1] - } else { - identifier.apiVersion = strings.Join(parts[:2], "/") - identifier.kind = parts[2] - } - identifier.namespace = match[5] - identifier.name = match[6] - eventOutputLine.identifier = identifier - eventOutputLine.status = status.FromStringOrDie(match[7]) - eventOutputLine.message = match[8] - } - - eventOutput.events = append(eventOutput.events, eventOutputLine) + kind: match[3], + namespace: match[1], + name: match[4], + status: status.FromStringOrDie(match[5]), + message: match[6], + }) } return eventOutput } diff --git a/kustomize/internal/commands/status/cmd/fetch.go b/kustomize/internal/commands/status/cmd/fetch.go index 5680a7b69..68546afef 100644 --- a/kustomize/internal/commands/status/cmd/fetch.go +++ b/kustomize/internal/commands/status/cmd/fetch.go @@ -50,7 +50,7 @@ type FetchRunner struct { func (r *FetchRunner) runE(c *cobra.Command, args []string) error { ctx := context.Background() - resolver, err := r.newResolverFunc(time.Minute) + resolver, mapper, err := r.newResolverFunc(time.Minute) if err != nil { return errors.Wrap(err, "error creating resolver") } @@ -58,7 +58,9 @@ func (r *FetchRunner) runE(c *cobra.Command, args []string) error { // Set up a CaptureIdentifierFilter and run all inputs through the // filter with the pipeline to capture the inventory of resources // which we are interested in. - captureFilter := &CaptureIdentifiersFilter{} + captureFilter := &CaptureIdentifiersFilter{ + Mapper: mapper, + } filters := []kio.Filter{captureFilter} var inputs []kio.Reader diff --git a/kustomize/internal/commands/status/cmd/helpers_test.go b/kustomize/internal/commands/status/cmd/helpers_test.go index c6892cb4b..b55df7e2c 100644 --- a/kustomize/internal/commands/status/cmd/helpers_test.go +++ b/kustomize/internal/commands/status/cmd/helpers_test.go @@ -241,7 +241,7 @@ func (f *FakeClient) List(context.Context, runtime.Object, ...client.ListOption) } func fakeResolver(fakeClient client.Reader, mapperTypes ...schema.GroupVersionKind) newResolverFunc { - return func(pollInterval time.Duration) (*wait.Resolver, error) { + return func(pollInterval time.Duration) (*wait.Resolver, meta.RESTMapper, error) { var groupVersions []schema.GroupVersion for _, gvk := range mapperTypes { groupVersions = append(groupVersions, gvk.GroupVersion()) @@ -251,7 +251,7 @@ func fakeResolver(fakeClient client.Reader, mapperTypes ...schema.GroupVersionKi mapper.Add(gvk, meta.RESTScopeNamespace) } - return wait.NewResolver(fakeClient, mapper, pollInterval), nil + return wait.NewResolver(fakeClient, mapper, pollInterval), mapper, nil } } diff --git a/kustomize/internal/commands/status/cmd/print.go b/kustomize/internal/commands/status/cmd/print.go index 8af0aaeb5..3371647a8 100644 --- a/kustomize/internal/commands/status/cmd/print.go +++ b/kustomize/internal/commands/status/cmd/print.go @@ -227,11 +227,11 @@ type eventColumnInfo struct { var ( eventColumns = []eventColumnInfo{ { - header: "EVENT TYPE", + header: "NAMESPACE", width: 15, - requireResourceUpdateEvent: false, + requireResourceUpdateEvent: true, contentFunc: func(event wait.Event) string { - return string(event.Type) + return event.EventResource.ResourceIdentifier.Namespace }, }, { @@ -247,16 +247,7 @@ var ( width: 20, requireResourceUpdateEvent: true, contentFunc: func(event wait.Event) string { - return fmt.Sprintf("%s/%s", event.EventResource.ResourceIdentifier.GroupKind.Group, - event.EventResource.ResourceIdentifier.GroupKind.Kind) - }, - }, - { - header: "NAMESPACE", - width: 15, - requireResourceUpdateEvent: true, - contentFunc: func(event wait.Event) string { - return event.EventResource.ResourceIdentifier.Namespace + return event.EventResource.ResourceIdentifier.GroupKind.Kind }, }, { @@ -278,12 +269,20 @@ var ( { header: "MESSAGE", width: 50, - requireResourceUpdateEvent: true, + requireResourceUpdateEvent: false, contentFunc: func(event wait.Event) string { - if event.EventResource.Error != nil { - return event.EventResource.Error.Error() + switch event.Type { + case wait.ResourceUpdate: + if event.EventResource.Error != nil { + return event.EventResource.Error.Error() + } + return event.EventResource.Message + case wait.Aborted: + return fmt.Sprint("Operation aborted before all resources have become Current") + case wait.Completed: + return fmt.Sprint("All resources have become Current") } - return event.EventResource.Message + return "" }, }, } @@ -308,11 +307,14 @@ func newEventPrinter(out io.Writer, err io.Writer) *EventPrinter { func (e *EventPrinter) printEvent(event wait.Event) { for _, column := range eventColumns { + var text string if event.Type != wait.ResourceUpdate && column.requireResourceUpdateEvent { - continue + text = "" + } else { + text = trimString(column.contentFunc(event), column.width) } format := fmt.Sprintf("%%-%ds ", column.width) - printOrDie(e.out, format, trimString(column.contentFunc(event), column.width)) + printOrDie(e.out, format, text) } printOrDie(e.out, "\n") } diff --git a/kustomize/internal/commands/status/cmd/util.go b/kustomize/internal/commands/status/cmd/util.go index 6071f145d..0388ea0c1 100644 --- a/kustomize/internal/commands/status/cmd/util.go +++ b/kustomize/internal/commands/status/cmd/util.go @@ -6,6 +6,7 @@ package cmd import ( "time" + "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" clientgoscheme "k8s.io/client-go/kubernetes/scheme" @@ -25,49 +26,64 @@ func init() { _ = clientgoscheme.AddToScheme(scheme) } -type newResolverFunc func(pollInterval time.Duration) (*wait.Resolver, error) +type newResolverFunc func(pollInterval time.Duration) (*wait.Resolver, meta.RESTMapper, error) // newResolver returns a new resolver that can resolve status for resources based // on polling the cluster. -func newResolver(pollInterval time.Duration) (*wait.Resolver, error) { +func newResolver(pollInterval time.Duration) (*wait.Resolver, meta.RESTMapper, error) { config := ctrl.GetConfigOrDie() mapper, err := apiutil.NewDiscoveryRESTMapper(config) if err != nil { - return nil, err + return nil, nil, err } c, err := client.New(config, client.Options{Scheme: scheme, Mapper: mapper}) if err != nil { - return nil, err + return nil, nil, err } - return wait.NewResolver(c, mapper, pollInterval), nil + return wait.NewResolver(c, mapper, pollInterval), mapper, nil } // CaptureIdentifiersFilter implements the Filter interface in the kio package. It // captures the identifiers for all resources passed through the pipeline. type CaptureIdentifiersFilter struct { Identifiers []wait.ResourceIdentifier + Mapper meta.RESTMapper } var _ kio.Filter = &CaptureIdentifiersFilter{} func (f *CaptureIdentifiersFilter) Filter(slice []*yaml.RNode) ([]*yaml.RNode, error) { for i := range slice { - meta, err := slice[i].GetMeta() + objectMeta, err := slice[i].GetMeta() if err != nil { return nil, err } // TODO(mortent): Update kyaml library - id := meta.GetIdentifier() + id := objectMeta.GetIdentifier() gv, err := schema.ParseGroupVersion(id.APIVersion) if err != nil { return nil, err } + gk := schema.GroupKind{ + Group: gv.Group, + Kind: id.Kind, + } + mapping, err := f.Mapper.RESTMapping(gk) + if err != nil { + return nil, err + } + var namespace string + if mapping.Scope.Name() == meta.RESTScopeNameNamespace && id.Namespace == "" { + namespace = "default" + } else { + namespace = id.Namespace + } if IsValidKubernetesResource(id) { f.Identifiers = append(f.Identifiers, wait.ResourceIdentifier{ Name: id.Name, - Namespace: id.Namespace, + Namespace: namespace, GroupKind: schema.GroupKind{ Group: gv.Group, Kind: id.Kind, diff --git a/kustomize/internal/commands/status/cmd/wait.go b/kustomize/internal/commands/status/cmd/wait.go index 60a319696..53665b6ce 100644 --- a/kustomize/internal/commands/status/cmd/wait.go +++ b/kustomize/internal/commands/status/cmd/wait.go @@ -60,12 +60,14 @@ type WaitRunner struct { func (r *WaitRunner) runE(c *cobra.Command, args []string) error { ctx := context.Background() - resolver, err := r.newResolverFunc(r.Interval) + resolver, mapper, err := r.newResolverFunc(r.Interval) if err != nil { return errors.Wrap(err, "errors creating resolver") } - captureFilter := &CaptureIdentifiersFilter{} + captureFilter := &CaptureIdentifiersFilter{ + Mapper: mapper, + } filters := []kio.Filter{captureFilter} var inputs []kio.Reader From b7b88cae7654aaec858d7c7a9fea1fe97faab5bb Mon Sep 17 00:00:00 2001 From: Haiyan Meng Date: Thu, 23 Jan 2020 16:04:55 -0800 Subject: [PATCH 21/46] Add curl commands for querying different filetypes --- .../crawl/search_cmds/creationTime.md | 211 +++++++++++++- api/internal/crawl/search_cmds/fileType.md | 178 ++++++++++++ .../crawl/search_cmds/repositoryUrl.md | 136 ++++++++- api/internal/crawl/search_cmds/user.md | 265 +++++++++++++++++- 4 files changed, 781 insertions(+), 9 deletions(-) diff --git a/api/internal/crawl/search_cmds/creationTime.md b/api/internal/crawl/search_cmds/creationTime.md index d7eafdc45..08b6b3e1e 100644 --- a/api/internal/crawl/search_cmds/creationTime.md +++ b/api/internal/crawl/search_cmds/creationTime.md @@ -46,7 +46,52 @@ curl -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'C "bool": { "must_not": { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" } - } + }, + "filter": [ + { "regexp": { "fileType": "resource" }} + ] + } + }, + "aggs" : { + "min_creationTime" : { "min" : { "field" : "creationTime" } } + } +} +' +``` + +Find out the smallest value of the `creationTime` field of all kustomize generator files: +``` +curl -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' +{ + "query": { + "bool": { + "must_not": { + "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" } + }, + "filter": [ + { "regexp": { "fileType": "generator" }} + ] + } + }, + "aggs" : { + "min_creationTime" : { "min" : { "field" : "creationTime" } } + } +} +' +``` + +Find out the smallest value of the `creationTime` field of all kustomize transformer files: +``` +curl -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' +{ + "query": { + "bool": { + "must_not": { + "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" } + }, + "filter": [ + { "regexp": { "fileType": "transformer" }} + ] } }, "aggs" : { @@ -142,6 +187,61 @@ curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Co "bool": { "must_not": [ { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" }} + ], + "filter": [ + { "regexp": { "fileType": "resource" }} + ] + } + }, + "aggs" : { + "newFiles_over_time" : { + "date_histogram" : { + "field" : "creationTime", + "interval" : "month" + } + } + } +} +' +``` + +Aggregate how many new kustomize generator files were added into Github each month: +``` +curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' +{ + "query": { + "bool": { + "must_not": [ + { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" }} + ], + "filter": [ + { "regexp": { "fileType": "generator" }} + ] + } + }, + "aggs" : { + "newFiles_over_time" : { + "date_histogram" : { + "field" : "creationTime", + "interval" : "month" + } + } + } +} +' +``` + +Aggregate how many new kustomize transformer files were added into Github each month: +``` +curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' +{ + "query": { + "bool": { + "must_not": [ + { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" }} + ], + "filter": [ + { "regexp": { "fileType": "transformer" }} ] } }, @@ -188,7 +288,10 @@ curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Co "bool": { "must_not": [ { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" }} - ] + ], + "filter": [ + { "regexp": { "fileType": "resource" }} + ] } }, "aggs" : { @@ -201,4 +304,108 @@ curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Co } } ' +``` + +Aggregate how many new kustomize generator files were added into Github each year: +``` +curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' +{ + "query": { + "bool": { + "must_not": [ + { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" }} + ], + "filter": [ + { "regexp": { "fileType": "generator" }} + ] + } + }, + "aggs" : { + "newFiles_over_time" : { + "date_histogram" : { + "field" : "creationTime", + "interval" : "year" + } + } + } +} +' +``` + +Aggregate how many new kustomize transformer files were added into Github each year: +``` +curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' +{ + "query": { + "bool": { + "must_not": [ + { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" }} + ], + "filter": [ + { "regexp": { "fileType": "transformer" }} + ] + } + }, + "aggs" : { + "newFiles_over_time" : { + "date_histogram" : { + "field" : "creationTime", + "interval" : "year" + } + } + } +} +' +``` + +Find the generator files created within the given time range: +``` +curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d' +{ + "query": { + "bool": { + "filter": [ + { "regexp": { "fileType": "generator" }} + ], + "must_not": { + "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" } + }, + "must": { + "range": { + "creationTime": { + "gte": "2019-04-26T16:40:02.000Z", + "lte": "2019-04-26T16:40:02.000Z" + } + } + } + } + } +} +' +``` + +Find the transformer files created within the given time range: +``` +curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d' +{ + "query": { + "bool": { + "filter": [ + { "regexp": { "fileType": "transformer" }} + ], + "must_not": { + "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" } + }, + "must": { + "range": { + "creationTime": { + "gte": "2019-04-26T16:40:02.000Z", + "lte": "2019-04-26T16:40:02.000Z" + } + } + } + } + } +} +' ``` \ No newline at end of file diff --git a/api/internal/crawl/search_cmds/fileType.md b/api/internal/crawl/search_cmds/fileType.md index c5c58357a..3be7b41e1 100644 --- a/api/internal/crawl/search_cmds/fileType.md +++ b/api/internal/crawl/search_cmds/fileType.md @@ -44,6 +44,184 @@ curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-T ' ``` +Search for all the kustomization files whose `fileType` field is `resource`: +``` +curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d' +{ + "query": { + "bool": { + "filter": [ + { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" }}, + { "regexp": { "fileType": "resource" }} + ] + } + } +} +' +``` + +Search for all the kustomize resource files: +``` +curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d' +{ + "query": { + "bool": { + "filter": [ + { "regexp": { "fileType": "resource" }} + ], + "must_not": { + "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" } + } + } + } +} +' +``` + +Search all the kustomization files including a `generators` field: +``` +curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d' +{ + "size": 10000, + "query": { + "bool": { + "must": { + "match" : { + "identifiers" : { + "query" : "generators" + } + } + }, + "filter": { + "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" } + } + } + } +} +' +``` + +Search for all the documents whose `fileType` field is `generator`: +``` +curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d' +{ + "query": { + "bool": { + "filter": [ + { "regexp": { "fileType": "generator" }} + ] + } + } +} +' +``` + +Search for all the kustomization files whose `fileType` field is `generator`: +``` +curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d' +{ + "query": { + "bool": { + "filter": [ + { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" }}, + { "regexp": { "fileType": "generator" }} + ] + } + } +} +' +``` + +Search for all the kustomize generator files: +``` +curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d' +{ + "query": { + "bool": { + "filter": [ + { "regexp": { "fileType": "generator" }} + ], + "must_not": { + "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" } + } + } + } +} +' +``` + +Search all the kustomization files including a `transformers` field: +``` +curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d' +{ + "size": 10000, + "query": { + "bool": { + "must": { + "match" : { + "identifiers" : { + "query" : "transformers" + } + } + }, + "filter": { + "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" } + } + } + } +} +' +``` + +Search for all the documents whose `fileType` field is `transformer`: +``` +curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d' +{ + "query": { + "bool": { + "filter": [ + { "regexp": { "fileType": "transformer" }} + ] + } + } +} +' +``` + +Search for all the kustomization files whose `fileType` field is `transformer`: +``` +curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d' +{ + "query": { + "bool": { + "filter": [ + { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" }}, + { "regexp": { "fileType": "transformer" }} + ] + } + } +} +' +``` + +Search for all the kustomize transformer files: +``` +curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d' +{ + "query": { + "bool": { + "filter": [ + { "regexp": { "fileType": "transformer" }} + ], + "must_not": { + "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" } + } + } + } +} +' +``` + Count distinct values of the `fileType` field: ``` curl -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' diff --git a/api/internal/crawl/search_cmds/repositoryUrl.md b/api/internal/crawl/search_cmds/repositoryUrl.md index 9ce169929..37e99d088 100644 --- a/api/internal/crawl/search_cmds/repositoryUrl.md +++ b/api/internal/crawl/search_cmds/repositoryUrl.md @@ -37,16 +37,143 @@ curl -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'C ' ``` -Count how many Github repositories include kustomize resource files: +Count distinct values of the `repositoryUrl` field for all the kustomize resource files in the index: ``` curl -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' { "query": { "bool": { "must_not": { - "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" } + "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" } + }, + "filter": [ + { "regexp": { "fileType": "resource" }} + ] + } + }, + "aggs" : { + "repositoryUrl_count" : { + "cardinality" : { + "field" : "repositoryUrl", + "precision_threshold": 40000 } } + } +} +' +``` + +Count distinct values of the `repositoryUrl` field for all the kustomize generator files in the index: +``` +curl -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' +{ + "query": { + "bool": { + "must_not": { + "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" } + }, + "filter": [ + { "regexp": { "fileType": "generator" }} + ] + } + }, + "aggs" : { + "repositoryUrl_count" : { + "cardinality" : { + "field" : "repositoryUrl", + "precision_threshold": 40000 + } + } + } +} +' +``` + +Count distinct values of the `repositoryUrl` field for all the kustomize transformer files in the index: +``` +curl -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' +{ + "query": { + "bool": { + "must_not": { + "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" } + }, + "filter": [ + { "regexp": { "fileType": "transformer" }} + ] + } + }, + "aggs" : { + "repositoryUrl_count" : { + "cardinality" : { + "field" : "repositoryUrl", + "precision_threshold": 40000 + } + } + } +} +' +``` + +Count distinct values of the `repositoryUrl` field for all the kustomize resource dirs in the index: +``` +curl -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' +{ + "query": { + "bool": { + "filter": [ + { "regexp": { "fileType": "resource" }}, + { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" }} + ] + } + }, + "aggs" : { + "repositoryUrl_count" : { + "cardinality" : { + "field" : "repositoryUrl", + "precision_threshold": 40000 + } + } + } +} +' +``` + +Count distinct values of the `repositoryUrl` field for all the kustomize generator dirs in the index: +``` +curl -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' +{ + "query": { + "bool": { + "filter": [ + { "regexp": { "fileType": "generator" }}, + { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" }} + ] + } + }, + "aggs" : { + "repositoryUrl_count" : { + "cardinality" : { + "field" : "repositoryUrl", + "precision_threshold": 40000 + } + } + } +} +' +``` + +Count distinct values of the `repositoryUrl` field for all the kustomize transformer dirs in the index: +``` +curl -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' +{ + "query": { + "bool": { + "filter": [ + { "regexp": { "fileType": "transformer" }}, + { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" }} + ] + } }, "aggs" : { "repositoryUrl_count" : { @@ -109,7 +236,10 @@ curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Co "bool": { "must_not": { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" } - } + }, + "filter": [ + { "regexp": { "fileType": "resource" }} + ] } }, "aggs" : { diff --git a/api/internal/crawl/search_cmds/user.md b/api/internal/crawl/search_cmds/user.md index b8ae3538c..83f4cd2e5 100644 --- a/api/internal/crawl/search_cmds/user.md +++ b/api/internal/crawl/search_cmds/user.md @@ -67,7 +67,8 @@ curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Co "aggs" : { "user" : { "terms" : { - "field" : "user" + "field" : "user", + "size" : 20 } } } @@ -75,6 +76,28 @@ curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Co ' ``` +Count distinct values of the `user` field for all the kustomization files in the index: +``` +curl -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' +{ + "query": { + "bool": { + "filter": [ + { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" }} + ] + } + }, + "aggs" : { + "user_count" : { + "cardinality" : { + "field" : "user", + "precision_threshold": 40000 + } + } + } +} +' +``` For all the kustomization files in the index, list all the values of the `user` field and the frequency of each value: @@ -91,7 +114,8 @@ curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Co "aggs" : { "user" : { "terms" : { - "field" : "user" + "field" : "user", + "size": 20 } } } @@ -99,7 +123,33 @@ curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Co ' ``` -For all the non-kustomization files in the index, list all the values of the +Count distinct values of the `user` field for all the kustomize resource files in the index: +``` +curl -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' +{ + "query": { + "bool": { + "must_not": { + "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" } + }, + "filter": [ + { "regexp": { "fileType": "resource" }} + ] + } + }, + "aggs" : { + "user_count" : { + "cardinality" : { + "field" : "user", + "precision_threshold": 40000 + } + } + } +} +' +``` + +For all the kustomize resource files in the index, list all the values of the `user` field and the frequency of each value: ``` curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' @@ -108,8 +158,215 @@ curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Co "bool": { "must_not": { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" } + }, + "filter": [ + { "regexp": { "fileType": "resource" }} + ] + } + }, + "aggs" : { + "user" : { + "terms" : { + "field" : "user", + "size": 20 } } + } +} +' +``` + +Count distinct values of the `user` field for all the kustomize generator files in the index: +``` +curl -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' +{ + "query": { + "bool": { + "must_not": { + "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" } + }, + "filter": [ + { "regexp": { "fileType": "generator" }} + ] + } + }, + "aggs" : { + "user_count" : { + "cardinality" : { + "field" : "user", + "precision_threshold": 40000 + } + } + } +} +' +``` + +For all the kustomize generator files in the index, list all the values of the +`user` field and the frequency of each value: +``` +curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' +{ + "query": { + "bool": { + "must_not": { + "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" } + }, + "filter": [ + { "regexp": { "fileType": "generator" }} + ] + } + }, + "aggs" : { + "user" : { + "terms" : { + "field" : "user", + "size": 20 + } + } + } +} +' +``` + +Count distinct values of the `user` field for all the kustomize transformer files in the index: +``` +curl -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' +{ + "query": { + "bool": { + "must_not": { + "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" } + }, + "filter": [ + { "regexp": { "fileType": "transformer" }} + ] + } + }, + "aggs" : { + "user_count" : { + "cardinality" : { + "field" : "user", + "precision_threshold": 40000 + } + } + } +} +' +``` + +For all the kustomize transformer files in the index, list all the values of the +`user` field and the frequency of each value: +``` +curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' +{ + "query": { + "bool": { + "must_not": { + "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" } + }, + "filter": [ + { "regexp": { "fileType": "transformer" }} + ] + } + }, + "aggs" : { + "user" : { + "terms" : { + "field" : "user", + "size": 20 + } + } + } +} +' +``` + +Count distinct values of the `user` field for all the kustomize generator dirs in the index: +``` +curl -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' +{ + "query": { + "bool": { + "filter": [ + { "regexp": { "fileType": "generator" }}, + { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" }} + ] + } + }, + "aggs" : { + "user_count" : { + "cardinality" : { + "field" : "user", + "precision_threshold": 40000 + } + } + } +} +' +``` + +For all the kustomize generator dirs in the index, list all the values of the +`user` field and the frequency of each value: +``` +curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' +{ + "query": { + "bool": { + "filter": [ + { "regexp": { "fileType": "generator" }}, + { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" }} + ] + } + }, + "aggs" : { + "user" : { + "terms" : { + "field" : "user", + "size": 20 + } + } + } +} +' +``` + +Count distinct values of the `user` field for all the kustomize transformer dirs in the index: +``` +curl -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' +{ + "query": { + "bool": { + "filter": [ + { "regexp": { "fileType": "transformer" }}, + { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" }} + ] + } + }, + "aggs" : { + "user_count" : { + "cardinality" : { + "field" : "user", + "precision_threshold": 40000 + } + } + } +} +' +``` + +For all the kustomize transformer dirs in the index, list all the values of the +`user` field and the frequency of each value: +``` +curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' +{ + "query": { + "bool": { + "filter": [ + { "regexp": { "fileType": "transformer" }}, + { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" }} + ] + } }, "aggs" : { "user" : { @@ -120,4 +377,4 @@ curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Co } } ' -``` +``` \ No newline at end of file From 7b44f71caf08fc8ecdbfaedc5bc7a8c7cb247385 Mon Sep 17 00:00:00 2001 From: Sean Sullivan Date: Thu, 23 Jan 2020 16:37:58 -0800 Subject: [PATCH 22/46] Adds the PruneOptions and implements the methods for this struct --- cmd/kubectl/kubectlcobra/prune.go | 258 +++++++++++++++++++++++++ cmd/kubectl/kubectlcobra/prune_test.go | 244 +++++++++++++++++++++++ 2 files changed, 502 insertions(+) create mode 100644 cmd/kubectl/kubectlcobra/prune.go create mode 100644 cmd/kubectl/kubectlcobra/prune_test.go diff --git a/cmd/kubectl/kubectlcobra/prune.go b/cmd/kubectl/kubectlcobra/prune.go new file mode 100644 index 000000000..c7d9ffbe2 --- /dev/null +++ b/cmd/kubectl/kubectlcobra/prune.go @@ -0,0 +1,258 @@ +// Copyright 2019 The Kubernetes Authors. +// SPDX-License-Identifier: Apache-2.0 + +// package kubectlcobra contains cobra commands from kubectl +package kubectlcobra + +import ( + "fmt" + "io" + "strings" + + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/cli-runtime/pkg/printers" + "k8s.io/cli-runtime/pkg/resource" + "k8s.io/client-go/dynamic" + "k8s.io/kubectl/pkg/cmd/apply" + "k8s.io/kubectl/pkg/cmd/util" + "k8s.io/kubectl/pkg/validation" +) + +// PruneOptions encapsulates the necessary information to +// implement the prune functionality. +type PruneOptions struct { + client dynamic.Interface + builder *resource.Builder + mapper meta.RESTMapper + namespace string + // The currently applied objects (as Infos), including the + // current grouping object. These objects are used to + // calculate the prune set after retreiving the previous + // grouping objects. + currentGroupingObject *resource.Info + // The set of retrieved grouping objects (as Infos) selected + // by the grouping label. This set should also include the + // current grouping object. Stored here to make testing + // easier by manually setting the retrieved grouping infos. + pastGroupingObjects []*resource.Info + retrievedGroupingObjects bool + + toPrinter func(string) (printers.ResourcePrinter, error) + out io.Writer + + validator validation.Schema + + // TODO: DeleteOptions--cascade? +} + +// NewPruneOptions returns a struct (PruneOptions) encapsulating the necessary +// information to run the prune. Returns an error if an error occurs +// gathering this information. +// TODO: Add dry-run options. +func NewPruneOptions(f util.Factory, ao *apply.ApplyOptions) (*PruneOptions, error) { + + po := &PruneOptions{} + var err error + // Fields copied from ApplyOptions. + po.namespace = ao.Namespace + po.toPrinter = ao.ToPrinter + po.out = ao.Out + // Client/Builder fields from the Factory. + po.client, err = f.DynamicClient() + if err != nil { + return nil, err + } + po.builder = f.NewBuilder() + po.mapper, err = f.ToRESTMapper() + if err != nil { + return nil, err + } + po.validator, err = f.Validator(false) + if err != nil { + return nil, err + } + // Retrieve/store the grouping object for current apply. + currentObjects, err := ao.GetObjects() + if err != nil { + return nil, err + } + currentGroupingObject, found := findGroupingObject(currentObjects) + if !found { + return nil, fmt.Errorf("Current grouping object not found during prune.") + } + po.currentGroupingObject = currentGroupingObject + // Initialize past grouping objects as empty. + po.pastGroupingObjects = []*resource.Info{} + po.retrievedGroupingObjects = false + + return po, nil +} + +// getPreviousGroupingObjects returns the set of grouping objects +// that have the same label as the current grouping object. Removes +// the current grouping object from this set. Returns an error +// if there is a problem retrieving the grouping objects. +func (po *PruneOptions) getPreviousGroupingObjects() ([]*resource.Info, error) { + + // Ensures the "pastGroupingObjects" is set. + if !po.retrievedGroupingObjects { + if err := po.retrievePreviousGroupingObjects(); err != nil { + return nil, err + } + } + + // Remove the current grouping info from the previous grouping infos. + currentInventory, err := infoToInventory(po.currentGroupingObject) + if err != nil { + return nil, err + } + pastGroupInfos := []*resource.Info{} + for _, pastInfo := range po.pastGroupingObjects { + pastInventory, err := infoToInventory(pastInfo) + if err != nil { + return nil, err + } + if !currentInventory.Equals(pastInventory) { + pastGroupInfos = append(pastGroupInfos, pastInfo) + } + } + return pastGroupInfos, nil +} + +// retrievePreviousGroupingObjects requests the previous grouping objects +// using the grouping label from the current grouping object. Sets +// the field "pastGroupingObjects". Returns an error if the grouping +// label doesn't exist for the current currentGroupingObject does not +// exist or if the call to retrieve the past grouping objects fails. +func (po *PruneOptions) retrievePreviousGroupingObjects() error { + // Get the grouping label for this grouping object, and create + // a label selector from it. + if po.currentGroupingObject == nil || po.currentGroupingObject.Object == nil { + return fmt.Errorf("Missing current grouping object.\n") + } + groupingLabel, err := retrieveGroupingLabel(po.currentGroupingObject.Object) + if err != nil { + return err + } + labelSelector := fmt.Sprintf("%s=%s", GroupingLabel, groupingLabel) + retrievedGroupingInfos, err := po.builder. + Unstructured(). + // TODO: Check if this validator is necessary. + Schema(po.validator). + ContinueOnError(). + NamespaceParam(po.namespace).DefaultNamespace(). + ResourceTypes("configmap"). + LabelSelectorParam(labelSelector). + Flatten(). + Do(). + Infos() + if err != nil { + return err + } + po.pastGroupingObjects = retrievedGroupingInfos + po.retrievedGroupingObjects = true + return nil +} + +// infoToInventory transforms the object represented by the passed "info" +// into its Inventory representation. Returns error if the passed Info +// is nil, or the Object in the Info is empty. +func infoToInventory(info *resource.Info) (*Inventory, error) { + if info == nil || info.Object == nil { + return nil, fmt.Errorf("Empty resource.Info can not calculate as inventory.\n") + } + obj := info.Object + gk := obj.GetObjectKind().GroupVersionKind().GroupKind() + return createInventory(info.Namespace, info.Name, gk) +} + +// unionPastInventory takes a set of grouping objects (infos), returning the +// union of the objects referenced by these grouping objects as an +// InventorySet. Returns an error if any of the passed objects are not +// grouping objects, or if unable to retrieve the inventory from any +// grouping object. +func unionPastInventory(infos []*resource.Info) (*InventorySet, error) { + inventorySet := NewInventorySet([]*Inventory{}) + for _, info := range infos { + inv, err := retrieveInventoryFromGroupingObj([]*resource.Info{info}) + if err != nil { + return nil, err + } + inventorySet.AddItems(inv) + } + return inventorySet, nil +} + +// calcPruneSet returns the InventorySet representing the objects to +// delete (prune). pastGroupInfos are the set of past applied grouping +// objects, storing the inventory of the objects applied at the same time. +// Calculates the prune set as: +// +// prune set = (prev1 U prev2 U ... U prevN) - (curr1, curr2, ..., currN) +// +// Returns an error if we are unable to retrieve the set of previously +// applied objects, or if we are unable to get the currently applied objects +// from the current grouping object. +func (po *PruneOptions) calcPruneSet(pastGroupingInfos []*resource.Info) (*InventorySet, error) { + pastInventory, err := unionPastInventory(pastGroupingInfos) + if err != nil { + return nil, err + } + // Current grouping object as inventory set. + c := []*resource.Info{po.currentGroupingObject} + currentInv, err := retrieveInventoryFromGroupingObj(c) + if err != nil { + return nil, err + } + return pastInventory.Subtract(NewInventorySet(currentInv)) +} + +// Prune deletes the set of resources which were previously applied +// (retrieved from previous grouping objects) but omitted in +// the current apply. Prune also delete all previous grouping +// objects. Returns an error if there was a problem. +func (po *PruneOptions) Prune() error { + + // Retrieve previous grouping objects, and calculate the + // union of the previous applies as an inventory set. + pastGroupingInfos, err := po.getPreviousGroupingObjects() + if err != nil { + return err + } + pruneSet, err := po.calcPruneSet(pastGroupingInfos) + if err != nil { + return err + } + + // Delete the prune objects. + for _, inv := range pruneSet.GetItems() { + mapping, err := po.mapper.RESTMapping(inv.GroupKind) + if err != nil { + return err + } + err = po.client.Resource(mapping.Resource).Namespace(inv.Namespace).Delete(inv.Name, &metav1.DeleteOptions{}) + if err != nil { + return err + } + fmt.Fprintf(po.out, "%s/%s deleted\n", strings.ToLower(inv.GroupKind.Kind), inv.Name) + } + // Delete previous grouping objects. + for _, pastGroupInfo := range pastGroupingInfos { + err = po.client.Resource(pastGroupInfo.Mapping.Resource). + Namespace(pastGroupInfo.Namespace). + Delete(pastGroupInfo.Name, &metav1.DeleteOptions{}) + if err != nil { + return err + } + printer, err := po.toPrinter("deleted") + if err != nil { + return err + } + if err = printer.PrintObj(pastGroupInfo.Object, po.out); err != nil { + return err + } + } + + return nil +} diff --git a/cmd/kubectl/kubectlcobra/prune_test.go b/cmd/kubectl/kubectlcobra/prune_test.go new file mode 100644 index 000000000..20faa9936 --- /dev/null +++ b/cmd/kubectl/kubectlcobra/prune_test.go @@ -0,0 +1,244 @@ +// Copyright 2019 The Kubernetes Authors. +// SPDX-License-Identifier: Apache-2.0 + +// package kubectlcobra contains cobra commands from kubectl +package kubectlcobra + +import ( + "testing" + + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/cli-runtime/pkg/resource" +) + +var pod1Inv = &Inventory{ + Namespace: testNamespace, + Name: pod1Name, + GroupKind: schema.GroupKind{ + Group: "", + Kind: "Pod", + }, +} + +var pod2Inv = &Inventory{ + Namespace: testNamespace, + Name: pod2Name, + GroupKind: schema.GroupKind{ + Group: "", + Kind: "Pod", + }, +} + +var pod3Inv = &Inventory{ + Namespace: testNamespace, + Name: pod3Name, + GroupKind: schema.GroupKind{ + Group: "", + Kind: "Pod", + }, +} + +var groupingInv = &Inventory{ + Namespace: testNamespace, + Name: groupingObjName, + GroupKind: schema.GroupKind{ + Group: "", + Kind: "ConfigMap", + }, +} + +func TestInfoToInventory(t *testing.T) { + tests := map[string]struct { + info *resource.Info + expected *Inventory + isError bool + }{ + "Nil info is an error": { + info: nil, + expected: nil, + isError: true, + }, + "Nil info object is an error": { + info: nilInfo, + expected: nil, + isError: true, + }, + "Pod 1 object becomes Pod 1 inventory": { + info: pod1Info, + expected: pod1Inv, + isError: false, + }, + "Grouping object becomes grouping inventory": { + info: copyGroupingInfo(), + expected: groupingInv, + isError: false, + }, + } + + for name, test := range tests { + t.Run(name, func(t *testing.T) { + actual, err := infoToInventory(test.info) + if test.isError && err == nil { + t.Errorf("Did not receive expected error.\n") + } + if !test.isError { + if err != nil { + t.Errorf("Receieved unexpected error: %s\n", err) + } + if !test.expected.Equals(actual) { + t.Errorf("Expected inventory (%s), got (%s)\n", test.expected, actual) + } + } + }) + } +} + +// Returns a grouping object with the inventory set from +// the passed "children". +func createGroupingInfo(name string, children ...(*resource.Info)) *resource.Info { + groupingObjCopy := groupingObj.DeepCopy() + var groupingInfo = &resource.Info{ + Namespace: testNamespace, + Name: groupingObjName, + Object: groupingObjCopy, + } + infos := []*resource.Info{groupingInfo} + infos = append(infos, children...) + _ = addInventoryToGroupingObj(infos) + return groupingInfo +} + +func TestUnionPastInventory(t *testing.T) { + tests := map[string]struct { + groupingInfos []*resource.Info + expected []*Inventory + }{ + "Empty grouping objects = empty inventory set": { + groupingInfos: []*resource.Info{}, + expected: []*Inventory{}, + }, + "No children in grouping object, equals no inventory": { + groupingInfos: []*resource.Info{createGroupingInfo("test-1")}, + expected: []*Inventory{}, + }, + "Grouping object with Pod1 returns inventory with Pod1": { + groupingInfos: []*resource.Info{createGroupingInfo("test-1", pod1Info)}, + expected: []*Inventory{pod1Inv}, + }, + "Grouping object with three pods returns inventory with three pods": { + groupingInfos: []*resource.Info{ + createGroupingInfo("test-1", pod1Info, pod2Info, pod3Info), + }, + expected: []*Inventory{pod1Inv, pod2Inv, pod3Inv}, + }, + "Two grouping objects with different pods returns inventory with both pods": { + groupingInfos: []*resource.Info{ + createGroupingInfo("test-1", pod1Info), + createGroupingInfo("test-2", pod2Info), + }, + expected: []*Inventory{pod1Inv, pod2Inv}, + }, + "Two grouping objects with overlapping pods returns set of pods": { + groupingInfos: []*resource.Info{ + createGroupingInfo("test-1", pod1Info, pod2Info), + createGroupingInfo("test-2", pod2Info, pod3Info), + }, + expected: []*Inventory{pod1Inv, pod2Inv, pod3Inv}, + }, + } + + for name, test := range tests { + t.Run(name, func(t *testing.T) { + actual, err := unionPastInventory(test.groupingInfos) + expected := NewInventorySet(test.expected) + if err != nil { + t.Errorf("Unexpected error received: %s\n", err) + } + if !expected.Equals(actual) { + t.Errorf("Expected inventory (%s), got (%s)\n", expected, actual) + } + }) + } +} + +func TestCalcPruneSet(t *testing.T) { + tests := map[string]struct { + past []*resource.Info + current *resource.Info + expected []*Inventory + isError bool + }{ + "Object not unstructured--error": { + past: []*resource.Info{nonUnstructuredGroupingInfo}, + current: &resource.Info{}, + expected: []*Inventory{}, + isError: true, + }, + "No past group objects--no prune set": { + + past: []*resource.Info{}, + current: createGroupingInfo("test-1"), + expected: []*Inventory{}, + isError: false, + }, + "Empty past grouping object--no prune set": { + past: []*resource.Info{createGroupingInfo("test-1")}, + current: createGroupingInfo("test-1"), + expected: []*Inventory{}, + isError: false, + }, + "Pod1 - Pod1 = empty set": { + past: []*resource.Info{ + createGroupingInfo("test-1", pod1Info), + }, + current: createGroupingInfo("test-1", pod1Info), + expected: []*Inventory{}, + isError: false, + }, + "(Pod1, Pod2) - Pod1 = Pod2": { + past: []*resource.Info{ + createGroupingInfo("test-1", pod1Info, pod2Info), + }, + current: createGroupingInfo("test-1", pod1Info), + expected: []*Inventory{pod2Inv}, + isError: false, + }, + "(Pod1, Pod2) - Pod2 = Pod1": { + past: []*resource.Info{ + createGroupingInfo("test-1", pod1Info, pod2Info), + }, + current: createGroupingInfo("test-1", pod2Info), + expected: []*Inventory{pod1Inv}, + isError: false, + }, + "(Pod1, Pod2, Pod3) - Pod2 = Pod1, Pod3": { + past: []*resource.Info{ + createGroupingInfo("test-1", pod1Info, pod2Info), + createGroupingInfo("test-1", pod2Info, pod3Info), + }, + current: createGroupingInfo("test-1", pod2Info), + expected: []*Inventory{pod1Inv, pod3Inv}, + isError: false, + }, + } + + for name, test := range tests { + t.Run(name, func(t *testing.T) { + po := &PruneOptions{} + po.currentGroupingObject = test.current + actual, err := po.calcPruneSet(test.past) + expected := NewInventorySet(test.expected) + if test.isError && err == nil { + t.Errorf("Did not receive expected error.\n") + } + if !test.isError { + if err != nil { + t.Errorf("Unexpected error received: %s\n", err) + } + if !expected.Equals(actual) { + t.Errorf("Expected prune set (%s), got (%s)\n", expected, actual) + } + } + }) + } +} From 0c022db1e6031ed4c3e2b2ef3c3f2050b2d5ab9b Mon Sep 17 00:00:00 2001 From: Sean Sullivan Date: Fri, 24 Jan 2020 15:10:27 -0800 Subject: [PATCH 23/46] Connect prune to apply --- cmd/kubectl/kubectlcobra/commands.go | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/cmd/kubectl/kubectlcobra/commands.go b/cmd/kubectl/kubectlcobra/commands.go index ade1205cc..8fe5146f8 100644 --- a/cmd/kubectl/kubectlcobra/commands.go +++ b/cmd/kubectl/kubectlcobra/commands.go @@ -81,6 +81,7 @@ func updateHelp(names []string, c *cobra.Command) { func NewCmdApply(baseName string, f util.Factory, ioStreams genericclioptions.IOStreams) *cobra.Command { o := apply.NewApplyOptions(ioStreams) so := newStatusOptions(f, ioStreams) + // Set up grouping object for this apply; used in subsequent prune. o.PreProcessorFn = PrependGroupingObject(o) cmd := &cobra.Command{ @@ -97,6 +98,9 @@ func NewCmdApply(baseName string, f util.Factory, ioStreams genericclioptions.IO } cmdutil.CheckErr(o.Complete(f, cmd)) + // Default PostProcessor is configured in "Complete" function, + // so the prune must happen after "Complete". + o.PostProcessorFn = prune(f, o) cmdutil.CheckErr(o.Run()) infos, _ := o.GetObjects() if so.wait { @@ -144,3 +148,16 @@ func PrependGroupingObject(o *apply.ApplyOptions) func() error { return nil } } + +// Prune deletes previously applied objects that have been +// omitted in the current apply. The previously applied objects +// are reached through ConfigMap grouping objects. +func prune(f util.Factory, o *apply.ApplyOptions) func() error { + return func() error { + po, err := NewPruneOptions(f, o) + if err != nil { + return err + } + return po.Prune() + } +} From 154208d331f9397b2c1b902761c565ea1f50e7da Mon Sep 17 00:00:00 2001 From: Haiyan Meng Date: Fri, 24 Jan 2020 19:55:56 -0800 Subject: [PATCH 24/46] Improve the efficiency of crawling github by skipping the documents already in the index --- api/internal/crawl/cmd/crawler/crawler.go | 6 ++++++ api/internal/crawl/crawler/crawler.go | 2 ++ 2 files changed, 8 insertions(+) diff --git a/api/internal/crawl/cmd/crawler/crawler.go b/api/internal/crawl/cmd/crawler/crawler.go index 54bf3f432..e534780d2 100644 --- a/api/internal/crawl/cmd/crawler/crawler.go +++ b/api/internal/crawl/cmd/crawler/crawler.go @@ -187,6 +187,12 @@ func main() { crawler.CrawlFromSeed(ctx, seedDocs, crawlers, docConverter, indexFunc, seen) case CrawlGithub: crawlers := []crawler.Crawler{ghCrawlerConstructor("", "")} + // add all the documents in the index into seen. + // this greatly reduces the time overhead of CrawlGithub. + getSeedDocsFunc() + for _, d := range seedDocs { + seen[d.ID()] = d.FileType + } crawler.CrawlGithub(ctx, crawlers, docConverter, indexFunc, seen) case CrawlUser: if *githubUserPtr == "" { diff --git a/api/internal/crawl/crawler/crawler.go b/api/internal/crawl/crawler/crawler.go index 9a6a401d6..7488fcf3d 100644 --- a/api/internal/crawl/crawler/crawler.go +++ b/api/internal/crawl/crawler/crawler.go @@ -82,6 +82,8 @@ func addBranches(cdoc CrawledDocument, match Crawler, indx IndexFunc, seen.Set(cdoc.ID(), cdoc.GetDocument().FileType) + match.SetDefaultBranch(cdoc.GetDocument()) + // Insert into index if err := indx(cdoc, index.InsertOrUpdate); err != nil { logger.Printf("Failed to insert or update doc(%s): %v", From 7eca29daeee6b583f5394a45d8edfd41c15dbe6d Mon Sep 17 00:00:00 2001 From: Karl Gustav Date: Sat, 25 Jan 2020 20:43:37 +0100 Subject: [PATCH 25/46] Fix install_kustomize.sh so it works on alpine linux The install script fails and thinks that alpine linux is in windows. This is because `$OSTYPE` in alpine linux is linux-musl, not linux-gnu as this script assumes. I tested these changes with this script: ``` set -euo pipefail opsys="" function check { opsys=windows if [[ "$OSTYPE" == linux* ]]; then opsys=linux elif [[ "$OSTYPE" == darwin* ]]; then opsys=darwin fi } OSTYPE="linux-gnu" check test "$opsys" == "linux" || echo $OSTYPE test failed OSTYPE="linuxsomething" check test "$opsys" == "linux" || echo $OSTYPE test failed OSTYPE="darwinsomething" check test "$opsys" == "darwin" || echo $OSTYPE test failed OSTYPE="either" check test "$opsys" == "windows" || echo $OSTYPE test failed ``` ref: #2146 --- hack/install_kustomize.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hack/install_kustomize.sh b/hack/install_kustomize.sh index e17a317fc..6e152d6dc 100755 --- a/hack/install_kustomize.sh +++ b/hack/install_kustomize.sh @@ -26,9 +26,9 @@ trap cleanup EXIT pushd $tmpDir >& /dev/null opsys=windows -if [[ "$OSTYPE" == "linux-gnu" ]]; then +if [[ "$OSTYPE" == linux* ]]; then opsys=linux -elif [[ "$OSTYPE" == "darwin"* ]]; then +elif [[ "$OSTYPE" == darwin* ]]; then opsys=darwin fi From 68195ffabbdaabf0c390699211b46f354b25a907 Mon Sep 17 00:00:00 2001 From: Morten Torkildsen Date: Wed, 15 Jan 2020 18:24:49 -0800 Subject: [PATCH 26/46] Restructure the Apply command to separate printing from the code that actually does the work --- cmd/kubectl/go.mod | 2 + cmd/kubectl/go.sum | 3 +- cmd/kubectl/kubectlcobra/applier.go | 205 ++++++++++++++++++ cmd/kubectl/kubectlcobra/applier_test.go | 48 ++++ cmd/kubectl/kubectlcobra/basic_printer.go | 62 ++++++ cmd/kubectl/kubectlcobra/commands.go | 42 ++-- cmd/kubectl/kubectlcobra/printer_adapter.go | 53 +++++ .../kubectlcobra/printer_adapter_test.go | 49 +++++ cmd/kubectl/kubectlcobra/status.go | 91 +------- 9 files changed, 448 insertions(+), 107 deletions(-) create mode 100644 cmd/kubectl/kubectlcobra/applier.go create mode 100644 cmd/kubectl/kubectlcobra/applier_test.go create mode 100644 cmd/kubectl/kubectlcobra/basic_printer.go create mode 100644 cmd/kubectl/kubectlcobra/printer_adapter.go create mode 100644 cmd/kubectl/kubectlcobra/printer_adapter_test.go diff --git a/cmd/kubectl/go.mod b/cmd/kubectl/go.mod index 3a42432e6..44372a890 100644 --- a/cmd/kubectl/go.mod +++ b/cmd/kubectl/go.mod @@ -3,7 +3,9 @@ module sigs.k8s.io/kustomize/cmd/kubectl go 1.13 require ( + github.com/go-errors/errors v1.0.1 github.com/spf13/cobra v0.0.5 + github.com/stretchr/testify v1.4.0 k8s.io/api v0.17.0 k8s.io/apimachinery v0.17.0 k8s.io/cli-runtime v0.17.0 diff --git a/cmd/kubectl/go.sum b/cmd/kubectl/go.sum index 1b5e06aa8..1f2220181 100644 --- a/cmd/kubectl/go.sum +++ b/cmd/kubectl/go.sum @@ -83,6 +83,7 @@ github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/globalsign/mgo v0.0.0-20180905125535-1ca0a4f7cbcb/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= +github.com/go-errors/errors v1.0.1 h1:LUHzmkK3GUKUrL/1gfBUxAHzcev3apQlezX/+O7ma6w= github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= @@ -364,7 +365,6 @@ golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190812203447-cdfb69ac37fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190909003024-a7b16738d86b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191004110552-13f9640d40b9 h1:rjwSpXsdiK0dV8/Naq3kAw9ymfAeJIyd0upUIElB+lI= golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -504,7 +504,6 @@ k8s.io/kubectl v0.0.0-20191219154910-1528d4eea6dd h1:nZX5+wEqTu/EBIYjrZlFOA63z4+ k8s.io/kubectl v0.0.0-20191219154910-1528d4eea6dd/go.mod h1:9ehGcuUGjXVZh0qbYSB0vvofQw2JQe6c6cO0k4wu/Oo= k8s.io/metrics v0.0.0-20191214191643-6b1944c9f765/go.mod h1:5V7rewilItwK0cz4nomU0b3XCcees2Ka5EBYWS1HBeM= k8s.io/utils v0.0.0-20190801114015-581e00157fb1/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= -k8s.io/utils v0.0.0-20191030222137-2b95a09bc58d/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= k8s.io/utils v0.0.0-20191114184206-e782cd3c129f h1:GiPwtSzdP43eI1hpPCbROQCCIgCuiMMNF8YUVLF3vJo= k8s.io/utils v0.0.0-20191114184206-e782cd3c129f/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= modernc.org/cc v1.0.0/go.mod h1:1Sk4//wdnYJiUIxnW8ddKpaOJCF37yAdqYnkxUpaYxw= diff --git a/cmd/kubectl/kubectlcobra/applier.go b/cmd/kubectl/kubectlcobra/applier.go new file mode 100644 index 000000000..b67258053 --- /dev/null +++ b/cmd/kubectl/kubectlcobra/applier.go @@ -0,0 +1,205 @@ +// Copyright 2019 The Kubernetes Authors. +// SPDX-License-Identifier: Apache-2.0 + +package kubectlcobra + +import ( + "context" + "time" + + "github.com/go-errors/errors" + "github.com/spf13/cobra" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/cli-runtime/pkg/genericclioptions" + "k8s.io/cli-runtime/pkg/resource" + "k8s.io/kubectl/pkg/cmd/apply" + "k8s.io/kubectl/pkg/cmd/util" + "k8s.io/kubectl/pkg/scheme" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/kustomize/kstatus/wait" +) + +// newApplier returns a new Applier. It will set up the applyOptions and +// statusOptions which are responsible for capturing any command line flags. +// It currently requires IOStreams, but this is a legacy from when +// the ApplyOptions were responsible for printing progress. This is now +// handled by a separate printer with the KubectlPrinterAdapter bridging +// between the two. +func newApplier(factory util.Factory, ioStreams genericclioptions.IOStreams) *Applier { + return &Applier{ + applyOptions: apply.NewApplyOptions(ioStreams), + statusOptions: NewStatusOptions(), + factory: factory, + ioStreams: ioStreams, + } +} + +// resolver defines the interface the applier needs to observe status for resources. +type resolver interface { + WaitForStatusOfObjects(ctx context.Context, objects []wait.KubernetesObject) <-chan wait.Event +} + +// Applier performs the step of applying a set of resources into a cluster, +// conditionally waits for all of them to be fully reconciled and finally +// performs prune to clean up any resources that has been deleted. +type Applier struct { + factory util.Factory + ioStreams genericclioptions.IOStreams + + applyOptions *apply.ApplyOptions + statusOptions *StatusOptions + resolver resolver +} + +// Initialize sets up the Applier for actually doing an apply against +// a cluster. This involves validating command line inputs and configuring +// clients for communicating with the cluster. +func (a *Applier) Initialize(cmd *cobra.Command) error { + a.applyOptions.PreProcessorFn = PrependGroupingObject(a.applyOptions) + err := a.applyOptions.Complete(a.factory, cmd) + if err != nil { + return errors.WrapPrefix(err, "error setting up ApplyOptions", 1) + } + // Default PostProcessor is configured in "Complete" function, + // so the prune must happen after "Complete". + a.applyOptions.PostProcessorFn = prune(a.factory, a.applyOptions) + + resolver, err := a.newResolver(a.statusOptions.period) + if err != nil { + return errors.WrapPrefix(err, "error creating resolver", 1) + } + a.resolver = resolver + return nil +} + +// SetFlags configures the command line flags needed for apply and +// status. This is a temporary solution as we should separate the configuration +// of cobra flags from the Applier. +func (a *Applier) SetFlags(cmd *cobra.Command) { + a.applyOptions.DeleteFlags.AddFlags(cmd) + a.applyOptions.RecordFlags.AddFlags(cmd) + a.applyOptions.PrintFlags.AddFlags(cmd) + a.statusOptions.AddFlags(cmd) + a.applyOptions.Overwrite = true +} + +// newResolver sets up a new Resolver for computing status. The configuration +// needed for the resolver is taken from the Factory. +func (a *Applier) newResolver(pollInterval time.Duration) (*wait.Resolver, error) { + config, err := a.factory.ToRESTConfig() + if err != nil { + return nil, errors.WrapPrefix(err, "error getting RESTConfig", 1) + } + + mapper, err := a.factory.ToRESTMapper() + if err != nil { + return nil, errors.WrapPrefix(err, "error getting RESTMapper", 1) + } + + c, err := client.New(config, client.Options{Scheme: scheme.Scheme, Mapper: mapper}) + if err != nil { + return nil, errors.WrapPrefix(err, "error creating client", 1) + } + + return wait.NewResolver(c, mapper, pollInterval), nil +} + +// Run performs the Apply step. This happens asynchronously with updates +// on progress and any errors are reported back on the event channel. +// Cancelling the operation or setting timeout on how long to wait +// for it complete can be done with the passed in context. +// Note: There sn't currently any way to interrupt the operation +// before all the given resources have been applied to the cluster. Any +// cancellation or timeout will only affect how long we wait for the +// resources to become current. +func (a *Applier) Run(ctx context.Context) <-chan Event { + ch := make(chan Event) + + go func() { + defer close(ch) + adapter := &KubectlPrinterAdapter{ + ch: ch, + } + // The adapter is used to intercept what is meant to be printing + // in the ApplyOptions, and instead turn those into events. + a.applyOptions.ToPrinter = adapter.toPrinterFunc() + // This provides us with a slice of all the objects that will be + // applied to the cluster. + infos, _ := a.applyOptions.GetObjects() + err := a.applyOptions.Run() + if err != nil { + // If we see an error here we just report it on the channel and then + // give up. Eventually we might be able to determine which errors + // are fatal and which might allow us to continue. + ch <- Event{ + EventType: ErrorEventType, + ErrorEvent: ErrorEvent{ + Err: errors.WrapPrefix(err, "error applying resources", 1), + }, + } + return + } + + if a.statusOptions.wait { + statusChannel := a.resolver.WaitForStatusOfObjects(ctx, infosToObjects(infos)) + // As long as the statusChannel remains open, we take every statusEvent, + // wrap it in an Event and send it on the channel. + for statusEvent := range statusChannel { + ch <- Event{ + EventType: StatusEventType, + StatusEvent: statusEvent, + } + } + } + + }() + return ch +} + +func infosToObjects(infos []*resource.Info) []wait.KubernetesObject { + var objects []wait.KubernetesObject + for _, info := range infos { + u := info.Object.(*unstructured.Unstructured) + objects = append(objects, u) + } + return objects +} + +// EventType determines the type of events that are available. +type EventType string + +const ( + ErrorEventType EventType = "error" + ApplyEventType EventType = "apply" + StatusEventType EventType = "status" +) + +// Event is the type of the objects that will be returned through +// the channel that is returned from a call to Run. It contains +// information about progress and errors encountered during +// the process of doing apply, waiting for status and doing a prune. +type Event struct { + // EventType is the type of event. + EventType EventType + + // ErrorEvent contains information about any errors encountered. + ErrorEvent ErrorEvent + + // ApplyEvent contains information about progress pertaining to + // applying a resource to the cluster. + ApplyEvent ApplyEvent + + // StatusEvents contains information about the status of one of + // the applied resources. + StatusEvent wait.Event +} + +type ErrorEvent struct { + Err error +} + +type ApplyEvent struct { + Operation string + Object runtime.Object +} diff --git a/cmd/kubectl/kubectlcobra/applier_test.go b/cmd/kubectl/kubectlcobra/applier_test.go new file mode 100644 index 000000000..0e3dae046 --- /dev/null +++ b/cmd/kubectl/kubectlcobra/applier_test.go @@ -0,0 +1,48 @@ +// Copyright 2019 The Kubernetes Authors. +// SPDX-License-Identifier: Apache-2.0 + +package kubectlcobra + +import ( + "context" + "github.com/stretchr/testify/assert" + "k8s.io/cli-runtime/pkg/genericclioptions" + cmdtesting "k8s.io/kubectl/pkg/cmd/testing" + "testing" +) + +// The applier is currently hard to test, as the dependencies on the ApplyOptions and +// the resolver are hard to stub out. As we work to better separate the different +// responsibilities of the apply functionality, we should also make it easier to test. +// This provides some basic tests for now. + +func TestApplierWithUnknownFile(t *testing.T) { + tf := cmdtesting.NewTestFactory() + defer tf.Cleanup() + iostreams, _, _, _ := genericclioptions.NewTestIOStreams() + cmd := NewCmdApply("base", tf, iostreams) + + applier := newApplier(tf, iostreams) + filenames := []string{"file.yaml"} + applier.applyOptions.DeleteFlags.FileNameFlags.Filenames = &filenames + + err := applier.Initialize(cmd) + assert.NoError(t, err) + + ch := applier.Run(context.TODO()) + + var events []Event + for msg := range ch { + events = append(events, msg) + } + + if !assert.Equal(t, 1, len(events)) { + return + } + + event := events[0] + if !assert.Equal(t, ErrorEventType, event.EventType) { + return + } + assert.Contains(t, event.ErrorEvent.Err.Error(), "does not exist") +} diff --git a/cmd/kubectl/kubectlcobra/basic_printer.go b/cmd/kubectl/kubectlcobra/basic_printer.go new file mode 100644 index 000000000..96db9755d --- /dev/null +++ b/cmd/kubectl/kubectlcobra/basic_printer.go @@ -0,0 +1,62 @@ +// Copyright 2019 The Kubernetes Authors. +// SPDX-License-Identifier: Apache-2.0 + +package kubectlcobra + +import ( + "fmt" + "k8s.io/apimachinery/pkg/runtime/schema" + "strings" + + "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/cli-runtime/pkg/genericclioptions" + cmdutil "k8s.io/kubectl/pkg/cmd/util" + "sigs.k8s.io/kustomize/kstatus/wait" +) + +// BasicPrinter is a simple implementation that just prints the events +// from the channel in the default format for kubectl. +// We need to support different printers for different output formats. +type BasicPrinter struct { + ioStreams genericclioptions.IOStreams +} + +// Print outputs the events from the provided channel in a simple +// format on StdOut. As we support other printer implementations +// this should probably be an interface. +// This function will block until the channel is closed. +func (b *BasicPrinter) Print(ch <-chan Event) { + for event := range ch { + switch event.EventType { + case ErrorEventType: + cmdutil.CheckErr(event.ErrorEvent.Err) + case ApplyEventType: + obj := event.ApplyEvent.Object + gvk := obj.GetObjectKind().GroupVersionKind() + name := "" + if acc, err := meta.Accessor(obj); err == nil { + if n := acc.GetName(); len(n) > 0 { + name = n + } + } + fmt.Fprintf(b.ioStreams.Out, "%s %s\n", resourceIdToString(gvk.GroupKind(), name), event.ApplyEvent.Operation) + case StatusEventType: + statusEvent := event.StatusEvent + switch statusEvent.Type { + case wait.ResourceUpdate: + id := statusEvent.EventResource.ResourceIdentifier + gk := id.GroupKind + fmt.Fprintf(b.ioStreams.Out, "%s is %s: %s\n", resourceIdToString(gk, id.Name), statusEvent.EventResource.Status.String(), statusEvent.EventResource.Message) + case wait.Completed: + fmt.Fprint(b.ioStreams.Out, "all resources has reached the Current status\n") + case wait.Aborted: + fmt.Fprintf(b.ioStreams.Out, "resources failed to the reached Current status\n") + } + } + } +} + +// resourceIdToString returns the string representation of a GroupKind and a resource name. +func resourceIdToString(gk schema.GroupKind, name string) string { + return fmt.Sprintf("%s/%s", strings.ToLower(gk.String()), name) +} diff --git a/cmd/kubectl/kubectlcobra/commands.go b/cmd/kubectl/kubectlcobra/commands.go index 8fe5146f8..d4da23e1a 100644 --- a/cmd/kubectl/kubectlcobra/commands.go +++ b/cmd/kubectl/kubectlcobra/commands.go @@ -5,6 +5,7 @@ package kubectlcobra import ( + "context" "flag" "fmt" "os" @@ -79,10 +80,10 @@ func updateHelp(names []string, c *cobra.Command) { // NewCmdApply creates the `apply` command func NewCmdApply(baseName string, f util.Factory, ioStreams genericclioptions.IOStreams) *cobra.Command { - o := apply.NewApplyOptions(ioStreams) - so := newStatusOptions(f, ioStreams) - // Set up grouping object for this apply; used in subsequent prune. - o.PreProcessorFn = PrependGroupingObject(o) + applier := newApplier(f, ioStreams) + printer := &BasicPrinter{ + ioStreams: ioStreams, + } cmd := &cobra.Command{ Use: "apply (-f FILENAME | -k DIRECTORY)", @@ -94,31 +95,28 @@ func NewCmdApply(baseName string, f util.Factory, ioStreams genericclioptions.IO Run: func(cmd *cobra.Command, args []string) { if len(args) > 0 { // check is kustomize, if so update - o.DeleteFlags.FileNameFlags.Kustomize = &args[0] + applier.applyOptions.DeleteFlags.FileNameFlags.Kustomize = &args[0] } - cmdutil.CheckErr(o.Complete(f, cmd)) - // Default PostProcessor is configured in "Complete" function, - // so the prune must happen after "Complete". - o.PostProcessorFn = prune(f, o) - cmdutil.CheckErr(o.Run()) - infos, _ := o.GetObjects() - if so.wait { - cmdutil.CheckErr(so.waitForStatus(infos)) - } + cmdutil.CheckErr(applier.Initialize(cmd)) + + // Create a context with the provided timout from the cobra parameter. + ctx, cancel := context.WithTimeout(context.Background(), applier.statusOptions.timeout) + defer cancel() + // Run the applier. It will return a channel where we can receive updates + // to keep track of progress and any issues. + ch := applier.Run(ctx) + + // The printer will print updates from the channel. It will block + // until the channel is closed. + printer.Print(ch) }, } - // bind flag structs - o.DeleteFlags.AddFlags(cmd) - o.RecordFlags.AddFlags(cmd) - o.PrintFlags.AddFlags(cmd) - so.AddFlags(cmd) - - o.Overwrite = true + applier.SetFlags(cmd) cmdutil.AddValidateFlags(cmd) - cmd.Flags().BoolVar(&o.ServerDryRun, "server-dry-run", o.ServerDryRun, "If true, request will be sent to server with dry-run flag, which means the modifications won't be persisted. This is an alpha feature and flag.") + cmd.Flags().BoolVar(&applier.applyOptions.ServerDryRun, "server-dry-run", applier.applyOptions.ServerDryRun, "If true, request will be sent to server with dry-run flag, which means the modifications won't be persisted. This is an alpha feature and flag.") cmd.Flags().Bool("dry-run", false, "If true, only print the object that would be sent, without sending it. Warning: --dry-run cannot accurately output the result of merging the local manifest and the server-side data. Use --server-dry-run to get the merged result instead.") cmdutil.AddServerSideApplyFlags(cmd) diff --git a/cmd/kubectl/kubectlcobra/printer_adapter.go b/cmd/kubectl/kubectlcobra/printer_adapter.go new file mode 100644 index 000000000..0836dad74 --- /dev/null +++ b/cmd/kubectl/kubectlcobra/printer_adapter.go @@ -0,0 +1,53 @@ +// Copyright 2019 The Kubernetes Authors. +// SPDX-License-Identifier: Apache-2.0 + +package kubectlcobra + +import ( + "io" + + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/cli-runtime/pkg/printers" +) + +// KubectlPrinterAdapter is a workaround for capturing progress from +// ApplyOptions. ApplyOptions were originally meant to print progress +// directly using a configurable printer. The KubectlPrinterAdapter +// plugs into ApplyOptions as a ToPrinter function, but instead of +// printing the info, it emits it as an event on the provided channel. +type KubectlPrinterAdapter struct { + ch chan<- Event +} + +// resourcePrinterImpl implements the ResourcePrinter interface. But +// instead of printing, it emits information on the provided channel. +type resourcePrinterImpl struct { + operation string + ch chan<- Event +} + +// PrintObj takes the provided object and operation and emits +// it on the channel. +func (r *resourcePrinterImpl) PrintObj(obj runtime.Object, _ io.Writer) error { + r.ch <- Event{ + EventType: ApplyEventType, + ApplyEvent: ApplyEvent{ + Operation: r.operation, + Object: obj, + }, + } + return nil +} + +type toPrinterFunc func(string) (printers.ResourcePrinter, error) + +// toPrinterFunc returns a function of type toPrinterFunc. This +// is the type required by the ApplyOptions. +func (p *KubectlPrinterAdapter) toPrinterFunc() toPrinterFunc { + return func(operation string) (printers.ResourcePrinter, error) { + return &resourcePrinterImpl{ + ch: p.ch, + operation: operation, + }, nil + } +} diff --git a/cmd/kubectl/kubectlcobra/printer_adapter_test.go b/cmd/kubectl/kubectlcobra/printer_adapter_test.go new file mode 100644 index 000000000..b7375b5a0 --- /dev/null +++ b/cmd/kubectl/kubectlcobra/printer_adapter_test.go @@ -0,0 +1,49 @@ +// Copyright 2019 The Kubernetes Authors. +// SPDX-License-Identifier: Apache-2.0 + +package kubectlcobra + +import ( + "bytes" + "testing" + + "github.com/stretchr/testify/assert" + appsv1 "k8s.io/api/apps/v1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestKubectlPrinterAdapter(t *testing.T) { + ch := make(chan Event) + buffer := bytes.Buffer{} + operation := "operation" + + adapter := KubectlPrinterAdapter{ + ch: ch, + } + + toPrinterFunc := adapter.toPrinterFunc() + resourcePrinter, err := toPrinterFunc(operation) + assert.NoError(t, err) + + deployment := appsv1.Deployment{ + TypeMeta: v1.TypeMeta{ + APIVersion: "apps/v1", + Kind: "Deployment", + }, + ObjectMeta: v1.ObjectMeta{ + Name: "name", + Namespace: "namespace", + }, + } + + // Need to run this in a separate gorutine since go channels + // are blocking. + go func() { + err = resourcePrinter.PrintObj(&deployment, &buffer) + }() + msg := <-ch + + assert.NoError(t, err) + assert.Equal(t, operation, msg.ApplyEvent.Operation) + assert.Equal(t, &deployment, msg.ApplyEvent.Object) +} diff --git a/cmd/kubectl/kubectlcobra/status.go b/cmd/kubectl/kubectlcobra/status.go index 3db008362..d69393fd9 100644 --- a/cmd/kubectl/kubectlcobra/status.go +++ b/cmd/kubectl/kubectlcobra/status.go @@ -4,102 +4,27 @@ package kubectlcobra import ( - "context" - "fmt" - "strings" "time" "github.com/spf13/cobra" - "k8s.io/apimachinery/pkg/api/meta" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/cli-runtime/pkg/genericclioptions" - "k8s.io/cli-runtime/pkg/resource" - "k8s.io/kubectl/pkg/cmd/util" - "k8s.io/kubectl/pkg/scheme" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/kustomize/kstatus/wait" ) -type StatusOptions struct { - factory util.Factory - ioStreams genericclioptions.IOStreams +func NewStatusOptions() *StatusOptions { + return &StatusOptions{ + wait: false, + period: 2 * time.Second, + timeout: time.Minute, + } +} +type StatusOptions struct { wait bool period time.Duration timeout time.Duration } -func newStatusOptions(factory util.Factory, ioStreams genericclioptions.IOStreams) *StatusOptions { - return &StatusOptions{ - factory: factory, - ioStreams: ioStreams, - - wait: false, - period: 2 * time.Second, - timeout: 1 * time.Minute, - } -} - func (s *StatusOptions) AddFlags(c *cobra.Command) { c.Flags().BoolVar(&s.wait, "status", s.wait, "Wait for all applied resources to reach the Current status.") c.Flags().DurationVar(&s.period, "status-period", s.period, "Polling period for resource statuses.") c.Flags().DurationVar(&s.timeout, "status-timeout", s.timeout, "Timeout threshold for waiting for all resources to reach the Current status.") } - -func (s *StatusOptions) waitForStatus(infos []*resource.Info) error { - mapper, err := getRESTMapper(s.factory) - if err != nil { - return err - } - - c, err := getClient(s.factory, mapper) - if err != nil { - return err - } - - ctx, cancel := context.WithTimeout(context.Background(), s.timeout) - defer cancel() - - resolver := wait.NewResolver(c, mapper, s.period) - ch := resolver.WaitForStatus(ctx, infosToResourceIdentifiers(infos)) - - for msg := range ch { - switch msg.Type { - case wait.ResourceUpdate: - id := msg.EventResource.ResourceIdentifier - gk := id.GroupKind - fmt.Fprintf(s.ioStreams.Out, "%s/%s is %s: %s\n", strings.ToLower(gk.String()), id.Name, msg.EventResource.Status.String(), msg.EventResource.Message) - case wait.Completed: - fmt.Fprint(s.ioStreams.Out, "all resources has reached the Current status\n") - case wait.Aborted: - fmt.Fprintf(s.ioStreams.Out, "resources failed to the reached Current status after %s\n", s.timeout.String()) - } - } - return nil -} - -func infosToResourceIdentifiers(infos []*resource.Info) []wait.ResourceIdentifier { - var resources []wait.ResourceIdentifier - for _, info := range infos { - u := info.Object.(*unstructured.Unstructured) - resources = append(resources, wait.ResourceIdentifier{ - GroupKind: u.GroupVersionKind().GroupKind(), - Namespace: u.GetNamespace(), - Name: u.GetName(), - }) - } - return resources -} - -func getRESTMapper(f util.Factory) (meta.RESTMapper, error) { - return f.ToRESTMapper() -} - -func getClient(f util.Factory, mapper meta.RESTMapper) (client.Reader, error) { - config, err := f.ToRESTConfig() - if err != nil { - return nil, err - } - - return client.New(config, client.Options{Scheme: scheme.Scheme, Mapper: mapper}) -} From 7372a371b404bb0bd4bb070ad77cc80f2fe23b03 Mon Sep 17 00:00:00 2001 From: Frank Farzan Date: Fri, 31 Jan 2020 14:41:56 -0800 Subject: [PATCH 27/46] kyaml: Handle functions in top-level directory. scope() method uses HasPrefix to detect whether a resource is in a subdirectory of function directory. It doesn't handle the case when the function is in top-level dir ('.'). --- kyaml/kio/filters/container.go | 4 +- kyaml/kio/filters/container_test.go | 96 +++++++++++++++++++++++++++++ 2 files changed, 98 insertions(+), 2 deletions(-) diff --git a/kyaml/kio/filters/container.go b/kyaml/kio/filters/container.go index ca84e027f..0ef509078 100644 --- a/kyaml/kio/filters/container.go +++ b/kyaml/kio/filters/container.go @@ -209,8 +209,8 @@ func (c *ContainerFilter) scope(dir string, nodes []*yaml.RNode) ([]*yaml.RNode, return nodes, nil, nil } - if dir == "" { - // global function + // global function + if dir == "" || dir == "." { return nodes, nil, nil } diff --git a/kyaml/kio/filters/container_test.go b/kyaml/kio/filters/container_test.go index c8152f128..f0faea2d3 100644 --- a/kyaml/kio/filters/container_test.go +++ b/kyaml/kio/filters/container_test.go @@ -773,6 +773,102 @@ metadata: `, b.String()) } +func TestFilter_Filter_scope_nested_resource(t *testing.T) { + // functions under "functions/" dir should be scoped to parent dir + cfg, err := yaml.Parse(`apiVersion: apps/v1 +kind: Deployment +metadata: + name: foo + annotations: + config.kubernetes.io/path: 'baz.yaml' +`) + if !assert.NoError(t, err) { + return + } + + input, err := (&kio.ByteReader{Reader: bytes.NewBufferString(` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: deployment-foo + annotations: + config.kubernetes.io/path: 'foo/bar/d.yaml' +--- +apiVersion: v1 +kind: Service +metadata: + name: service-foo + annotations: + config.kubernetes.io/path: 'foo/bar/s.yaml' +`)}).Read() + if !assert.NoError(t, err) { + return + } + + // no resources match the scope + called := false + result, err := (&ContainerFilter{ + Image: "example.com:version", + Config: cfg, + args: []string{"sed", "s/Deployment/StatefulSet/g"}, + checkInput: func(s string) { + called = true + if !assert.Equal(t, `apiVersion: config.kubernetes.io/v1alpha1 +kind: ResourceList +items: +- apiVersion: apps/v1 + kind: Deployment + metadata: + name: deployment-foo + annotations: + config.kubernetes.io/path: 'foo/bar/d.yaml' + config.kubernetes.io/index: '0' +- apiVersion: v1 + kind: Service + metadata: + name: service-foo + annotations: + config.kubernetes.io/path: 'foo/bar/s.yaml' + config.kubernetes.io/index: '1' +functionConfig: {apiVersion: apps/v1, kind: Deployment, metadata: {name: foo, annotations: { + config.kubernetes.io/path: 'baz.yaml'}}} +`, s) { + t.FailNow() + } + }, + }).Filter(input) + if !assert.NoError(t, err) { + return + } + if !assert.True(t, called) { + return + } + + b := &bytes.Buffer{} + err = kio.ByteWriter{Writer: b, KeepReaderAnnotations: true}.Write(result) + if !assert.NoError(t, err) { + return + } + + // Resources should be modified + assert.Equal(t, `apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: deployment-foo + annotations: + config.kubernetes.io/path: 'foo/bar/d.yaml' + config.kubernetes.io/index: '0' +--- +apiVersion: v1 +kind: Service +metadata: + name: service-foo + annotations: + config.kubernetes.io/path: 'foo/bar/s.yaml' + config.kubernetes.io/index: '1' +`, b.String()) +} + func TestFilter_Filter_scopeDir(t *testing.T) { // functions under "functions/" dir should be scoped to parent dir cfg, err := yaml.Parse(`apiVersion: apps/v1 From 74e1b5d54b9a1a9fd9887c52b2dbd16f45d50741 Mon Sep 17 00:00:00 2001 From: Haiyan Meng Date: Tue, 28 Jan 2020 10:28:03 -0800 Subject: [PATCH 28/46] Add GCP service account into ESCluster config This is necessary for index backup into GCS and index recovery from GCS --- api/internal/crawl/config/elastic/escluster.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/api/internal/crawl/config/elastic/escluster.yaml b/api/internal/crawl/config/elastic/escluster.yaml index 47e1f5458..c18a9f729 100644 --- a/api/internal/crawl/config/elastic/escluster.yaml +++ b/api/internal/crawl/config/elastic/escluster.yaml @@ -8,6 +8,8 @@ spec: - repository-gcs - ingest-user-agent - ingest-geoip + gcpserviceaccount: + name: kustomizesa config: env: example: test From bb409a5ea84d7c56cc17fe601357bb58d429cfa4 Mon Sep 17 00:00:00 2001 From: Haiyan Meng Date: Tue, 28 Jan 2020 10:30:21 -0800 Subject: [PATCH 29/46] Set up cronjob to run crawler every 7 days --- api/internal/crawl/config/crawler/cronjob/cronjob.yaml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/api/internal/crawl/config/crawler/cronjob/cronjob.yaml b/api/internal/crawl/config/crawler/cronjob/cronjob.yaml index ab333c782..2a9433252 100644 --- a/api/internal/crawl/config/crawler/cronjob/cronjob.yaml +++ b/api/internal/crawl/config/crawler/cronjob/cronjob.yaml @@ -1,9 +1,10 @@ apiVersion: batch/v1beta1 kind: CronJob metadata: - name: crawler + name: crawler-cronjob spec: - schedule: "5 0 * * */1" + # run the cronjob at 00:00 every 7 days + schedule: "0 0 */7 * *" jobTemplate: spec: template: @@ -11,7 +12,9 @@ spec: restartPolicy: OnFailure containers: - name: crawler - image: gcr.io/kustomize-search/crawler:latest + image: gcr.io/haiyanmeng-gke-dev/crawler:v1 + command: ["/crawler"] + args: ["--mode=index+github", "--github-repo=kubernetes-sigs/kustomize", "--index=kustomize"] imagePullPolicy: Always env: - name: GITHUB_ACCESS_TOKEN From b35b5aa73df61279aadc98e032a0de1958bbfef3 Mon Sep 17 00:00:00 2001 From: Haiyan Meng Date: Tue, 28 Jan 2020 10:59:38 -0800 Subject: [PATCH 30/46] Check the checksums of documents in the index --- api/internal/crawl/cmd/kustomize_stats/main.go | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/api/internal/crawl/cmd/kustomize_stats/main.go b/api/internal/crawl/cmd/kustomize_stats/main.go index 103405079..00c256d65 100644 --- a/api/internal/crawl/cmd/kustomize_stats/main.go +++ b/api/internal/crawl/cmd/kustomize_stats/main.go @@ -2,6 +2,7 @@ package main import ( "context" + "crypto/sha256" "flag" "fmt" "log" @@ -210,11 +211,20 @@ If you only want to list the 10 most popular features, set the flag to 10.`) // transformersDocs includes all the docs using transformers transformersDocs := make([]*doc.Document, 0) + checksums := make(map[string]int) + // get all the documents in the index query := []byte(`{ "query":{ "match_all":{} } }`) it := idx.IterateQuery(query, 10000, 60*time.Second) for it.Next() { for _, hit := range it.Value().Hits.Hits { + sum := fmt.Sprintf("%x", sha256.Sum256([]byte(hit.Document.DocumentData))) + if _, ok := checksums[sum]; ok { + checksums[sum]++ + } else { + checksums[sum] = 1 + } + // check whether there is any duplicate IDs in the index if _, ok := ids[hit.ID]; !ok { ids[hit.ID] = struct{}{} @@ -282,4 +292,12 @@ There are %d documents in the kustomize index. GeneratorOrTransformerStats(ctx, generatorDocs, true, idx) GeneratorOrTransformerStats(ctx, transformersDocs, false, idx) + + fmt.Printf("There are total %d checksums of document contents\n", len(checksums)) + sortedChecksums := SortMapKeyByValue(checksums) + sortedChecksums = sortedChecksums[:20] + fmt.Printf("The top 20 checksums are:\n") + for _, key := range sortedChecksums { + fmt.Printf("checksum %s apprears %d\n", key, checksums[key]) + } } From d5c66cb3d4d8d82dbf5fad1af9f897f242e4eb1f Mon Sep 17 00:00:00 2001 From: Haiyan Meng Date: Tue, 28 Jan 2020 14:13:14 -0800 Subject: [PATCH 31/46] Add KustomizationDocument.Copy method --- api/internal/crawl/cmd/crawler/crawler.go | 2 +- api/internal/crawl/doc/doc.go | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/api/internal/crawl/cmd/crawler/crawler.go b/api/internal/crawl/cmd/crawler/crawler.go index e534780d2..d87c47996 100644 --- a/api/internal/crawl/cmd/crawler/crawler.go +++ b/api/internal/crawl/cmd/crawler/crawler.go @@ -167,7 +167,7 @@ func main() { it := idx.IterateQuery(query, 10000, 60*time.Second) for it.Next() { for _, hit := range it.Value().Hits.Hits { - seedDocs = append(seedDocs, hit.Document.Copy()) + seedDocs = append(seedDocs, hit.Document.Document.Copy()) } } if err := it.Err(); err != nil { diff --git a/api/internal/crawl/doc/doc.go b/api/internal/crawl/doc/doc.go index 0e5965ac1..d37fd19ca 100644 --- a/api/internal/crawl/doc/doc.go +++ b/api/internal/crawl/doc/doc.go @@ -46,6 +46,15 @@ type KustomizationDocument struct { type set map[string]struct{} +func (doc *KustomizationDocument) Copy() *KustomizationDocument { + return &KustomizationDocument{ + Document: *(doc.Document.Copy()), + Kinds: doc.Kinds, + Identifiers: doc.Identifiers, + Values: doc.Values, + } +} + func (doc *KustomizationDocument) String() string { return fmt.Sprintf("%s %s %s %v %v %v len(identifiers):%v len(values):%v", doc.RepositoryURL, doc.FilePath, doc.DefaultBranch, doc.CreationTime, From 0b38e6d284d3b33bec9b12292103ea71f5330234 Mon Sep 17 00:00:00 2001 From: Haiyan Meng Date: Tue, 28 Jan 2020 15:24:36 -0800 Subject: [PATCH 32/46] Improve the analysis on generator and transformer --- .../crawl/cmd/kustomize_stats/main.go | 179 +++++++----------- 1 file changed, 65 insertions(+), 114 deletions(-) diff --git a/api/internal/crawl/cmd/kustomize_stats/main.go b/api/internal/crawl/cmd/kustomize_stats/main.go index 00c256d65..b2c3d5a04 100644 --- a/api/internal/crawl/cmd/kustomize_stats/main.go +++ b/api/internal/crawl/cmd/kustomize_stats/main.go @@ -6,13 +6,9 @@ import ( "flag" "fmt" "log" - "net/http" - "os" "sort" "time" - "sigs.k8s.io/kustomize/api/internal/crawl/crawler/github" - "sigs.k8s.io/kustomize/api/internal/crawl/doc" "sigs.k8s.io/kustomize/api/internal/crawl/index" @@ -35,9 +31,9 @@ func iterateArr(arr []string, countMap map[string]int) { } -// SortMapKeyByValue takes a map as its input, sorts its keys according to their values +// SortMapKeyByValueInt takes a map as its input, sorts its keys according to their values // in the map, and outputs the sorted keys as a slice. -func SortMapKeyByValue(m map[string]int) []string { +func SortMapKeyByValueInt(m map[string]int) []string { keys := make([]string, 0, len(m)) for key := range m { keys = append(keys, key) @@ -47,56 +43,58 @@ func SortMapKeyByValue(m map[string]int) []string { return keys } -func GeneratorOrTransformerStats(ctx context.Context, - docs []*doc.Document, isGenerator bool, idx *index.KustomizeIndex) { +// SortMapKeyByValue takes a map as its input, sorts its keys according to their values +// in the map, and outputs the sorted keys as a slice. +func SortMapKeyByValueLen(m map[string][]string) []string { + keys := make([]string, 0, len(m)) + for key := range m { + keys = append(keys, key) + } + // sort keys according to their values in the map m + sort.Slice(keys, func(i, j int) bool { return len(m[keys[i]]) > len(m[keys[j]]) }) + return keys +} - fieldName := "generators" - if !isGenerator { - fieldName = "transformers" +func GeneratorOrTransformerStats(docs []*doc.KustomizationDocument) { + n := len(docs) + if n == 0 { + return } - // allReferredDocs includes all the documents referred in the field - allReferredDocs := doc.NewUniqueDocuments() + fileType := docs[0].FileType + fmt.Printf("There are totally %d %s files.\n", n, fileType) - // docUsingGeneratorCount counts the number of the kustomization files using generators or transformers - docCount := 0 + GitRepositorySummary(docs, fileType) + + // key of kindToUrls: a string in the KustomizationDocument.Kinds field + // value of kindToUrls: a slice of string urls defining a given kind. + kindToUrls := make(map[string][]string) - // collect all the documents referred in the field for _, d := range docs { - kdoc := doc.KustomizationDocument{ - Document: *d, - } - referredDocs, err := kdoc.GetResources(false, !isGenerator, isGenerator) - if err != nil { - log.Printf("failed to parse the %s field of the Document (%s): %v", - fieldName, d.Path(), err) - } - if len(referredDocs) > 0 { - docCount++ - allReferredDocs.AddDocuments(referredDocs) + url := fmt.Sprintf("%s/blob/%s/%s", d.RepositoryURL, d.DefaultBranch, d.FilePath) + for _, kind := range d.Kinds { + if _, ok := kindToUrls[kind]; !ok { + kindToUrls[kind] = []string{url} + } else { + kindToUrls[kind] = append(kindToUrls[kind], url) + } + } + } + fmt.Printf("There are totally %d kinds of %s\n", len(kindToUrls), fileType) + sortedKeys := SortMapKeyByValueLen(kindToUrls) + for _, k := range sortedKeys { + sort.Strings(kindToUrls[k]) + fmt.Printf("%s kind %s appears %d times\n", fileType, k, len(kindToUrls[k])) + for _, url := range kindToUrls[k] { + fmt.Printf("%s\n", url) } } - - fileCount, dirCount, fileTypeDocs, dirTypeDocs := DocumentTypeSummary(ctx, allReferredDocs.Documents()) - - // check whether any of the files are not in the index - nonExistFileCount := ExistInIndex(idx, fileTypeDocs, fieldName + " file ") - // check whether any of the dirs are not in the index - nonExistDirCount := ExistInIndex(idx, dirTypeDocs, fieldName + " dir ") - - GitRepositorySummary(fileTypeDocs, fieldName + " files") - GitRepositorySummary(dirTypeDocs, fieldName + " dirs") - - fmt.Printf("%d kustomization files use %s: %d %s are files and %d %s are dirs.\n", - docCount, fieldName, fileCount, fieldName, dirCount, fieldName) - fmt.Printf("%d %s files do not exist in the index\n", nonExistFileCount, fieldName) - fmt.Printf("%d %s dirs do not exist in the index\n", nonExistDirCount, fieldName) } // GitRepositorySummary counts the distribution of docs: // 1) how many git repositories are these docs from? // 2) how many docs are from each git repository? -func GitRepositorySummary(docs []*doc.Document, msgPrefix string) { +func GitRepositorySummary(docs []*doc.KustomizationDocument, fileType string) { m := make(map[string]int) for _, d := range docs { if _, ok := m[d.RepositoryURL]; ok { @@ -105,65 +103,16 @@ func GitRepositorySummary(docs []*doc.Document, msgPrefix string) { m[d.RepositoryURL] = 1 } } - sortedKeys := SortMapKeyByValue(m) + sortedKeys := SortMapKeyByValueInt(m) + topN := 10 + i := 0 for _, k := range sortedKeys { - fmt.Printf("%d %s are from %s\n", m[k], msgPrefix, k) - } -} - -// ExistInIndex goes through each Document in docs, and check whether it is in the index or not. -// It returns the number of documents which does not exist in the index. -func ExistInIndex(idx *index.KustomizeIndex, docs []*doc.Document, msgPrefix string) int { - nonExistCount := 0 - for _, d := range docs { - exists, err := idx.Exists(d.ID()) - if err != nil { - log.Println(err) - } - if !exists { - log.Printf("%s (%s) does not exist in the index", msgPrefix, d.Path()) - nonExistCount++ + if i >= topN { + break } + fmt.Printf("%d %s are from %s\n", m[k], fileType, k) + i++ } - return nonExistCount -} - -// DocumentTypeSummary goes through each doc in docs, and determines whether it is a file or dir. -func DocumentTypeSummary(ctx context.Context, docs []*doc.Document) ( - fileCount, dirCount int, files, dirs []*doc.Document) { - githubToken := os.Getenv(githubAccessTokenVar) - if githubToken == "" { - log.Fatalf("Must set the variable '%s' to make github requests.\n", - githubAccessTokenVar) - } - ghCrawler := github.NewCrawler(githubToken, retryCount, &http.Client{}, github.QueryWith()) - - for _, d := range docs { - oldFilePath := d.FilePath - if err := ghCrawler.FetchDocument(ctx, d); err != nil { - log.Printf("FetchDocument failed on %s: %v", d.Path(), err) - continue - } - - if d.FilePath == oldFilePath { - fileCount++ - files = append(files, d) - } else { - dirCount++ - dirs = append(dirs, d) - } - } - return fileCount, dirCount, files, dirs -} - -// ExistInSlice checks where target exits in items. -func ExistInSlice(items []string, target string) bool { - for _, item := range items { - if item == target { - return true - } - } - return false } func main() { @@ -205,11 +154,11 @@ If you only want to list the 10 most popular features, set the flag to 10.`) // ids tracks the unique IDs of the documents in the index ids := make(map[string]struct{}) - // generatorDocs includes all the docs using generators - generatorDocs := make([]*doc.Document, 0) + // generatorFiles include all the non-kustomization files whose FileType is generator + generatorFiles := make([]*doc.KustomizationDocument, 0) - // transformersDocs includes all the docs using transformers - transformersDocs := make([]*doc.Document, 0) + // transformersFiles include all the non-kustomization files whose FileType is transformer + transformersFiles := make([]*doc.KustomizationDocument, 0) checksums := make(map[string]int) @@ -239,11 +188,13 @@ If you only want to list the 10 most popular features, set the flag to 10.`) if doc.IsKustomizationFile(hit.Document.FilePath) { kustomizationFilecount++ iterateArr(hit.Document.Identifiers, kustomizeIdentifiersMap) - if ExistInSlice(hit.Document.Identifiers, "generators") { - generatorDocs = append(generatorDocs, hit.Document.Copy()) - } - if ExistInSlice(hit.Document.Identifiers, "transformers") { - transformersDocs = append(transformersDocs, hit.Document.Copy()) + + } else { + switch hit.Document.FileType { + case "generator": + generatorFiles = append(generatorFiles, hit.Document.Copy()) + case "transformer": + transformersFiles = append(transformersFiles, hit.Document.Copy()) } } } @@ -253,9 +204,9 @@ If you only want to list the 10 most popular features, set the flag to 10.`) log.Fatalf("Error iterating: %v\n", err) } - sortedKindsMapKeys := SortMapKeyByValue(kindsMap) - sortedIdentifiersMapKeys := SortMapKeyByValue(identifiersMap) - sortedKustomizeIdentifiersMapKeys := SortMapKeyByValue(kustomizeIdentifiersMap) + sortedKindsMapKeys := SortMapKeyByValueInt(kindsMap) + sortedIdentifiersMapKeys := SortMapKeyByValueInt(identifiersMap) + sortedKustomizeIdentifiersMapKeys := SortMapKeyByValueInt(kustomizeIdentifiersMap) fmt.Printf(`The count of unique document IDs in the kustomize index: %d There are %d documents in the kustomize index. @@ -290,11 +241,11 @@ There are %d documents in the kustomize index. } } - GeneratorOrTransformerStats(ctx, generatorDocs, true, idx) - GeneratorOrTransformerStats(ctx, transformersDocs, false, idx) + GeneratorOrTransformerStats(generatorFiles) + GeneratorOrTransformerStats(transformersFiles) fmt.Printf("There are total %d checksums of document contents\n", len(checksums)) - sortedChecksums := SortMapKeyByValue(checksums) + sortedChecksums := SortMapKeyByValueInt(checksums) sortedChecksums = sortedChecksums[:20] fmt.Printf("The top 20 checksums are:\n") for _, key := range sortedChecksums { From 0fcb3a014c73bd0a39ca5229dc0895c9d380b1d7 Mon Sep 17 00:00:00 2001 From: Haiyan Meng Date: Tue, 28 Jan 2020 15:57:57 -0800 Subject: [PATCH 33/46] Add config for index backup and restore --- api/internal/crawl/config/elastic/esbackup.yaml | 13 +++++++++++++ api/internal/crawl/config/elastic/esrestore.yaml | 13 +++++++++++++ api/internal/crawl/config/elastic/essnaptshot.yaml | 12 ++++++++++++ 3 files changed, 38 insertions(+) create mode 100644 api/internal/crawl/config/elastic/esbackup.yaml create mode 100644 api/internal/crawl/config/elastic/esrestore.yaml create mode 100644 api/internal/crawl/config/elastic/essnaptshot.yaml diff --git a/api/internal/crawl/config/elastic/esbackup.yaml b/api/internal/crawl/config/elastic/esbackup.yaml new file mode 100644 index 000000000..52ee49efa --- /dev/null +++ b/api/internal/crawl/config/elastic/esbackup.yaml @@ -0,0 +1,13 @@ +apiVersion: elasticsearch.cloud.google.com/v1alpha1 +kind: ESBackup +metadata: + name: kustomize-backup +spec: + storage: + gcs: + bucket: kustomize-backup + path: kustomize + secret: + name: kustomizesa + escluster: + name: esbasic diff --git a/api/internal/crawl/config/elastic/esrestore.yaml b/api/internal/crawl/config/elastic/esrestore.yaml new file mode 100644 index 000000000..9db48795d --- /dev/null +++ b/api/internal/crawl/config/elastic/esrestore.yaml @@ -0,0 +1,13 @@ +apiVersion: elasticsearch.cloud.google.com/v1alpha1 +kind: ESRestore +metadata: + name: kustomize-restore +spec: + include_global_state: true + ignore_unavailable: true + rename_pattern: kustomize + rename_replacement: kustomize-restore1 + essnapshot: + name: kustomize-snapshot + escluster: + name: esbasic diff --git a/api/internal/crawl/config/elastic/essnaptshot.yaml b/api/internal/crawl/config/elastic/essnaptshot.yaml new file mode 100644 index 000000000..f36e94897 --- /dev/null +++ b/api/internal/crawl/config/elastic/essnaptshot.yaml @@ -0,0 +1,12 @@ +apiVersion: elasticsearch.cloud.google.com/v1alpha1 +kind: ESSnapshot +metadata: + name: kustomize-snapshot +spec: + # indices are optional. If not specified all indices are selected. + indices: + - kustomize + include_global_state: true + ignore_unavailable: true + esbackup: + name: kustomize-backup From 7a87c84403d98c88c735f2d46f2a96584e36f0fe Mon Sep 17 00:00:00 2001 From: Haiyan Meng Date: Wed, 29 Jan 2020 11:34:41 -0800 Subject: [PATCH 34/46] Reprocess the github filesize search ranges which have more than 1000 items --- api/internal/crawl/crawler/github/crawler.go | 56 ++++++++++++-- .../crawler/github/split_search_ranges.go | 75 ++++++++++--------- .../github/split_search_ranges_test.go | 24 ++++-- 3 files changed, 104 insertions(+), 51 deletions(-) diff --git a/api/internal/crawl/crawler/github/crawler.go b/api/internal/crawl/crawler/github/crawler.go index 21cfd6f81..23bcbcd72 100644 --- a/api/internal/crawl/crawler/github/crawler.go +++ b/api/internal/crawl/crawler/github/crawler.go @@ -80,6 +80,36 @@ func (gc githubCrawler) DefaultBranch(repo string) string { func (gc githubCrawler) Crawl(ctx context.Context, output chan<- crawler.CrawledDocument, seen utils.SeenMap) error { + ranges := []RangeWithin{ + RangeWithin{ + start: uint64(0), + end: githubMaxFileSize, + }, + } + + errs := make(multiError, 0) + for len(ranges) > 0 { + tailRange := ranges[len(ranges) - 1] + ranges = ranges[:(len(ranges) - 1)] + reProcessQueryRanges, err := gc.CrawlSingleRange(ctx, output, seen, tailRange.start, tailRange.end) + if err != nil { + errs = append(errs, err) + } + ranges = append(ranges, reProcessQueryRanges...) + } + + if len(errs) > 0 { + return errs + } + return nil +} + +func (gc githubCrawler) CrawlSingleRange(ctx context.Context, + output chan<- crawler.CrawledDocument, seen utils.SeenMap, + lowerBound, upperBound uint64) ([]RangeWithin, error) { + + log.Printf("CrawlSingleRange [%d, %d]", lowerBound, upperBound) + noETagClient := GhClient{ RequestConfig: gc.client.RequestConfig, client: &http.Client{Timeout: gc.client.client.Timeout}, @@ -87,13 +117,16 @@ func (gc githubCrawler) Crawl(ctx context.Context, accessToken: gc.client.accessToken, } + var reProcessQueryRanges []RangeWithin + var ranges []string var err error // Since Github returns a max of 1000 results per query, we can use // multiple queries that split the search space into chunks of at most // 1000 files to get all of the data. for i := 0; i < 5; i++ { - ranges, err = FindRangesForRepoSearch(newCache(noETagClient, gc.query)) + ranges, err = FindRangesForRepoSearch(newCache(noETagClient, gc.query), + lowerBound, upperBound) if err == nil { logger.Printf("FindRangesForRepoSearch succeeded after %d retries", i) break @@ -102,7 +135,7 @@ func (gc githubCrawler) Crawl(ctx context.Context, } } if err != nil { - return fmt.Errorf("could not split %v into ranges, %v\n", + return reProcessQueryRanges, fmt.Errorf("could not split %v into ranges, %v\n", gc.query, err) } @@ -112,20 +145,23 @@ func (gc githubCrawler) Crawl(ctx context.Context, errs := make(multiError, 0) queryResult := RangeQueryResult{} for _, query := range ranges { - rangeResult, err := processQuery(ctx, gc.client, query, output, seen, gc.branchMap) + reProcessQuery, rangeResult, err := processQuery(ctx, gc.client, query, output, seen, gc.branchMap) if err != nil { errs = append(errs, err) } queryResult.Add(rangeResult) + if reProcessQuery { + reProcessQueryRanges = append(reProcessQueryRanges, RangeSizes(query)) + } } logger.Printf("Summary of Crawl: %s", queryResult.String()) if len(errs) > 0 { - return errs + return reProcessQueryRanges, errs } - return nil + return reProcessQueryRanges, nil } // FetchDocument first tries to fetch the document with d.FilePath. If it fails, @@ -225,7 +261,7 @@ func (r *RangeQueryResult) String() string { // documents from the crawl to the datastore/index. func processQuery(ctx context.Context, gcl GhClient, query string, output chan<- crawler.CrawledDocument, seen utils.SeenMap, - branchMap map[string]string) (RangeQueryResult, error) { + branchMap map[string]string) (bool, RangeQueryResult, error) { queryPages := make(chan GhResponseInfo) @@ -241,6 +277,8 @@ func processQuery(ctx context.Context, gcl GhClient, query string, close(queryPages) }() + reProcessQuery := false + errs := make(multiError, 0) result := RangeQueryResult{} pageID := 1 @@ -271,11 +309,15 @@ func processQuery(ctx context.Context, gcl GhClient, query string, result.Add(pageResult) pageID++ + + if page.Parsed.TotalCount > githubMaxResultsPerQuery { + reProcessQuery = true + } } logger.Printf("Summary of processQuery: %s", result.String()) - return result, errs + return reProcessQuery, result, errs } func kustomizationResultAdapter(gcl GhClient, k GhFileSpec, seen utils.SeenMap, diff --git a/api/internal/crawl/crawler/github/split_search_ranges.go b/api/internal/crawl/crawler/github/split_search_ranges.go index 8d9037478..81482a1b9 100644 --- a/api/internal/crawl/crawler/github/split_search_ranges.go +++ b/api/internal/crawl/crawler/github/split_search_ranges.go @@ -100,6 +100,8 @@ package github import ( "fmt" "math/bits" + "strconv" + "strings" ) // Files cannot be more than 2^19 bytes, according to @@ -112,7 +114,7 @@ const ( // Interface instead of struct for testing purposes. // Not expecting to have multiple implementations. type cachedSearch interface { - CountResults(uint64) (uint64, error) + CountResults(uint64, uint64) (uint64, error) RequestString(filesize rangeFormatter) string } @@ -161,13 +163,13 @@ func newCache(client GhClient, query Query) githubCachedSearch { } } -func (c githubCachedSearch) CountResults(upperBound uint64) (uint64, error) { +func (c githubCachedSearch) CountResults(lowerBound, upperBound uint64) (uint64, error) { count, cached := c.cache[upperBound] if cached { return count, nil } - sizeRange := RangeWithin{0, upperBound} + sizeRange := RangeWithin{lowerBound, upperBound} rangeRequest := c.RequestString(sizeRange) result := c.gcl.parseGithubResponse(rangeRequest) @@ -238,8 +240,8 @@ func (c githubCachedSearch) RequestString(filesize rangeFormatter) string { // This would mean that the search as it is could not find all files. If queries // are sorted by last indexed, and retrieved on regular intervals, it should be // sufficient to get most if not all documents. -func FindRangesForRepoSearch(cache cachedSearch) ([]string, error) { - totalFiles, err := cache.CountResults(githubMaxFileSize) +func FindRangesForRepoSearch(cache cachedSearch, lowerBound, upperBound uint64) ([]string, error) { + totalFiles, err := cache.CountResults(lowerBound, upperBound) if err != nil { return nil, err } @@ -247,7 +249,7 @@ func FindRangesForRepoSearch(cache cachedSearch) ([]string, error) { if githubMaxResultsPerQuery >= totalFiles { return []string{ - cache.RequestString(RangeWithin{0, githubMaxFileSize}), + cache.RequestString(RangeWithin{lowerBound, upperBound}), }, nil } @@ -275,6 +277,7 @@ func FindRangesForRepoSearch(cache cachedSearch) ([]string, error) { // range. filesAccessible := uint64(0) sizes := make([]uint64, 0) + sizes = append(sizes, lowerBound) for filesAccessible < totalFiles { target := filesAccessible + githubMaxResultsPerQuery if target >= totalFiles { @@ -284,22 +287,22 @@ func FindRangesForRepoSearch(cache cachedSearch) ([]string, error) { logger.Printf("%d accessible files, next target = %d\n", filesAccessible, target) - cur, err := lowerBoundFileCount(cache, target) + size, err := FindFileSize(cache, target, lowerBound, upperBound) if err != nil { return nil, err } // If there are more than 1000 files in the next bucket, we must // advance anyway and lose out on some files :(. - if l := len(sizes); l > 0 && sizes[l-1] == cur { - cur++ + if l := len(sizes); l > 0 && sizes[l-1] == size { + size++ } - nextAccessible, err := cache.CountResults(cur) + nextAccessible, err := cache.CountResults(lowerBound, size) if err != nil { return nil, fmt.Errorf( "cache should be populated at %d already, got %v", - cur, err) + size, err) } if nextAccessible < filesAccessible { return nil, fmt.Errorf( @@ -309,31 +312,31 @@ func FindRangesForRepoSearch(cache cachedSearch) ([]string, error) { filesAccessible = nextAccessible if nextAccessible < totalFiles { - sizes = append(sizes, cur) + sizes = append(sizes, size) } } - + sizes = append(sizes, upperBound) return formatFilesizeRanges(cache, sizes), nil } -// lowerBoundFileCount finds the filesize range from [0, return value] that has +// FindFileSize finds the filesize range from [lowerBound, return value] that has // the largest file count that is smaller than or equal to // githubMaxResultsPerQuery. It is important to note that this returned value // could already be in a previous range if the next file size has more than 1000 // results. It is left to the caller to handle this bit of logic and guarantee // forward progession in this case. -func lowerBoundFileCount( - cache cachedSearch, targetFileCount uint64) (uint64, error) { +func FindFileSize( + cache cachedSearch, targetFileCount, lowerBound, upperBound uint64) (uint64, error) { // Binary search for file sizes that make up the next <=1000 element // chunk. - cur := uint64(0) - increase := githubMaxFileSize / 2 + cur := lowerBound + increase := (upperBound - lowerBound) / 2 for increase > 0 { mid := cur + increase - count, err := cache.CountResults(mid) + count, err := cache.CountResults(lowerBound, mid) if err != nil { return count, err } @@ -353,26 +356,24 @@ func lowerBoundFileCount( } func formatFilesizeRanges(cache cachedSearch, sizes []uint64) []string { - ranges := make([]string, 0, len(sizes)+1) - - if len(sizes) > 0 { - ranges = append(ranges, cache.RequestString( - RangeLessThan{sizes[0] + 1}, - )) + n := len(sizes) + if n < 2 { + return []string{} } - for i := 0; i < len(sizes)-1; i += 1 { - ranges = append(ranges, cache.RequestString( - RangeWithin{sizes[i] + 1, sizes[i+1]}, - )) - - if i != len(sizes)-2 { - continue - } - ranges = append(ranges, cache.RequestString( - RangeGreaterThan{sizes[i+1]}, - )) + ranges := make([]string, 0, n-1) + ranges = append(ranges, cache.RequestString(RangeWithin{sizes[0], sizes[1]})) + for i := 1; i < n-1; i++ { + ranges = append(ranges, cache.RequestString(RangeWithin{sizes[i] + 1, sizes[i+1]})) } - return ranges } + +func RangeSizes(s string) RangeWithin { + start := strings.Index(s, "+size:") + len("+size:") + end := strings.Index(s, "&") + ranges := strings.Split(s[start:end], "..") + lowerBound, _ := strconv.ParseUint(ranges[0], 10, 64) + upperBound, _ := strconv.ParseUint(ranges[1], 10, 64) + return RangeWithin{lowerBound, upperBound} +} diff --git a/api/internal/crawl/crawler/github/split_search_ranges_test.go b/api/internal/crawl/crawler/github/split_search_ranges_test.go index ad332388d..788ef17bc 100644 --- a/api/internal/crawl/crawler/github/split_search_ranges_test.go +++ b/api/internal/crawl/crawler/github/split_search_ranges_test.go @@ -11,7 +11,7 @@ type testCachedSearch struct { cache map[uint64]uint64 } -func (c testCachedSearch) CountResults(upperBound uint64) (uint64, error) { +func (c testCachedSearch) CountResults(lowerBound, upperBound uint64) (uint64, error) { log.Printf("CountResults(%05x)\n", upperBound) count, ok := c.cache[upperBound] if !ok { @@ -73,19 +73,29 @@ func TestRangeSplitting(t *testing.T) { }, } - requests, err := FindRangesForRepoSearch(cache) + requests, err := FindRangesForRepoSearch(cache, 0, 524288) if err != nil { t.Errorf("Error while finding ranges: %v", err) } expected := []string{ - "<107", // cache.RequestString(RangeLessThan{0x6b}), - "107..128", // cache.RequestString(RangeWithin{0x6b, 0x80}), - "129..256", // cache.RequestString(RangeWithin{0x81, 0x100}), - "257..4095", // cache.RequestString(RangeWithin{0x101, 0xfff}), - ">4095", // cache.RequestString(RangeGreaterThan{0xfff}), + "0..106", // cache.RequestString(RangeWithin{0x00, 0x6a}), + "107..128", // cache.RequestString(RangeWithin{0x6b, 0x80}), + "129..256", // cache.RequestString(RangeWithin{0x81, 0x100}), + "257..4095", // cache.RequestString(RangeWithin{0x101, 0xfff}), + "4096..524288", // cache.RequestString(RangeWithin{0x1000, 0x80000}), } if !reflect.DeepEqual(requests, expected) { t.Errorf("Expected requests (%v) to equal (%v)", requests, expected) } } + +func TestRangeSizes(t *testing.T) { + s := "https://api.github.com/search/code?q=filename:kustomization.yaml+filename:kustomization.yml" + + "+filename:kustomization+size:2365..10000&order=desc&per_page=100&sort=indexed" + returnedResult := RangeSizes(s) + expectedResult := RangeWithin{uint64(2365), uint64(10000)} + if !reflect.DeepEqual(returnedResult, expectedResult) { + t.Errorf("RangeSizes expected (%v), got (%v)",expectedResult, returnedResult) + } +} From f5419e9f72fa6c524b4acfbb362accc216954572 Mon Sep 17 00:00:00 2001 From: Haiyan Meng Date: Thu, 30 Jan 2020 11:12:21 -0800 Subject: [PATCH 35/46] Check the `incomplete_results` field of github query responses Currently, we don't check the `incomplete_results` field of a github query response, which is problematic when incomplete query results are used to split the query ranges: the splitted query ranges will be very wild. --- api/internal/crawl/crawler/github/crawler.go | 17 +++++++++++++++-- .../crawl/crawler/github/split_search_ranges.go | 8 ++++---- 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/api/internal/crawl/crawler/github/crawler.go b/api/internal/crawl/crawler/github/crawler.go index 23bcbcd72..a749c6784 100644 --- a/api/internal/crawl/crawler/github/crawler.go +++ b/api/internal/crawl/crawler/github/crawler.go @@ -379,7 +379,7 @@ func (gcl GhClient) ForwardPaginatedQuery(ctx context.Context, query string, output chan<- GhResponseInfo) error { logger.Println("querying: ", query) - response := gcl.parseGithubResponse(query) + response := gcl.parseGithubResponseWithRetry(query) if response.Error != nil { return response.Error @@ -392,7 +392,7 @@ func (gcl GhClient) ForwardPaginatedQuery(ctx context.Context, query string, case <-ctx.Done(): return nil default: - response = gcl.parseGithubResponse(response.NextURL) + response = gcl.parseGithubResponseWithRetry(response.NextURL) if response.Error != nil { return response.Error } @@ -587,6 +587,8 @@ type githubResponse struct { // This is the number of files that match the query. TotalCount uint64 `json:"total_count,omitempty"` + IncompleteResults bool `json:"incomplete_results,omitempty"` + // Github representation of a file. Items []GhFileSpec `json:"items,omitempty"` } @@ -629,6 +631,17 @@ func parseGithubLinkFormat(links string) (string, string) { return next, last } +func (gcl GhClient) parseGithubResponseWithRetry(getRequest string) GhResponseInfo { + resp := gcl.parseGithubResponse(getRequest) + retries := 0 + for resp.Parsed.IncompleteResults { + resp = gcl.parseGithubResponse(getRequest) + retries++ + } + log.Printf("The result of query(%s) is complete after %d retries", getRequest, retries) + return resp +} + func (gcl GhClient) parseGithubResponse(getRequest string) GhResponseInfo { resp, err := gcl.SearchGithubAPI(getRequest) requestInfo := GhResponseInfo{ diff --git a/api/internal/crawl/crawler/github/split_search_ranges.go b/api/internal/crawl/crawler/github/split_search_ranges.go index 81482a1b9..a852330c8 100644 --- a/api/internal/crawl/crawler/github/split_search_ranges.go +++ b/api/internal/crawl/crawler/github/split_search_ranges.go @@ -172,7 +172,7 @@ func (c githubCachedSearch) CountResults(lowerBound, upperBound uint64) (uint64, sizeRange := RangeWithin{lowerBound, upperBound} rangeRequest := c.RequestString(sizeRange) - result := c.gcl.parseGithubResponse(rangeRequest) + result := c.gcl.parseGithubResponseWithRetry(rangeRequest) if result.Error != nil { return count, result.Error } @@ -206,7 +206,7 @@ func (c githubCachedSearch) CountResults(lowerBound, upperBound uint64) (uint64, "Retrying query... current lower bound: %d, got: %d\n", c.cache[prev], result.Parsed.TotalCount) - result = c.gcl.parseGithubResponse(rangeRequest) + result = c.gcl.parseGithubResponseWithRetry(rangeRequest) if result.Error != nil { return count, result.Error } @@ -221,8 +221,8 @@ func (c githubCachedSearch) CountResults(lowerBound, upperBound uint64) (uint64, } count = result.Parsed.TotalCount - logger.Printf("Caching new query %s, with count %d\n", - sizeRange.RangeString(), count) + logger.Printf("Caching new query %s, with count %d (incomplete_results: %v)\n", + sizeRange.RangeString(), count, result.Parsed.IncompleteResults) c.cache[upperBound] = count return count, nil } From 1b8488da2ca3618c7771bdc4e89f241bb8170b8f Mon Sep 17 00:00:00 2001 From: Haiyan Meng Date: Mon, 3 Feb 2020 09:51:56 -0800 Subject: [PATCH 36/46] Add curl commands for snapshoting --- api/internal/crawl/search_cmds/snapshot.md | 24 ++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 api/internal/crawl/search_cmds/snapshot.md diff --git a/api/internal/crawl/search_cmds/snapshot.md b/api/internal/crawl/search_cmds/snapshot.md new file mode 100644 index 000000000..2397cb09e --- /dev/null +++ b/api/internal/crawl/search_cmds/snapshot.md @@ -0,0 +1,24 @@ +Retrieve information about all registered snapshot repositories: +``` +curl -X GET "${ElasticSearchURL}:9200/_snapshot?pretty" +``` + +Retrieve information about a given snapshot repository, `gcskustomize`: +``` +curl -X GET "${ElasticSearchURL}:9200/_snapshot/gcskustomize?pretty" +``` + +Verify a snapshot repository, `gcskustomize`, manually: +``` +curl -X POST "${ElasticSearchURL}:9200/_snapshot/gcskustomize/_verify?pretty" +``` + +Retrieve a summary information about a given snapshot: +``` +curl -X GET "${ElasticSearchURL}:9200/_snapshot/gcskustomize/kustomize-snapshot?pretty" +``` + +Retrieve a detailed information about a given snapshot: +``` +curl -X GET "${ElasticSearchURL}:9200/_snapshot/gcskustomize/kustomize-snapshot/_status?pretty" +``` From a3b3449b1fbcf790b8f1fb394509c3a62a417c34 Mon Sep 17 00:00:00 2001 From: Haiyan Meng Date: Mon, 3 Feb 2020 09:52:24 -0800 Subject: [PATCH 37/46] Add curl commands for generator/transformer exploration --- api/internal/crawl/search_cmds/generator.md | 29 +++++++++++++++++++ api/internal/crawl/search_cmds/transformer.md | 29 +++++++++++++++++++ 2 files changed, 58 insertions(+) create mode 100644 api/internal/crawl/search_cmds/generator.md create mode 100644 api/internal/crawl/search_cmds/transformer.md diff --git a/api/internal/crawl/search_cmds/generator.md b/api/internal/crawl/search_cmds/generator.md new file mode 100644 index 000000000..0b324600f --- /dev/null +++ b/api/internal/crawl/search_cmds/generator.md @@ -0,0 +1,29 @@ +Find all the generator files whose `kinds` field includes `ChartRenderer`, and +only output certain fields of each document: +``` +curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d' +{ + "size": 200, + "_source": { + "includes": ["kinds", "repositoryUrl", "defaultBranch", "filePath"] + }, + "query": { + "bool": { + "filter": [ + { "regexp": { "fileType": "generator" }} + ], + "must_not": { + "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" } + }, + "must": { + "match" : { + "kinds" : { + "query" : "ChartRenderer" + } + } + } + } + } +} +' +``` \ No newline at end of file diff --git a/api/internal/crawl/search_cmds/transformer.md b/api/internal/crawl/search_cmds/transformer.md new file mode 100644 index 000000000..0b35aeb6e --- /dev/null +++ b/api/internal/crawl/search_cmds/transformer.md @@ -0,0 +1,29 @@ +Find all the trasnformer files whose `kinds` field includes `HelmValues`, and +only output certain fields of each document: +``` +curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d' +{ + "size": 200, + "_source": { + "includes": ["kinds", "repositoryUrl", "defaultBranch", "filePath"] + }, + "query": { + "bool": { + "filter": [ + { "regexp": { "fileType": "transformer" }} + ], + "must_not": { + "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" } + }, + "must": { + "match" : { + "kinds" : { + "query" : "HelmValues" + } + } + } + } + } +} +' +``` \ No newline at end of file From 3ebeebabde1f76905880fcb1dc45baffd48275d4 Mon Sep 17 00:00:00 2001 From: Haiyan Meng Date: Mon, 3 Feb 2020 12:20:57 -0800 Subject: [PATCH 38/46] Add comments for backup and restore --- api/internal/crawl/config/elastic/esbackup.yaml | 3 +++ .../crawl/config/elastic/escluster.yaml | 5 +++++ .../crawl/config/elastic/esrestore.yaml | 5 ++++- .../crawl/config/elastic/essnaptshot.yaml | 3 +++ api/internal/crawl/search_cmds/snapshot.md | 17 +++++++++++------ 5 files changed, 26 insertions(+), 7 deletions(-) diff --git a/api/internal/crawl/config/elastic/esbackup.yaml b/api/internal/crawl/config/elastic/esbackup.yaml index 52ee49efa..c509ba206 100644 --- a/api/internal/crawl/config/elastic/esbackup.yaml +++ b/api/internal/crawl/config/elastic/esbackup.yaml @@ -1,3 +1,6 @@ +# Creating `esbackup/kustomize-backbup` will create the `kustomize-backup` snapshot repository. +# Deleting `esbackup/kustomize-backbup` will delete the `kustomize-backup` snapshot repository and all the snapshots in the repository. +# Deleting `esbackup/kustomize-backbup` will NOT delete essnapshot and esrestore objects. apiVersion: elasticsearch.cloud.google.com/v1alpha1 kind: ESBackup metadata: diff --git a/api/internal/crawl/config/elastic/escluster.yaml b/api/internal/crawl/config/elastic/escluster.yaml index c18a9f729..4515cee47 100644 --- a/api/internal/crawl/config/elastic/escluster.yaml +++ b/api/internal/crawl/config/elastic/escluster.yaml @@ -8,6 +8,11 @@ spec: - repository-gcs - ingest-user-agent - ingest-geoip + # To set `gcpserviceaccount`, + # First, create and download a GCP service account into a json file, named `sakey.json` following the instruction: + # https://www.elastic.co/guide/en/elasticsearch/plugins/6.5/repository-gcs-usage.html#repository-gcs-using-service-account + # Second, create a secret for the service account using the following command: + # $ kubectl create secret generic kustomizesa --from-file=./sakey.json gcpserviceaccount: name: kustomizesa config: diff --git a/api/internal/crawl/config/elastic/esrestore.yaml b/api/internal/crawl/config/elastic/esrestore.yaml index 9db48795d..0d77bed93 100644 --- a/api/internal/crawl/config/elastic/esrestore.yaml +++ b/api/internal/crawl/config/elastic/esrestore.yaml @@ -1,3 +1,6 @@ +# Creating `esrestore/kustomize-restore` will restore the `kuostmize` index in the `kustomize-snapshot` snapshot to a new index named `kusotmize-restore`. +# Deleting `esrestore/kustomize-restore` will not delete the restored index. +# Deleting `esrestore/kustomize-restore` should happen before deleting `essnapshot/kustomize-snapshot`. apiVersion: elasticsearch.cloud.google.com/v1alpha1 kind: ESRestore metadata: @@ -6,7 +9,7 @@ spec: include_global_state: true ignore_unavailable: true rename_pattern: kustomize - rename_replacement: kustomize-restore1 + rename_replacement: kustomize-restore essnapshot: name: kustomize-snapshot escluster: diff --git a/api/internal/crawl/config/elastic/essnaptshot.yaml b/api/internal/crawl/config/elastic/essnaptshot.yaml index f36e94897..2a9468960 100644 --- a/api/internal/crawl/config/elastic/essnaptshot.yaml +++ b/api/internal/crawl/config/elastic/essnaptshot.yaml @@ -1,3 +1,6 @@ +# Creating `essnapshot/kustomize-snapshot` will create a snapshot named `kustomize-snapshot` in the `kustomize-backup` snapshot repository. +# Deleting `essnapshot/kustomize-snapshot` will delete the snapshot. +# Deleting `essnapshot/kustomize-snapshot` should happen before deleting `esbackup/kustomize-backup`. apiVersion: elasticsearch.cloud.google.com/v1alpha1 kind: ESSnapshot metadata: diff --git a/api/internal/crawl/search_cmds/snapshot.md b/api/internal/crawl/search_cmds/snapshot.md index 2397cb09e..fd7cc5301 100644 --- a/api/internal/crawl/search_cmds/snapshot.md +++ b/api/internal/crawl/search_cmds/snapshot.md @@ -3,22 +3,27 @@ Retrieve information about all registered snapshot repositories: curl -X GET "${ElasticSearchURL}:9200/_snapshot?pretty" ``` -Retrieve information about a given snapshot repository, `gcskustomize`: +Retrieve information about a given snapshot repository, `kustomize-backup`: ``` -curl -X GET "${ElasticSearchURL}:9200/_snapshot/gcskustomize?pretty" +curl -X GET "${ElasticSearchURL}:9200/_snapshot/kustomize-backup?pretty" ``` -Verify a snapshot repository, `gcskustomize`, manually: +Verify a snapshot repository, `kustomize-backup`, manually: ``` -curl -X POST "${ElasticSearchURL}:9200/_snapshot/gcskustomize/_verify?pretty" +curl -X POST "${ElasticSearchURL}:9200/_snapshot/kustomize-backup/_verify?pretty" +``` + +List all the snapshots in a given snapshot repository: +``` +curl -X GET "${ElasticSearchURL}:9200/_cat/snapshots/kustomize-backup?v&s=id&pretty" ``` Retrieve a summary information about a given snapshot: ``` -curl -X GET "${ElasticSearchURL}:9200/_snapshot/gcskustomize/kustomize-snapshot?pretty" +curl -X GET "${ElasticSearchURL}:9200/_snapshot/kustomize-backup/kustomize-snapshot?pretty" ``` Retrieve a detailed information about a given snapshot: ``` -curl -X GET "${ElasticSearchURL}:9200/_snapshot/gcskustomize/kustomize-snapshot/_status?pretty" +curl -X GET "${ElasticSearchURL}:9200/_snapshot/kustomize-backup/kustomize-snapshot/_status?pretty" ``` From 0d36ff958ff6241d3a217345c42ee49f63180273 Mon Sep 17 00:00:00 2001 From: Jeffrey Regan Date: Mon, 3 Feb 2020 13:22:13 -0800 Subject: [PATCH 39/46] kyaml patch increment to 0.0.10 --- releasing/VERSIONS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/releasing/VERSIONS b/releasing/VERSIONS index 08e9f2cf9..dc9c766e2 100644 --- a/releasing/VERSIONS +++ b/releasing/VERSIONS @@ -6,7 +6,7 @@ # kyaml version export kyaml_major=0 export kyaml_minor=0 -export kyaml_patch=9 +export kyaml_patch=10 # kstatus version export kstatus_major=0 From beb30d79ecf5ff071b11023d95d522e986b6ff50 Mon Sep 17 00:00:00 2001 From: Phillip Wittrock Date: Sun, 19 Jan 2020 09:09:41 -0800 Subject: [PATCH 40/46] Fix tshirt-size image resources --- functions/examples/injection-tshirt-sizes/image/main.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/functions/examples/injection-tshirt-sizes/image/main.go b/functions/examples/injection-tshirt-sizes/image/main.go index 25891e7ac..30c998e17 100644 --- a/functions/examples/injection-tshirt-sizes/image/main.go +++ b/functions/examples/injection-tshirt-sizes/image/main.go @@ -48,9 +48,9 @@ var cpuSizes = map[string]string{ // memorySizes is the mapping from tshirt-size to memory reservation quantity var memorySizes = map[string]string{ - "small": "50MiB", - "medium": "1GiB", - "large": "32GiB", + "small": "50M", + "medium": "1G", + "large": "32G", } // inject sets the cpu and memory reservations on all containers for Resources annotated From 2d54981bcd869c5c9f6fc70b932e49e206972fed Mon Sep 17 00:00:00 2001 From: Phillip Wittrock Date: Mon, 3 Feb 2020 15:52:38 -0800 Subject: [PATCH 41/46] optionally use filepath as part of merge key for merge3 --- cmd/config/internal/commands/merge3.go | 4 + kyaml/kio/filters/merge3.go | 42 ++++++++++- kyaml/kio/filters/merge3_test.go | 75 +++++++++++++++++++ .../java/java-deployment-1.resource.yaml | 38 ++++++++++ .../java/java-deployment-2.resource.yaml | 38 ++++++++++ .../java/java-deployment-1.resource.yaml | 38 ++++++++++ .../java/java-deployment-2.resource.yaml | 38 ++++++++++ .../java/java-deployment-1.resource.yaml | 38 ++++++++++ .../java/java-deployment-2.resource.yaml | 38 ++++++++++ .../java/java-deployment-1.resource.yaml | 38 ++++++++++ .../java/java-deployment-2.resource.yaml | 38 ++++++++++ 11 files changed, 422 insertions(+), 3 deletions(-) create mode 100644 kyaml/kio/filters/testdata2/dataset1-expected/java/java-deployment-1.resource.yaml create mode 100644 kyaml/kio/filters/testdata2/dataset1-expected/java/java-deployment-2.resource.yaml create mode 100644 kyaml/kio/filters/testdata2/dataset1-localupdates/java/java-deployment-1.resource.yaml create mode 100644 kyaml/kio/filters/testdata2/dataset1-localupdates/java/java-deployment-2.resource.yaml create mode 100644 kyaml/kio/filters/testdata2/dataset1-remoteupdates/java/java-deployment-1.resource.yaml create mode 100644 kyaml/kio/filters/testdata2/dataset1-remoteupdates/java/java-deployment-2.resource.yaml create mode 100644 kyaml/kio/filters/testdata2/dataset1/java/java-deployment-1.resource.yaml create mode 100644 kyaml/kio/filters/testdata2/dataset1/java/java-deployment-2.resource.yaml diff --git a/cmd/config/internal/commands/merge3.go b/cmd/config/internal/commands/merge3.go index 405638463..6c31186c5 100644 --- a/cmd/config/internal/commands/merge3.go +++ b/cmd/config/internal/commands/merge3.go @@ -25,6 +25,8 @@ func GetMerge3Runner(name string) *Merge3Runner { "Path to updated package") c.Flags().StringVar(&r.toDir, "to", "", "Path to destination package") + c.Flags().BoolVar(&r.path, "path-merge-key", false, + "Use the path as part of the merge key when merging resources") r.Command = c return r @@ -40,6 +42,7 @@ type Merge3Runner struct { ancestor string fromDir string toDir string + path bool } func (r *Merge3Runner) runE(c *cobra.Command, args []string) error { @@ -47,6 +50,7 @@ func (r *Merge3Runner) runE(c *cobra.Command, args []string) error { OriginalPath: r.ancestor, UpdatedPath: r.fromDir, DestPath: r.toDir, + MergeOnPath: r.path, }.Merge() if err != nil { return err diff --git a/kyaml/kio/filters/merge3.go b/kyaml/kio/filters/merge3.go index f302eeada..dfbb47827 100644 --- a/kyaml/kio/filters/merge3.go +++ b/kyaml/kio/filters/merge3.go @@ -7,6 +7,7 @@ import ( "fmt" "sigs.k8s.io/kustomize/kyaml/kio" + "sigs.k8s.io/kustomize/kyaml/kio/kioutil" "sigs.k8s.io/kustomize/kyaml/yaml" "sigs.k8s.io/kustomize/kyaml/yaml/merge3" ) @@ -24,6 +25,11 @@ type Merge3 struct { UpdatedPath string DestPath string MatchFilesGlob []string + + // MergeOnPath will use the relative filepath as part of the merge key. + // This may be necessary if the directory contains multiple copies of + // the same resource, or resources patches. + MergeOnPath bool } func (m Merge3) Merge() error { @@ -61,7 +67,7 @@ func (m Merge3) Merge() error { // Filter combines Resources with the same GVK + N + NS into tuples, and then merges them func (m Merge3) Filter(nodes []*yaml.RNode) ([]*yaml.RNode, error) { // index the nodes by their identity - tl := tuples{} + tl := tuples{mergeOnPath: m.MergeOnPath} for i := range nodes { if err := tl.add(nodes[i]); err != nil { return nil, err @@ -102,6 +108,37 @@ func (m Merge3) Filter(nodes []*yaml.RNode) ([]*yaml.RNode, error) { // tuples combines nodes with the same GVK + N + NS type tuples struct { list []*tuple + + // mergeOnPath if set to true will use the resource filepath + // as part of the merge key + mergeOnPath bool +} + +// isSameResource returns true if meta1 and meta2 are for the same logic resource +func (ts *tuples) isSameResource(meta1, meta2 yaml.ResourceMeta) bool { + if meta1.Name != meta2.Name { + return false + } + if meta1.Namespace != meta2.Namespace { + return false + } + if meta1.APIVersion != meta2.APIVersion { + return false + } + if meta1.Kind != meta2.Kind { + return false + } + if ts.mergeOnPath { + // directories may contain multiple copies of a resource with the same + // name, namespace, apiVersion and kind -- e.g. kustomize patches, or + // multiple environments + // mergeOnPath configures the merge logic to use the path as part of the + // resource key + if meta1.Annotations[kioutil.PathAnnotation] != meta2.Annotations[kioutil.PathAnnotation] { + return false + } + } + return true } // add adds a node to the list, combining it with an existing matching Resource if found @@ -112,8 +149,7 @@ func (ts *tuples) add(node *yaml.RNode) error { } for i := range ts.list { t := ts.list[i] - if t.meta.Name == nodeMeta.Name && t.meta.Namespace == nodeMeta.Namespace && - t.meta.APIVersion == nodeMeta.APIVersion && t.meta.Kind == nodeMeta.Kind { + if ts.isSameResource(t.meta, nodeMeta) { return t.add(node) } } diff --git a/kyaml/kio/filters/merge3_test.go b/kyaml/kio/filters/merge3_test.go index d8615b33d..66974c2b5 100644 --- a/kyaml/kio/filters/merge3_test.go +++ b/kyaml/kio/filters/merge3_test.go @@ -54,3 +54,78 @@ func TestMerge3_Merge(t *testing.T) { t.FailNow() } } + +// TestMerge3_Merge_path tests that if the same resource is specified multiple times +// with MergeOnPath, that the resources will be merged by the filepath name. +func TestMerge3_Merge_path(t *testing.T) { + _, datadir, _, ok := runtime.Caller(0) + if !assert.True(t, ok) { + t.FailNow() + } + datadir = filepath.Join(filepath.Dir(datadir), "testdata2") + + // setup the local directory + dir, err := ioutil.TempDir("", "kyaml-test") + if !assert.NoError(t, err) { + t.FailNow() + } + defer os.RemoveAll(dir) + + if !assert.NoError(t, copyutil.CopyDir( + filepath.Join(datadir, "dataset1-localupdates"), + filepath.Join(dir, "dataset1"))) { + t.FailNow() + } + + err = filters.Merge3{ + OriginalPath: filepath.Join(datadir, "dataset1"), + UpdatedPath: filepath.Join(datadir, "dataset1-remoteupdates"), + DestPath: filepath.Join(dir, "dataset1"), + MergeOnPath: true, + }.Merge() + if !assert.NoError(t, err) { + t.FailNow() + } + + diffs, err := copyutil.Diff( + filepath.Join(dir, "dataset1"), + filepath.Join(datadir, "dataset1-expected")) + if !assert.NoError(t, err) { + t.FailNow() + } + if !assert.Empty(t, diffs.List()) { + t.FailNow() + } +} + +// TestMerge3_Merge_fail tests that if the same resource is defined multiple times +// that merge will fail +func TestMerge3_Merge_fail(t *testing.T) { + _, datadir, _, ok := runtime.Caller(0) + if !assert.True(t, ok) { + t.FailNow() + } + datadir = filepath.Join(filepath.Dir(datadir), "testdata2") + + // setup the local directory + dir, err := ioutil.TempDir("", "kyaml-test") + if !assert.NoError(t, err) { + t.FailNow() + } + defer os.RemoveAll(dir) + + if !assert.NoError(t, copyutil.CopyDir( + filepath.Join(datadir, "dataset1-localupdates"), + filepath.Join(dir, "dataset1"))) { + t.FailNow() + } + + err = filters.Merge3{ + OriginalPath: filepath.Join(datadir, "dataset1"), + UpdatedPath: filepath.Join(datadir, "dataset1-remoteupdates"), + DestPath: filepath.Join(dir, "dataset1"), + }.Merge() + if !assert.Error(t, err) { + t.FailNow() + } +} diff --git a/kyaml/kio/filters/testdata2/dataset1-expected/java/java-deployment-1.resource.yaml b/kyaml/kio/filters/testdata2/dataset1-expected/java/java-deployment-1.resource.yaml new file mode 100644 index 000000000..ea1d0f540 --- /dev/null +++ b/kyaml/kio/filters/testdata2/dataset1-expected/java/java-deployment-1.resource.yaml @@ -0,0 +1,38 @@ +# Copyright 2019 The Kubernetes Authors. +# SPDX-License-Identifier: Apache-2.0 +# +apiVersion: apps/v1 +kind: Deployment +metadata: + name: app + labels: + app: java +spec: + replicas: 1 + selector: + matchLabels: + app: java2 + template: + metadata: + labels: + app: java2 + spec: + restartPolicy: Always + containers: + - name: app + image: gcr.io/project/app:version + command: + - java + - -jar + - /app.jar + ports: + - containerPort: 8080 + envFrom: + - configMapRef: + name: app-config + env: + - name: JAVA_OPTS + value: -XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap + -Djava.security.egd=file:/dev/./urandom + imagePullPolicy: Always + minReadySeconds: 15 diff --git a/kyaml/kio/filters/testdata2/dataset1-expected/java/java-deployment-2.resource.yaml b/kyaml/kio/filters/testdata2/dataset1-expected/java/java-deployment-2.resource.yaml new file mode 100644 index 000000000..6b13e7872 --- /dev/null +++ b/kyaml/kio/filters/testdata2/dataset1-expected/java/java-deployment-2.resource.yaml @@ -0,0 +1,38 @@ +# Copyright 2019 The Kubernetes Authors. +# SPDX-License-Identifier: Apache-2.0 +# +apiVersion: apps/v1 +kind: Deployment +metadata: + name: app + labels: + app: java3 +spec: + replicas: 4 + selector: + matchLabels: + app: java + template: + metadata: + labels: + app: java + spec: + restartPolicy: Always + containers: + - name: app + image: gcr.io/project/app:version + command: + - java + - -jar + - /app.jar + ports: + - containerPort: 8080 + envFrom: + - configMapRef: + name: app-config + env: + - name: JAVA_OPTS + value: -XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap + -Djava.security.egd=file:/dev/./urandom + imagePullPolicy: Always + minReadySeconds: 25 diff --git a/kyaml/kio/filters/testdata2/dataset1-localupdates/java/java-deployment-1.resource.yaml b/kyaml/kio/filters/testdata2/dataset1-localupdates/java/java-deployment-1.resource.yaml new file mode 100644 index 000000000..658a1ec80 --- /dev/null +++ b/kyaml/kio/filters/testdata2/dataset1-localupdates/java/java-deployment-1.resource.yaml @@ -0,0 +1,38 @@ +# Copyright 2019 The Kubernetes Authors. +# SPDX-License-Identifier: Apache-2.0 +# +apiVersion: apps/v1 +kind: Deployment +metadata: + name: app + labels: + app: java +spec: + replicas: 1 + selector: + matchLabels: + app: java + template: + metadata: + labels: + app: java + spec: + restartPolicy: Always + containers: + - name: app + image: gcr.io/project/app:version + command: + - java + - -jar + - /app.jar + ports: + - containerPort: 8080 + envFrom: + - configMapRef: + name: app-config + env: + - name: JAVA_OPTS + value: -XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap + -Djava.security.egd=file:/dev/./urandom + imagePullPolicy: Always + minReadySeconds: 15 diff --git a/kyaml/kio/filters/testdata2/dataset1-localupdates/java/java-deployment-2.resource.yaml b/kyaml/kio/filters/testdata2/dataset1-localupdates/java/java-deployment-2.resource.yaml new file mode 100644 index 000000000..64baafe94 --- /dev/null +++ b/kyaml/kio/filters/testdata2/dataset1-localupdates/java/java-deployment-2.resource.yaml @@ -0,0 +1,38 @@ +# Copyright 2019 The Kubernetes Authors. +# SPDX-License-Identifier: Apache-2.0 +# +apiVersion: apps/v1 +kind: Deployment +metadata: + name: app + labels: + app: java +spec: + replicas: 4 + selector: + matchLabels: + app: java + template: + metadata: + labels: + app: java + spec: + restartPolicy: Always + containers: + - name: app + image: gcr.io/project/app:version + command: + - java + - -jar + - /app.jar + ports: + - containerPort: 8080 + envFrom: + - configMapRef: + name: app-config + env: + - name: JAVA_OPTS + value: -XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap + -Djava.security.egd=file:/dev/./urandom + imagePullPolicy: Always + minReadySeconds: 25 diff --git a/kyaml/kio/filters/testdata2/dataset1-remoteupdates/java/java-deployment-1.resource.yaml b/kyaml/kio/filters/testdata2/dataset1-remoteupdates/java/java-deployment-1.resource.yaml new file mode 100644 index 000000000..c8174a0b3 --- /dev/null +++ b/kyaml/kio/filters/testdata2/dataset1-remoteupdates/java/java-deployment-1.resource.yaml @@ -0,0 +1,38 @@ +# Copyright 2019 The Kubernetes Authors. +# SPDX-License-Identifier: Apache-2.0 +# +apiVersion: apps/v1 +kind: Deployment +metadata: + name: app + labels: + app: java +spec: + replicas: 1 + selector: + matchLabels: + app: java2 + template: + metadata: + labels: + app: java2 + spec: + restartPolicy: Always + containers: + - name: app + image: gcr.io/project/app:version + command: + - java + - -jar + - /app.jar + ports: + - containerPort: 8080 + envFrom: + - configMapRef: + name: app-config + env: + - name: JAVA_OPTS + value: -XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap + -Djava.security.egd=file:/dev/./urandom + imagePullPolicy: Always + minReadySeconds: 5 diff --git a/kyaml/kio/filters/testdata2/dataset1-remoteupdates/java/java-deployment-2.resource.yaml b/kyaml/kio/filters/testdata2/dataset1-remoteupdates/java/java-deployment-2.resource.yaml new file mode 100644 index 000000000..9caeaaea1 --- /dev/null +++ b/kyaml/kio/filters/testdata2/dataset1-remoteupdates/java/java-deployment-2.resource.yaml @@ -0,0 +1,38 @@ +# Copyright 2019 The Kubernetes Authors. +# SPDX-License-Identifier: Apache-2.0 +# +apiVersion: apps/v1 +kind: Deployment +metadata: + name: app + labels: + app: java3 +spec: + replicas: 4 + selector: + matchLabels: + app: java + template: + metadata: + labels: + app: java + spec: + restartPolicy: Always + containers: + - name: app + image: gcr.io/project/app:version + command: + - java + - -jar + - /app.jar + ports: + - containerPort: 8080 + envFrom: + - configMapRef: + name: app-config + env: + - name: JAVA_OPTS + value: -XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap + -Djava.security.egd=file:/dev/./urandom + imagePullPolicy: Always + minReadySeconds: 10 diff --git a/kyaml/kio/filters/testdata2/dataset1/java/java-deployment-1.resource.yaml b/kyaml/kio/filters/testdata2/dataset1/java/java-deployment-1.resource.yaml new file mode 100644 index 000000000..53ffa5b4f --- /dev/null +++ b/kyaml/kio/filters/testdata2/dataset1/java/java-deployment-1.resource.yaml @@ -0,0 +1,38 @@ +# Copyright 2019 The Kubernetes Authors. +# SPDX-License-Identifier: Apache-2.0 +# +apiVersion: apps/v1 +kind: Deployment +metadata: + name: app + labels: + app: java +spec: + replicas: 1 + selector: + matchLabels: + app: java + template: + metadata: + labels: + app: java + spec: + restartPolicy: Always + containers: + - name: app + image: gcr.io/project/app:version + command: + - java + - -jar + - /app.jar + ports: + - containerPort: 8080 + envFrom: + - configMapRef: + name: app-config + env: + - name: JAVA_OPTS + value: -XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap + -Djava.security.egd=file:/dev/./urandom + imagePullPolicy: Always + minReadySeconds: 5 diff --git a/kyaml/kio/filters/testdata2/dataset1/java/java-deployment-2.resource.yaml b/kyaml/kio/filters/testdata2/dataset1/java/java-deployment-2.resource.yaml new file mode 100644 index 000000000..9c5cdc3aa --- /dev/null +++ b/kyaml/kio/filters/testdata2/dataset1/java/java-deployment-2.resource.yaml @@ -0,0 +1,38 @@ +# Copyright 2019 The Kubernetes Authors. +# SPDX-License-Identifier: Apache-2.0 +# +apiVersion: apps/v1 +kind: Deployment +metadata: + name: app + labels: + app: java +spec: + replicas: 4 + selector: + matchLabels: + app: java + template: + metadata: + labels: + app: java + spec: + restartPolicy: Always + containers: + - name: app + image: gcr.io/project/app:version + command: + - java + - -jar + - /app.jar + ports: + - containerPort: 8080 + envFrom: + - configMapRef: + name: app-config + env: + - name: JAVA_OPTS + value: -XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap + -Djava.security.egd=file:/dev/./urandom + imagePullPolicy: Always + minReadySeconds: 10 From 79d591e2b01c4c4825de56f743f2f78de0d1fda4 Mon Sep 17 00:00:00 2001 From: Phani Teja Marupaka Date: Tue, 4 Feb 2020 09:34:10 -0800 Subject: [PATCH 42/46] Opt in to use schema --- cmd/config/internal/commands/fmt.go | 7 +- kyaml/kio/filters/fmtr.go | 11 ++- kyaml/kio/filters/fmtr_test.go | 107 +++++++++++++++++++++++----- 3 files changed, 103 insertions(+), 22 deletions(-) diff --git a/cmd/config/internal/commands/fmt.go b/cmd/config/internal/commands/fmt.go index d175d6bc9..3a481d36b 100644 --- a/cmd/config/internal/commands/fmt.go +++ b/cmd/config/internal/commands/fmt.go @@ -31,6 +31,8 @@ formatting substitution verbs {'%n': 'metadata.name', '%s': 'metadata.namespace' `if true, keep index and filename annotations set on Resources.`) c.Flags().BoolVar(&r.Override, "override", false, `if true, override existing filepath annotations.`) + c.Flags().BoolVar(&r.UseSchema, "use-schema", false, + `if true, uses openapi resource schema to format resources.`) r.Command = c return r } @@ -46,6 +48,7 @@ type FmtRunner struct { SetFilenames bool KeepAnnotations bool Override bool + UseSchema bool } func (r *FmtRunner) preRunE(c *cobra.Command, args []string) error { @@ -56,7 +59,9 @@ func (r *FmtRunner) preRunE(c *cobra.Command, args []string) error { } func (r *FmtRunner) runE(c *cobra.Command, args []string) error { - f := []kio.Filter{filters.FormatFilter{}} + f := []kio.Filter{filters.FormatFilter{ + UseSchema: r.UseSchema, + }} // format with file names if r.SetFilenames { diff --git a/kyaml/kio/filters/fmtr.go b/kyaml/kio/filters/fmtr.go index 0266fc550..5cb2b0c00 100644 --- a/kyaml/kio/filters/fmtr.go +++ b/kyaml/kio/filters/fmtr.go @@ -56,7 +56,8 @@ func FormatFileOrDirectory(path string) error { } type FormatFilter struct { - Process func(n *yaml.Node) error + Process func(n *yaml.Node) error + UseSchema bool } var _ kio.Filter = FormatFilter{} @@ -78,7 +79,12 @@ func (f FormatFilter) Filter(slice []*yaml.RNode) ([]*yaml.RNode, error) { continue } kind, apiVersion := kindNode.YNode().Value, apiVersionNode.YNode().Value - s := openapi.SchemaForResourceType(yaml.TypeMeta{APIVersion: apiVersion, Kind: kind}) + var s *openapi.ResourceSchema + if f.UseSchema { + s = openapi.SchemaForResourceType(yaml.TypeMeta{APIVersion: apiVersion, Kind: kind}) + } else { + s = nil + } err = (&formatter{apiVersion: apiVersion, kind: kind, process: f.Process}). fmtNode(slice[i].YNode(), "", s) if err != nil { @@ -151,7 +157,6 @@ func (f *formatter) fmtNode(n *yaml.Node, path string, schema *openapi.ResourceS s = schema.Elements() } } - // format the node using the schema err := f.fmtNode(n.Content[i], p, s) if err != nil { diff --git a/kyaml/kio/filters/fmtr_test.go b/kyaml/kio/filters/fmtr_test.go index dff6148f6..67e1be700 100644 --- a/kyaml/kio/filters/fmtr_test.go +++ b/kyaml/kio/filters/fmtr_test.go @@ -74,6 +74,73 @@ spec: containerPort: 80 ` + buff := &bytes.Buffer{} + err := kio.Pipeline{ + Inputs: []kio.Reader{&kio.ByteReader{Reader: strings.NewReader(y)}}, + Filters: []kio.Filter{FormatFilter{ + UseSchema: true, + }}, + Outputs: []kio.Writer{kio.ByteWriter{Writer: buff}}, + }.Execute() + assert.NoError(t, err) + assert.Equal(t, expected, buff.String()) +} + +func TestFormat_UnsortedInput_No_Schema(t *testing.T) { + y := ` +apiVersion: apps/v1 +spec: + template: + spec: + containers: + - name: nginx + image: nginx:1.0.0 + args: + - on + - 1 + - hello + ports: + - name: http + targetPort: 80 + containerPort: 80 +kind: Deployment +metadata: + name: foo + labels: + foo: on + foo2: hello1 + annotations: + bar: 1 + bar2: hello2 +` + + // keep the style on values that parse as non-string types + expected := `apiVersion: apps/v1 +kind: Deployment +metadata: + name: foo + labels: + foo: on + foo2: hello1 + annotations: + bar: 1 + bar2: hello2 +spec: + template: + spec: + containers: + - name: nginx + image: nginx:1.0.0 + args: + - on + - 1 + - hello + ports: + - name: http + targetPort: 80 + containerPort: 80 +` + buff := &bytes.Buffer{} err := kio.Pipeline{ Inputs: []kio.Reader{&kio.ByteReader{Reader: strings.NewReader(y)}}, @@ -120,16 +187,18 @@ spec: buff := &bytes.Buffer{} err := kio.Pipeline{ Inputs: []kio.Reader{&kio.ByteReader{Reader: strings.NewReader(y)}}, - Filters: []kio.Filter{FormatFilter{Process: func(n *yaml.Node) error { - if yaml.IsYaml1_1NonString(n) { - // don't change these styles, they are important for backwards compatibility - // e.g. "on" must remain quoted, on must remain unquoted + Filters: []kio.Filter{FormatFilter{ + UseSchema: true, + Process: func(n *yaml.Node) error { + if yaml.IsYaml1_1NonString(n) { + // don't change these styles, they are important for backwards compatibility + // e.g. "on" must remain quoted, on must remain unquoted + return nil + } + // style does not have semantic meaning + n.Style = 0 return nil - } - // style does not have semantic meaning - n.Style = 0 - return nil - }}}, + }}}, Outputs: []kio.Writer{kio.ByteWriter{Writer: buff}}, }.Execute() assert.NoError(t, err) @@ -186,16 +255,18 @@ spec: buff = &bytes.Buffer{} err = kio.Pipeline{ Inputs: []kio.Reader{&kio.ByteReader{Reader: strings.NewReader(y)}}, - Filters: []kio.Filter{FormatFilter{Process: func(n *yaml.Node) error { - if yaml.IsYaml1_1NonString(n) { - // don't change these styles, they are important for backwards compatibility - // e.g. "on" must remain quoted, on must remain unquoted + Filters: []kio.Filter{FormatFilter{ + UseSchema: true, + Process: func(n *yaml.Node) error { + if yaml.IsYaml1_1NonString(n) { + // don't change these styles, they are important for backwards compatibility + // e.g. "on" must remain quoted, on must remain unquoted + return nil + } + // style does not have semantic meaning + n.Style = yaml.SingleQuotedStyle return nil - } - // style does not have semantic meaning - n.Style = yaml.SingleQuotedStyle - return nil - }}}, + }}}, Outputs: []kio.Writer{kio.ByteWriter{Writer: buff}}, }.Execute() assert.NoError(t, err) From 32fc17fedd6553cdb62debdbd82567f8a271fd26 Mon Sep 17 00:00:00 2001 From: Phillip Wittrock Date: Tue, 4 Feb 2020 13:50:51 -0800 Subject: [PATCH 43/46] kio: don't apply folded style to wrapped items --- kyaml/kio/byteio_writer.go | 1 - 1 file changed, 1 deletion(-) diff --git a/kyaml/kio/byteio_writer.go b/kyaml/kio/byteio_writer.go index 9b56e4e56..93198d9bb 100644 --- a/kyaml/kio/byteio_writer.go +++ b/kyaml/kio/byteio_writer.go @@ -113,7 +113,6 @@ func (w ByteWriter) Write(nodes []*yaml.RNode) error { } doc := &yaml.Node{ Kind: yaml.DocumentNode, - Style: yaml.FoldedStyle, Content: []*yaml.Node{list}} for i := range nodes { items.Content = append(items.Content, nodes[i].YNode()) From 073a11f3f114911aeced0b9da02e99332373ebba Mon Sep 17 00:00:00 2001 From: Phillip Wittrock Date: Tue, 4 Feb 2020 16:27:56 -0800 Subject: [PATCH 44/46] Support loading openapi from []bytes --- kyaml/openapi/openapi.go | 181 ++++++++++++++++++++++++++-------- kyaml/openapi/openapi_test.go | 61 +++++++++++- 2 files changed, 197 insertions(+), 45 deletions(-) diff --git a/kyaml/openapi/openapi.go b/kyaml/openapi/openapi.go index 9cb8bfb19..7587968f1 100644 --- a/kyaml/openapi/openapi.go +++ b/kyaml/openapi/openapi.go @@ -4,15 +4,26 @@ package openapi import ( + "encoding/json" + "fmt" "sync" "github.com/go-openapi/spec" + "sigs.k8s.io/kustomize/kyaml/errors" "sigs.k8s.io/kustomize/kyaml/yaml" ) -var setup sync.Once -var schema spec.Schema -var schemaByResourceType map[yaml.TypeMeta]*spec.Schema +// globalSchema contains global state information about the openapi +var globalSchema openapiData + +// openapiData contains the parsed openapi state. this is in a struct rather than +// a list of vars so that it can be reset from tests. +type openapiData struct { + setup sync.Once + schema spec.Schema + schemaByResourceType map[yaml.TypeMeta]*spec.Schema + noUseBuiltInSchema bool +} // ResourceSchema wraps the OpenAPI Schema. type ResourceSchema struct { @@ -26,13 +37,95 @@ type ResourceSchema struct { // as metadata, replicas and spec.template.spec func SchemaForResourceType(t yaml.TypeMeta) *ResourceSchema { initSchema() - rs, found := schemaByResourceType[t] + rs, found := globalSchema.schemaByResourceType[t] if !found { return nil } return &ResourceSchema{Schema: rs} } +// AddSchema parses s, and adds definitions from s to the global schema. +func AddSchema(s []byte) (*spec.Schema, error) { + return parse(s) +} + +// AddDefinitions adds the definitions to the global schema. +func AddDefinitions(definitions spec.Definitions) { + // initialize values if they have not yet been set + if globalSchema.schemaByResourceType == nil { + globalSchema.schemaByResourceType = map[yaml.TypeMeta]*spec.Schema{} + } + if globalSchema.schema.Definitions == nil { + globalSchema.schema.Definitions = spec.Definitions{} + } + + // index the schema definitions so we can lookup them up for Resources + for k := range definitions { + // index by GVK, if no GVK is found then it is the schema for a subfield + // of a Resource + d := definitions[k] + + // copy definitions to the schema + globalSchema.schema.Definitions[k] = d + gvk, found := d.VendorExtensible.Extensions[kubernetesGVKExtensionKey] + if !found { + continue + } + // cast the extension to a []map[string]string + exts, ok := gvk.([]interface{}) + if !ok || len(exts) != 1 { + continue + } + m, ok := exts[0].(map[string]interface{}) + if !ok { + continue + } + + // build the index key and save it + g := m[groupKey].(string) + apiVersion := m[versionKey].(string) + if g != "" { + apiVersion = g + "/" + apiVersion + } + globalSchema.schemaByResourceType[yaml.TypeMeta{Kind: m[kindKey].(string), APIVersion: apiVersion}] = &d + } +} + +// Resolve resolves the reference against the global schema +func Resolve(ref *spec.Ref) (*spec.Schema, error) { + return resolve(Schema(), ref) +} + +// Schema returns the global schema +func Schema() *spec.Schema { + return rootSchema() +} + +// GetSchema parses s into a ResourceSchema, resolving References within the +// global schema. +func GetSchema(s string) (*ResourceSchema, error) { + var sc spec.Schema + if err := sc.UnmarshalJSON([]byte(s)); err != nil { + return nil, errors.Wrap(err) + } + if sc.Ref.String() != "" { + r, err := Resolve(&sc.Ref) + if err != nil { + return nil, errors.Wrap(err) + } + sc = *r + } + + return &ResourceSchema{Schema: &sc}, nil +} + +// SuppressBuiltInSchemaUse can be called to prevent using the built-in Kubernetes +// schema as part of the global schema. +// Must be called before the schema is used. +func SuppressBuiltInSchemaUse() { + globalSchema.noUseBuiltInSchema = false +} + // Elements returns the Schema for the elements of an array. func (r *ResourceSchema) Elements() *ResourceSchema { // load the schema from swagger.json @@ -44,7 +137,7 @@ func (r *ResourceSchema) Elements() *ResourceSchema { } s := *r.Schema.Items.Schema for s.Ref.String() != "" { - sc, e := spec.ResolveRef(rootSchema(), &s.Ref) + sc, e := Resolve(&s.Ref) if e != nil { return nil } @@ -96,7 +189,7 @@ func (r *ResourceSchema) Field(field string) *ResourceSchema { // resolve the reference to the Schema if the Schema has one for s.Ref.String() != "" { - sc, e := spec.ResolveRef(rootSchema(), &s.Ref) + sc, e := Resolve(&s.Ref) if e != nil { return nil } @@ -148,53 +241,57 @@ const ( // initSchema parses the json schema func initSchema() { - setup.Do(func() { - // initialize the map - schemaByResourceType = map[yaml.TypeMeta]*spec.Schema{} + globalSchema.setup.Do(func() { + if globalSchema.noUseBuiltInSchema { + // don't parse the built in schema + return + } // parse the swagger, this should never fail - parse(MustAsset(openAPIAssetName)) - - // TODO(pwittrock): add support for parsing additional schemas from - // environment variables, files or other sources + if _, err := parse(MustAsset(openAPIAssetName)); err != nil { + // this should never happen + panic(err) + } }) } // parse parses and indexes a single json schema -func parse(b []byte) { - if err := schema.UnmarshalJSON(b); err != nil { - panic(err) - } - // index the schema definitions so we can lookup them up for Resources - for k := range schema.Definitions { - // index by GVK, if no GVK is found then it is the schema for a subfield - // of a Resource - d := schema.Definitions[k] - gvk, found := d.VendorExtensible.Extensions[kubernetesGVKExtensionKey] - if !found { - continue - } - // cast the extension to a []map[string]string - exts, ok := gvk.([]interface{}) - if !ok || len(exts) != 1 { - continue - } - m, ok := exts[0].(map[string]interface{}) - if !ok { - continue - } +func parse(b []byte) (*spec.Schema, error) { + var sc spec.Schema - // build the index key and save it - g := m[groupKey].(string) - apiVersion := m[versionKey].(string) - if g != "" { - apiVersion = g + "/" + apiVersion + if err := sc.UnmarshalJSON(b); err != nil { + return nil, errors.Wrap(err) + } + AddDefinitions(sc.Definitions) + return &sc, nil +} + +func resolve(root interface{}, ref *spec.Ref) (*spec.Schema, error) { + res, _, err := ref.GetPointer().Get(root) + if err != nil { + return nil, errors.Wrap(err) + } + switch sch := res.(type) { + case spec.Schema: + return &sch, nil + case *spec.Schema: + return sch, nil + case map[string]interface{}: + b, err := json.Marshal(sch) + if err != nil { + return nil, err } - schemaByResourceType[yaml.TypeMeta{Kind: m[kindKey].(string), APIVersion: apiVersion}] = &d + newSch := new(spec.Schema) + if err = json.Unmarshal(b, newSch); err != nil { + return nil, err + } + return newSch, nil + default: + return nil, errors.Wrap(fmt.Errorf("unknown type for the resolved reference")) } } func rootSchema() *spec.Schema { initSchema() - return &schema + return &globalSchema.schema } diff --git a/kyaml/openapi/openapi_test.go b/kyaml/openapi/openapi_test.go index 8318dbc7e..04bf26f0b 100644 --- a/kyaml/openapi/openapi_test.go +++ b/kyaml/openapi/openapi_test.go @@ -1,18 +1,73 @@ // Copyright 2019 The Kubernetes Authors. // SPDX-License-Identifier: Apache-2.0 -package openapi_test +package openapi import ( + "fmt" "testing" "github.com/stretchr/testify/assert" - "sigs.k8s.io/kustomize/kyaml/openapi" "sigs.k8s.io/kustomize/kyaml/yaml" ) +func TestAddSchema(t *testing.T) { + // reset package vars + globalSchema = openapiData{} + + _, err := AddSchema(additionalSchema) + if !assert.NoError(t, err) { + t.FailNow() + } + s, err := GetSchema(`{"$ref": "#/definitions/io.k8s.config.setters.replicas"}`) + if !assert.Greater(t, len(globalSchema.schema.Definitions), 200) { + t.FailNow() + } + if !assert.NoError(t, err) { + t.FailNow() + } + assert.Equal(t, `map[x-kustomize:map[setBy:Jane setter:map[name:replicas value:5]]]`, + fmt.Sprintf("%v", s.Schema.Extensions)) +} + +func TestNoUseBuiltInSchema_AddSchema(t *testing.T) { + // reset package vars + globalSchema = openapiData{} + + SuppressBuiltInSchemaUse() + _, err := AddSchema(additionalSchema) + if !assert.NoError(t, err) { + t.FailNow() + } + s, err := GetSchema(`{"$ref": "#/definitions/io.k8s.config.setters.replicas"}`) + if !assert.Greater(t, len(globalSchema.schema.Definitions), 1) { + t.FailNow() + } + if !assert.NoError(t, err) { + t.FailNow() + } + assert.Equal(t, `map[x-kustomize:map[setBy:Jane setter:map[name:replicas value:5]]]`, + fmt.Sprintf("%v", s.Schema.Extensions)) +} + +var additionalSchema = []byte(` +{ + "definitions": { + "io.k8s.config.setters.replicas": { + "description": "replicas description.", + "type": "integer", + "x-kustomize": {"setBy":"Jane","setter": {"name":"replicas","value":"5"}} + } + }, + "invalid": "field" +} +`) + func TestSchemaForResourceType(t *testing.T) { - s := openapi.SchemaForResourceType( + // reset package vars + globalSchema = openapiData{} + + s := SchemaForResourceType( yaml.TypeMeta{APIVersion: "apps/v1", Kind: "Deployment"}) if !assert.NotNil(t, s) { t.FailNow() From 72e7084639b0e853e74a30bfba3ca478b39cece8 Mon Sep 17 00:00:00 2001 From: Phani Teja Marupaka Date: Wed, 5 Feb 2020 09:21:45 -0800 Subject: [PATCH 45/46] Make filed optional flag for create setter --- cmd/config/internal/commands/cmdcreatesetter.go | 1 - kyaml/setters/addyaml.go | 5 +++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/config/internal/commands/cmdcreatesetter.go b/cmd/config/internal/commands/cmdcreatesetter.go index 78ebb2d27..010606c8e 100644 --- a/cmd/config/internal/commands/cmdcreatesetter.go +++ b/cmd/config/internal/commands/cmdcreatesetter.go @@ -38,7 +38,6 @@ func NewCreateSetterRunner(parent string) *CreateSetterRunner { "create a partial setter for only part of the field value.") fixDocs(parent, set) set.MarkFlagRequired("type") - set.MarkFlagRequired("field") r.Command = set return r } diff --git a/kyaml/setters/addyaml.go b/kyaml/setters/addyaml.go index 0a70fa84a..18fb7918d 100644 --- a/kyaml/setters/addyaml.go +++ b/kyaml/setters/addyaml.go @@ -55,8 +55,9 @@ func (m *customFieldSetter) Filter(object *yaml.RNode) (*yaml.RNode, error) { return node.PipeE(m) }) case yaml.ScalarNode: - // only create the setter for fields with the given name - if m.currentFieldName != m.Field { + // if filed is empty, create the setter for all fields with given value + // else only create the setter for given field and value combination, with given name + if m.Field != "" && m.currentFieldName != m.Field { return object, nil } if err := m.create(object); err != nil { From 90e1dbe5d05182d3b49b89fb7b706ca0d498c3f8 Mon Sep 17 00:00:00 2001 From: Phillip Wittrock Date: Wed, 5 Feb 2020 16:08:02 -0800 Subject: [PATCH 46/46] cmd/config: support for stdin/stdout in source/sink --- cmd/config/docs/commands/sink.md | 4 +- cmd/config/docs/commands/source.md | 5 +- cmd/config/internal/commands/sink.go | 23 ++-- cmd/config/internal/commands/sink_test.go | 117 +++++++++++++++++- cmd/config/internal/commands/source.go | 13 +- cmd/config/internal/commands/source_test.go | 64 ++++++++++ .../internal/generateddocs/commands/docs.go | 9 +- 7 files changed, 211 insertions(+), 24 deletions(-) diff --git a/cmd/config/docs/commands/sink.md b/cmd/config/docs/commands/sink.md index ed31de43e..264fb66ac 100644 --- a/cmd/config/docs/commands/sink.md +++ b/cmd/config/docs/commands/sink.md @@ -6,10 +6,10 @@ [Alpha] Implement a Sink by writing input to a local directory. - kustomize config sink DIR + kustomize config sink [DIR] DIR: - Path to local directory. + Path to local directory. If unspecified, sink will write to stdout as if it were a single file. `sink` writes its input to a directory diff --git a/cmd/config/docs/commands/source.md b/cmd/config/docs/commands/source.md index 66f12e9f8..978277b38 100644 --- a/cmd/config/docs/commands/source.md +++ b/cmd/config/docs/commands/source.md @@ -6,10 +6,11 @@ [Alpha] Implement a Source by reading a local directory. - kustomize config source DIR + kustomize config source DIR... DIR: - Path to local directory. + One or more paths to local directories. Contents from directories will be concatenated. + If no directories are provided, source will read from stdin as if it were a single file. `source` emits configuration to act as input to a function diff --git a/cmd/config/internal/commands/sink.go b/cmd/config/internal/commands/sink.go index f1b05d9d6..74e1ec8a3 100644 --- a/cmd/config/internal/commands/sink.go +++ b/cmd/config/internal/commands/sink.go @@ -7,6 +7,7 @@ import ( "github.com/spf13/cobra" "sigs.k8s.io/kustomize/cmd/config/internal/generateddocs/commands" "sigs.k8s.io/kustomize/kyaml/kio" + "sigs.k8s.io/kustomize/kyaml/kio/kioutil" ) // GetSinkRunner returns a command for Sink. @@ -18,7 +19,7 @@ func GetSinkRunner(name string) *SinkRunner { Long: commands.SinkLong, Example: commands.SinkExamples, RunE: r.runE, - Args: cobra.ExactArgs(1), + Args: cobra.MaximumNArgs(1), } fixDocs(name, c) r.Command = c @@ -35,14 +36,18 @@ type SinkRunner struct { } func (r *SinkRunner) runE(c *cobra.Command, args []string) error { + var outputs []kio.Writer + if len(args) == 1 { + outputs = []kio.Writer{&kio.LocalPackageWriter{PackagePath: args[0]}} + } else { + outputs = []kio.Writer{&kio.ByteWriter{ + Writer: c.OutOrStdout(), + ClearAnnotations: []string{kioutil.PathAnnotation}}, + } + } + err := kio.Pipeline{ - Inputs: []kio.Reader{&kio.ByteReader{Reader: c.InOrStdin()}}, - Outputs: []kio.Writer{ - &kio.LocalPackageWriter{ - PackagePath: args[0], - ClearAnnotations: []string{"config.kubernetes.io/path"}, - }, - }, - }.Execute() + Inputs: []kio.Reader{&kio.ByteReader{Reader: c.InOrStdin()}}, + Outputs: outputs}.Execute() return handleError(c, err) } diff --git a/cmd/config/internal/commands/sink_test.go b/cmd/config/internal/commands/sink_test.go index d09e9bd6f..9fc4d869c 100644 --- a/cmd/config/internal/commands/sink_test.go +++ b/cmd/config/internal/commands/sink_test.go @@ -21,8 +21,6 @@ func TestSinkCommand(t *testing.T) { } defer os.RemoveAll(d) - // fmt the files - b := &bytes.Buffer{} r := commands.GetSinkRunner("") r.Command.SetIn(bytes.NewBufferString(`apiVersion: config.kubernetes.io/v1alpha1 kind: ResourceList @@ -75,7 +73,6 @@ items: replicas: 3 `)) r.Command.SetArgs([]string{d}) - r.Command.SetOut(b) if !assert.NoError(t, r.Command.Execute()) { t.FailNow() } @@ -138,3 +135,117 @@ spec: t.FailNow() } } + +func TestSinkCommand_Stdout(t *testing.T) { + d, err := ioutil.TempDir("", "kustomize-source-test") + if !assert.NoError(t, err) { + t.FailNow() + } + defer os.RemoveAll(d) + + // fmt the files + out := &bytes.Buffer{} + r := commands.GetSinkRunner("") + r.Command.SetIn(bytes.NewBufferString(`apiVersion: config.kubernetes.io/v1alpha1 +kind: ResourceList +items: +- kind: Deployment + metadata: + labels: + app: nginx2 + name: foo + annotations: + app: nginx2 + config.kubernetes.io/index: '0' + config.kubernetes.io/path: 'f1.yaml' + spec: + replicas: 1 +- kind: Service + metadata: + name: foo + annotations: + app: nginx + config.kubernetes.io/index: '1' + config.kubernetes.io/path: 'f1.yaml' + spec: + selector: + app: nginx +- apiVersion: v1 + kind: Abstraction + metadata: + name: foo + annotations: + config.kubernetes.io/function: | + container: + image: gcr.io/example/reconciler:v1 + config.kubernetes.io/local-config: "true" + config.kubernetes.io/index: '0' + config.kubernetes.io/path: 'f2.yaml' + spec: + replicas: 3 +- apiVersion: apps/v1 + kind: Deployment + metadata: + labels: + app: nginx + name: bar + annotations: + app: nginx + config.kubernetes.io/index: '1' + config.kubernetes.io/path: 'f2.yaml' + spec: + replicas: 3 +`)) + + r.Command.SetOut(out) + r.Command.SetArgs([]string{}) + if !assert.NoError(t, r.Command.Execute()) { + t.FailNow() + } + + expected := `kind: Deployment +metadata: + labels: + app: nginx2 + name: foo + annotations: + app: nginx2 +spec: + replicas: 1 +--- +kind: Service +metadata: + name: foo + annotations: + app: nginx +spec: + selector: + app: nginx +--- +apiVersion: v1 +kind: Abstraction +metadata: + name: foo + annotations: + config.kubernetes.io/function: | + container: + image: gcr.io/example/reconciler:v1 + config.kubernetes.io/local-config: "true" +spec: + replicas: 3 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: nginx + name: bar + annotations: + app: nginx +spec: + replicas: 3 +` + if !assert.Equal(t, expected, out.String()) { + t.FailNow() + } +} diff --git a/cmd/config/internal/commands/source.go b/cmd/config/internal/commands/source.go index 5926d3af9..27382bb2f 100644 --- a/cmd/config/internal/commands/source.go +++ b/cmd/config/internal/commands/source.go @@ -21,7 +21,6 @@ func GetSourceRunner(name string) *SourceRunner { Long: commands.SourceLong, Example: commands.SourceExamples, RunE: r.runE, - Args: cobra.ExactArgs(1), } fixDocs(name, c) c.Flags().StringVar(&r.WrapKind, "wrap-kind", kio.ResourceListKind, @@ -70,8 +69,14 @@ func (r *SourceRunner) runE(c *cobra.Command, args []string) error { FunctionConfig: functionConfig, }) - err := kio.Pipeline{ - Inputs: []kio.Reader{kio.LocalPackageReader{PackagePath: args[0]}}, - Outputs: outputs}.Execute() + var inputs []kio.Reader + for _, a := range args { + inputs = append(inputs, kio.LocalPackageReader{PackagePath: a}) + } + if len(inputs) == 0 { + inputs = []kio.Reader{&kio.ByteReader{Reader: c.InOrStdin()}} + } + + err := kio.Pipeline{Inputs: inputs, Outputs: outputs}.Execute() return handleError(c, err) } diff --git a/cmd/config/internal/commands/source_test.go b/cmd/config/internal/commands/source_test.go index a4c01bb1c..141541143 100644 --- a/cmd/config/internal/commands/source_test.go +++ b/cmd/config/internal/commands/source_test.go @@ -134,3 +134,67 @@ items: return } } + +func TestSourceCommand_Stdin(t *testing.T) { + d, err := ioutil.TempDir("", "kustomize-source-test") + if !assert.NoError(t, err) { + return + } + defer os.RemoveAll(d) + + in := bytes.NewBufferString(` +kind: Deployment +metadata: + labels: + app: nginx2 + name: foo + annotations: + app: nginx2 +spec: + replicas: 1 +--- +kind: Service +metadata: + name: foo + annotations: + app: nginx +spec: + selector: + app: nginx +`) + + out := &bytes.Buffer{} + r := commands.GetSourceRunner("") + r.Command.SetArgs([]string{}) + r.Command.SetIn(in) + r.Command.SetOut(out) + if !assert.NoError(t, r.Command.Execute()) { + return + } + + if !assert.Equal(t, `apiVersion: config.kubernetes.io/v1alpha1 +kind: ResourceList +items: +- kind: Deployment + metadata: + labels: + app: nginx2 + name: foo + annotations: + app: nginx2 + config.kubernetes.io/index: '0' + spec: + replicas: 1 +- kind: Service + metadata: + name: foo + annotations: + app: nginx + config.kubernetes.io/index: '1' + spec: + selector: + app: nginx +`, out.String()) { + return + } +} diff --git a/cmd/config/internal/generateddocs/commands/docs.go b/cmd/config/internal/generateddocs/commands/docs.go index b4cfce577..7c500a51d 100644 --- a/cmd/config/internal/generateddocs/commands/docs.go +++ b/cmd/config/internal/generateddocs/commands/docs.go @@ -350,10 +350,10 @@ var SinkShort = `[Alpha] Implement a Sink by writing input to a local directory. var SinkLong = ` [Alpha] Implement a Sink by writing input to a local directory. - kustomize config sink DIR + kustomize config sink [DIR] DIR: - Path to local directory. + Path to local directory. If unspecified, sink will write to stdout as if it were a single file. ` + "`" + `sink` + "`" + ` writes its input to a directory ` @@ -364,10 +364,11 @@ var SourceShort = `[Alpha] Implement a Source by reading a local directory.` var SourceLong = ` [Alpha] Implement a Source by reading a local directory. - kustomize config source DIR + kustomize config source DIR... DIR: - Path to local directory. + One or more paths to local directories. Contents from directories will be concatenated. + If no directories are provided, source will read from stdin as if it were a single file. ` + "`" + `source` + "`" + ` emits configuration to act as input to a function `