From c7307a9b28d7ccd7caaec90b5d5cbd575bd2cdab Mon Sep 17 00:00:00 2001 From: Morten Torkildsen Date: Mon, 2 Dec 2019 14:21:45 -0800 Subject: [PATCH 01/27] Add documentation for the kstatus library --- kstatus/README.md | 115 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 115 insertions(+) create mode 100644 kstatus/README.md diff --git a/kstatus/README.md b/kstatus/README.md new file mode 100644 index 000000000..4c3a776b7 --- /dev/null +++ b/kstatus/README.md @@ -0,0 +1,115 @@ +# kstatus + +kstatus provides tools for checking the status of Kubernetes resources. The primary use case is knowing when +(or if) a given set of resources in cluster has successfully reconciled an apply operation. + +## Concepts + +This effort has several goals, some with a shorter timeline than others. Initially, we want to provide +a library that makes it easier to decide when changes to a set of resources have been reconciled in a cluster. +To support types that do not yet publish status information, we will initially fallback on type specific rules. +The library already contains rules for many of the most common built-in types such a Deployment and StatefulSet. + +For custom resource definitions (CRDs), there currently isn't much guidance on which properties should be exposed in the status +object and which conditions should be used. As part of this effort we want to define a set of standard conditions +that the library will understand and that we encourage developers to adopt in their CRDs. These standard conditions will +be focused on providing the necessary information for understanding status of the reconcile after `apply` and it is not +expected that these will necessarily be the only conditions exposed in a custom resource. Developers will be free to add as many conditions +as they wish, but if the CRDs adopt the standard conditions defined here, this library will handle them correctly. + +The `status` objects for built-in types don't all conform to a common behavior. Not all built-in types expose conditions, +and even among the types that does, the types of conditions vary widely. Long-term, we hope to add support for the +standard conditions to the built-in types as well. This would remove the need for type-specific rules for determining +status. + +### Statuses + +The library currently defines the following statuses for resource: +* __InProgress__: The actual state of the resource has not yet reached the desired state as specified in the +resource manifest, i.e. the resource reconcile has not yet completed. Newly created resources will usually +start with this status, although some resources like ConfigMaps are Current immediately. +* __Failed__: The process of reconciling the actual state with the desired state has encountered and error +or it has made insufficient progress. +* __Current__: The actual state of the resource matches the desired state. The reconcile process is considered +complete until there are changes to either the desired or the actual state. +* __Terminating__: The resource is in the process of being deleted. +* __Unknown__: This is for situations when the library are unable to determine the status of a resource. + +### Conditions + +The conditions defined in the library are designed to adhere to the "abnormal-true" pattern, i.e. that +conditions are present and with a value of true whenever something unusual happens. So the absence of +any conditions means everything is normal. Normal in this situation simply means that the latest observed +generation of the resource manifest by the controller have been fully reconciled with the actual state. + +* __InProgress__: The controller is currently working on reconciling the latest changes. +* __Failed__: The controller has encountered an error during the reconcile process or it has made +insufficient progress (timeout). + +The use of the "abnormal-true" pattern has some challenges. If the controller is not running, or for some +reason not able to update the resource, it will look like it is in a good state when that is not true. The +solution to this issue is to adopt the pattern used by several of the built-in types where there is an +`observedGeneration` property on the status object which is set by the controller during the reconcile loop. +If the `generation` and the `observedGeneration` of a resource does not match, it means there are changes +that the controller has not yet seen, and therefore not acted upon. + +## Features + +The library is currently separated into two packages, one that provides the basic functionality, and another that +builds upon the basics to provide a higher level API. + +**sigs.k8s.io/kustomize/kstatus/status**: Provides two basic functions. First, it provides the `Compute` function +that takes a single resource and computes the status for this resource based on the fields in the status object for +the resource. Second, it provides the `Augment` function that computes the appropriate standard conditions based on +the status object and then amends them to the conditions in the resource. Both of these functions currently operate +on Unstructured types, but this should eventually be changed to rely on the kyaml library. Both of these functions +compute the status and conditions solely based on the data in the resource passed in. It does not communicate with +a cluster to get the latest state of the resources. + +**sigs.k8s.io/kustomize/kstatus/wait**: This package builds upon the status package and provides functionality that +will fetch the latest state from a cluster. It provides the `FetchAndResolve` function that takes list of resource +identifiers, fetches the latest state for all the resources from the cluster, computes the status for all of them and +returns the results. `WaitForStatus` accepts a list of resource identifiers and will poll cluster for the status of +the resources until all resources have reached the `Current` status. + +## Challenges + +### Status is not obvious for all resource types + +For some types of resources, it is pretty clear what the different statuses mean. For others, it +is far less obvious. For example, what does it mean that a PodDisruptionBudget is Current? Based on +the assumptions above it probably should be whenever the controller has observed the resource +and updated the status object of the PDB with information on allowed disruptions. But currently, a PDB is +considered Current when the number of healthy replicas meets the threshold given in the PDB. Also, should +the presence of a PDB influence when a Deployment is considered Current? This would mean that a Deployment +should be considered Current whenever the number of replicas reach the threshold set by the corresponding +PDB. This is not currently supported as described below. + +### Status is decided based on single resource +Currently the status of a resource is decided solely based on information from +the state of that resource. This is an issue for resources that create other resources +and that doesn't provide sufficient information within their own status object. An example +is the Service resource that doesn't provide much status information but do generate Endpoint +resources that could be used to determine status. Similar, the status of a Deployment could be +based on its generated ReplicaSets and Pods. + +Not having the generated resources also limits the amount of details that can be provided +when something isn't working as expected. + + +## Future + +### Depend on kyaml instead of k8s libraries +The sigs.k8s.io/kustomize/kstatus/status package currently depends on k8s libraries. This can be +challenging if someone wants to vendor the library within their own project. We want to replace +the dependencies on k8s libraries with kyaml for the status package. The wait package needs to +talk to a k8s cluster, so this package will continue to rely on the k8s libraries. + +### Use watches instead of polling + +We currently poll for updates to resources, but it would be possible to set up +watches instead. This could also be combined with deciding status based on not only a single +resource, but also all its generated resources. This would lead to a design that seems similar +to a controller, so maybe a solution like this could be built on top of controller-runtime. +A challenge here is that the rules for each built-in type would need to be expressed in a different +way that what we currently do. \ No newline at end of file From e0597683d47a74e2d1c924e4630b95536a73ce30 Mon Sep 17 00:00:00 2001 From: guoxudong Date: Fri, 13 Dec 2019 09:59:24 +0800 Subject: [PATCH 02/27] Update zh/INSTALL.md --- docs/zh/INSTALL.md | 94 +++++++++++++++++++++++++++++++++------------- 1 file changed, 67 insertions(+), 27 deletions(-) diff --git a/docs/zh/INSTALL.md b/docs/zh/INSTALL.md index 8e7ec50ba..bb370fcfb 100644 --- a/docs/zh/INSTALL.md +++ b/docs/zh/INSTALL.md @@ -1,39 +1,79 @@ [release 页面]: /../../releases [Go]: https://golang.org -[golang.org]: https://golang.org +[脚本]: https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh +[快速开始]: https://www.arp242.net/curl-to-sh.html ## 安装 -在 macOS ，您可以使用软件包管理器 Homebrew 来安装 kustomize 。 +适用于 Linux、MacOS 和 Windows 的各版本的二进制可执行文件可以在 [release 页面] 上手动下载。 - brew install kustomize +如果希望[快速开始]，可以执行: -在 windows ，您可以使用软件包管理器 Chocolatey 来安装 kustomize 。 +```bash +curl -s "https://raw.githubusercontent.com/\ +kubernetes-sigs/kustomize/master/hack/install_kustomize.sh" | bash +``` - choco install kustomize +这个[脚本]会： + +- 尝试检测您的操作系统 +- 在临时目录中下载并解压 tar 文件 +- 将 kustomize 二进制可执行文件复制到您当前的工作目录中 +- 删除临时目录 + +## 尝试 `go` + +这种方式只是为了更好的展示 kustomize 作为 `go` 语言工具的工作原理。开发者应该拉取 repo（详见下一部分），而 CI/CD 脚本中应下载随时可以运行的可执行文件，而不要依赖 `go` 语言工具。 + +安装 v3 版本中最新的 kustomize 到 `$GOPATH/bin`: + +```bash +GO111MODULE=on go install sigs.k8s.io/kustomize/kustomize/v3 +``` + +安装指定版本 + +```bash +GO111MODULE=on go get sigs.k8s.io/kustomize/kustomize/v3@v3.3.0 +``` + +## 本地源码构建 kustomize CLI + +```bash +# 需要 go 1.13 或更高版本 +unset GOPATH +# 详见 https://golang.org/doc/go1.13#modules +unset GO111MODULES + +# 拉取 repo +git clone git@github.com:kubernetes-sigs/kustomize.git +# 进入目录 +cd kustomize + +# 如果您不想从头开始构建，则可以选择 checkout 特定的标签 +git checkout kustomize/v3.2.3 + +# 开始构建 +(cd kustomize; go install .) + +# 运行 +~/go/bin/kustomize version +``` + +### 其他方式 + +#### macOS + +```bash +brew install kustomize +``` + +#### windows + +```bash +choco install kustomize +``` 有关软件包管理器 chocolatey 的使用以及对之前版本的支持，请参考以下链接： - [Choco Package](https://chocolatey.org/packages/kustomize) - [Package Source](https://github.com/kenmaglio/choco-kustomize) - -对于其他系统，请在 [release 页面] 下载相应系统的二进制文件。 - -或者使用命令行获取最新的官方版本： - -``` -opsys=linux # or darwin, or windows -curl -s https://api.github.com/repos/kubernetes-sigs/kustomize/releases/latest |\ - grep browser_download |\ - grep $opsys |\ - cut -d '"' -f 4 |\ - xargs curl -O -L -mv kustomize_*_${opsys}_amd64 kustomize -chmod u+x kustomize -``` - -使用 [Go] v1.10.1 或更高版本安装（如果可以访问 [golang.org]）： - - -``` -go install sigs.k8s.io/kustomize/kustomize -``` From 68af986e091abc97f10dde84dfdf92e68c0eb1ee Mon Sep 17 00:00:00 2001 From: bzub Date: Thu, 12 Dec 2019 19:44:03 -0600 Subject: [PATCH 03/27] Add kubeval function example. --- .../validator-kubeval/LICENSE_TEMPLATE | 2 + functions/examples/validator-kubeval/Makefile | 42 ++++++ .../examples/validator-kubeval/README.md | 44 ++++++ .../validator-kubeval/image/Dockerfile | 17 +++ .../examples/validator-kubeval/image/go.mod | 8 ++ .../examples/validator-kubeval/image/go.sum | 68 +++++++++ .../examples/validator-kubeval/image/main.go | 134 ++++++++++++++++++ .../local-resource/example-use.yaml | 33 +++++ 8 files changed, 348 insertions(+) create mode 100644 functions/examples/validator-kubeval/LICENSE_TEMPLATE create mode 100644 functions/examples/validator-kubeval/Makefile create mode 100644 functions/examples/validator-kubeval/README.md create mode 100644 functions/examples/validator-kubeval/image/Dockerfile create mode 100644 functions/examples/validator-kubeval/image/go.mod create mode 100644 functions/examples/validator-kubeval/image/go.sum create mode 100644 functions/examples/validator-kubeval/image/main.go create mode 100644 functions/examples/validator-kubeval/local-resource/example-use.yaml diff --git a/functions/examples/validator-kubeval/LICENSE_TEMPLATE b/functions/examples/validator-kubeval/LICENSE_TEMPLATE new file mode 100644 index 000000000..0c2b3b655 --- /dev/null +++ b/functions/examples/validator-kubeval/LICENSE_TEMPLATE @@ -0,0 +1,2 @@ +Copyright {{.Year}} {{.Holder}} +SPDX-License-Identifier: Apache-2.0 diff --git a/functions/examples/validator-kubeval/Makefile b/functions/examples/validator-kubeval/Makefile new file mode 100644 index 000000000..b8030c006 --- /dev/null +++ b/functions/examples/validator-kubeval/Makefile @@ -0,0 +1,42 @@ +# Copyright 2019 The Kubernetes Authors. +# SPDX-License-Identifier: Apache-2.0 + +.PHONY: generate license fix vet fmt test build tidy image + +GOBIN := $(shell go env GOPATH)/bin + +build: + (cd image && go build -v -o $(GOBIN)/config-function .) + +all: generate license build fix vet fmt test lint tidy + +fix: + (cd image && go fix ./...) + +fmt: + (cd image && go fmt ./...) + +generate: + (which $(GOBIN)/mdtogo || go get sigs.k8s.io/kustomize/cmd/mdtogo) + (cd image && GOBIN=$(GOBIN) go generate ./...) + +license: + (which $(GOPATH)/bin/addlicense || go get github.com/google/addlicense) + $(GOPATH)/bin/addlicense -y 2019 -c "The Kubernetes Authors." -f LICENSE_TEMPLATE . + +tidy: + (cd image && go mod tidy) + +lint: + (which $(GOBIN)/golangci-lint || go get github.com/golangci/golangci-lint/cmd/golangci-lint@v1.19.1) + (cd image && $(GOBIN)/golangci-lint run ./...) + +test: + (cd image && go test -cover ./...) + +vet: + (cd image && go vet ./...) + +image: + docker build image -t gcr.io/kustomize-functions/example-validator-kubeval:v0.1.0 + docker push gcr.io/kustomize-functions/example-validator-kubeval:v0.1.0 diff --git a/functions/examples/validator-kubeval/README.md b/functions/examples/validator-kubeval/README.md new file mode 100644 index 000000000..aabcb87f3 --- /dev/null +++ b/functions/examples/validator-kubeval/README.md @@ -0,0 +1,44 @@ +# Validation + +This is an example of implementing a validation function against +[kubeval](https://github.com/instrumenta/kubeval). + +## Function implementation + +The function is implemented as an [image](image), and built using `make image`. + +The function is implemented as a go program, which reads a collection of input +Resource configuration, passing each to kubeval. + +### Function configuration + +A number of settings can be modified for `kubeval` in the function `spec`. See +the `API` struct definition in [main.go](image/main.go) for documentation. + +## Function invocation + +The function is invoked by authoring a [local Resource](local-resource) +with `metadata.configFn` and running: + + kustomize config run local-resource/ + +This exists non-zero if kubeval detects an invalid Resource. + +## Running the Example + +Run the validator with: + + kustomize config run local-resource/ + +This will return an error: + + Resource invalid: (Kind: Service, Name: svc) + prots: Additional property prots is not allowed + Error: exit status 1 + +Now fix the typo in [example-use.yaml](local-resource/example-use.yaml) and +run: + + kustomize config run local-resource/ + +This will return success (no output). diff --git a/functions/examples/validator-kubeval/image/Dockerfile b/functions/examples/validator-kubeval/image/Dockerfile new file mode 100644 index 000000000..7181aa545 --- /dev/null +++ b/functions/examples/validator-kubeval/image/Dockerfile @@ -0,0 +1,17 @@ +FROM alpine:latest as schemas +RUN apk --no-cache add git +RUN git clone --depth 1 https://github.com/instrumenta/kubernetes-json-schema.git + +FROM golang:1.13-stretch as function +ENV CGO_ENABLED=0 +WORKDIR /go/src/ +COPY go.mod . +COPY go.sum . +RUN go mod download +COPY main.go . +RUN go build -v -o /usr/local/bin/config-function ./ + +FROM alpine:latest +COPY --from=schemas /kubernetes-json-schema/v1.16.0-standalone-strict /schemas/v1.16.0-standalone-strict +COPY --from=function /usr/local/bin/config-function /usr/local/bin/config-function +CMD ["config-function"] diff --git a/functions/examples/validator-kubeval/image/go.mod b/functions/examples/validator-kubeval/image/go.mod new file mode 100644 index 000000000..9cc966038 --- /dev/null +++ b/functions/examples/validator-kubeval/image/go.mod @@ -0,0 +1,8 @@ +module sigs.k8s.io/kustomize/functions/examples/validator-kubeval + +go 1.13 + +require ( + github.com/instrumenta/kubeval v0.0.0-20190918223246-8d013ec9fc56 + sigs.k8s.io/kustomize/kyaml v0.0.0-20191212230447-6309af43a718 +) diff --git a/functions/examples/validator-kubeval/image/go.sum b/functions/examples/validator-kubeval/image/go.sum new file mode 100644 index 000000000..83c84467c --- /dev/null +++ b/functions/examples/validator-kubeval/image/go.sum @@ -0,0 +1,68 @@ +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys= +github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/go-errors/errors v1.0.1 h1:LUHzmkK3GUKUrL/1gfBUxAHzcev3apQlezX/+O7ma6w= +github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= +github.com/hashicorp/errwrap v0.0.0-20180715044906-d6c0cd880357 h1:Rem2+U35z1QtPQc6r+WolF7yXiefXqDKyk+lN2pE164= +github.com/hashicorp/errwrap v0.0.0-20180715044906-d6c0cd880357/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-multierror v0.0.0-20180717150148-3d5d8f294aa0 h1:j30noezaCfvNLcdMYSvHLv81DxYRSt1grlpseG67vhU= +github.com/hashicorp/go-multierror v0.0.0-20180717150148-3d5d8f294aa0/go.mod h1:JMRHfdO9jKNzS/+BTlxCjKNQHg/jZAft8U7LloJvN7I= +github.com/hashicorp/hcl v0.0.0-20180404174102-ef8a98b0bbce/go.mod h1:oZtUIOe8dh44I2q6ScRibXws4Ajl+d+nod3AaR9vL5w= +github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/instrumenta/kubeval v0.0.0-20190918223246-8d013ec9fc56 h1:kKOrEaxR9KvCDdnQqjiBxbaeJg/goLvJvW0lno6aWm4= +github.com/instrumenta/kubeval v0.0.0-20190918223246-8d013ec9fc56/go.mod h1:bpiMYvNpVxWjdJsS0hDRu9TrobT5GfWCZwJseGUstxE= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/mattn/go-colorable v0.1.0 h1:v2XXALHHh6zHfYTJ+cSkwtyffnaOyR1MXaA91mTrb8o= +github.com/mattn/go-colorable v0.1.0/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-isatty v0.0.4 h1:bnP0vzxcAdeI1zdubAl5PjU6zsERjGZb7raWodagDYs= +github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mitchellh/mapstructure v0.0.0-20180715050151-f15292f7a699/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/pelletier/go-toml v0.0.0-20180724185102-c2dbbc24a979/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/spf13/afero v1.1.1/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/cast v1.2.0/go.mod h1:r2rcYCSwa1IExKTDiTfzaxqT2FNHs8hODu4LnUfgKEg= +github.com/spf13/cobra v0.0.0-20180820174524-ff0d02e85550 h1:LB9SHuuXO8gnsHtexOQSpsJrrAHYA35lvHUaE74kznU= +github.com/spf13/cobra v0.0.0-20180820174524-ff0d02e85550/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= +github.com/spf13/jwalterweatherman v0.0.0-20180814060501-14d3d4c51834/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/pflag v0.0.0-20180821114517-d929dcbb1086 h1:iU+nPfqRqK8ShQqnpZLv8cZ9oklo6NFUcmX1JT5Rudg= +github.com/spf13/pflag v0.0.0-20180821114517-d929dcbb1086/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/viper v1.1.0/go.mod h1:A8kyI5cUJhb8N+3pkfONlcEcZbueH6nhAm0Fq7SrnBM= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f h1:J9EGpcZtP0E/raorCMxlFGSTBrsSlaDGf3jU/qvAE2c= +github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= +github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0= +github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= +github.com/xeipuuv/gojsonschema v0.0.0-20180816142147-da425ebb7609 h1:BcMExZAULPkihVZ7UJXK7t8rwGqisXFw75tILnafhBY= +github.com/xeipuuv/gojsonschema v0.0.0-20180816142147-da425ebb7609/go.mod h1:5yf86TLmAcydyeJq5YvxkGPE2fm/u4myDekKRoLuqhs= +github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca h1:1CFlNzQhALwjS9mBAUkycX616GzgsuYUOCHA5+HSlXI= +github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg= +golang.org/x/sys v0.0.0-20180821044426-4ea2f632f6e9 h1:0RHCP7KEw0rDuVXXaT2gfV77uu6lTKa5aItB+EoFbQk= +golang.org/x/sys v0.0.0-20180821044426-4ea2f632f6e9/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/text v0.0.0-20180810153555-6e3c4e7365dd/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20191026110619-0b21df46bc1d h1:LCPbGQ34PMrwad11aMZ+dbz5SAsq/0ySjRwQ8I9Qwd8= +gopkg.in/yaml.v3 v3.0.0-20191026110619-0b21df46bc1d/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +sigs.k8s.io/kustomize/kyaml v0.0.0-20191212230447-6309af43a718 h1:Kbn9yTkj4eMwY/LQLXnwDm4eTLqQsaOL81iSsr/UhQU= +sigs.k8s.io/kustomize/kyaml v0.0.0-20191212230447-6309af43a718/go.mod h1:rywm/rcR5LmCBghz9956tE45OdUPChFoXVVs+WmhMTI= +sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs= +sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= diff --git a/functions/examples/validator-kubeval/image/main.go b/functions/examples/validator-kubeval/image/main.go new file mode 100644 index 000000000..cc178c187 --- /dev/null +++ b/functions/examples/validator-kubeval/image/main.go @@ -0,0 +1,134 @@ +// Copyright 2019 The Kubernetes Authors. +// SPDX-License-Identifier: Apache-2.0 + +// Package main implements a validator function run by `kustomize config run` +package main + +import ( + "errors" + "fmt" + "os" + "strings" + + "github.com/instrumenta/kubeval/kubeval" + "sigs.k8s.io/kustomize/kyaml/kio" + "sigs.k8s.io/kustomize/kyaml/yaml" +) + +func main() { + rw := &kio.ByteReadWriter{ + Reader: os.Stdin, + Writer: os.Stdout, + OmitReaderAnnotations: true, + KeepReaderAnnotations: true, + } + p := kio.Pipeline{ + Inputs: []kio.Reader{rw}, // read the inputs into a slice + Filters: []kio.Filter{kubevalFilter{rw: rw}}, + Outputs: []kio.Writer{rw}, // copy the inputs to the output + } + if err := p.Execute(); err != nil { + fmt.Fprintf(os.Stderr, "%v\n", err) + os.Exit(1) + } +} + +// kubevalFilter implements kio.Filter +type kubevalFilter struct { + rw *kio.ByteReadWriter +} + +// define the input API schema as a struct +type API struct { + Spec struct { + // Strict disallows additional properties not in schema if set. + Strict bool `yaml:"strict"` + + // IgnoreMissingSchemas skips validation for resource + // definitions without a schema. + IgnoreMissingSchemas bool `yaml:"ignoreMissingSchemas"` + + // KubernetesVersion is the version of Kubernetes to validate + // against (default "master"). + KubernetesVersion string `yaml:"kubernetesVersion"` + + // SchemaLocation is the base URL used to download schemas. + SchemaLocation string `yaml:"schemaLocation"` + } `yaml:"spec"` +} + +// Filter checks each resource for validity, otherwise returning an error. +func (f kubevalFilter) Filter(in []*yaml.RNode) ([]*yaml.RNode, error) { + api := f.parseAPI() + config := kubeval.NewDefaultConfig() + config.Strict = api.Spec.Strict + config.IgnoreMissingSchemas = api.Spec.IgnoreMissingSchemas + config.KubernetesVersion = api.Spec.KubernetesVersion + config.SchemaLocation = api.Spec.SchemaLocation + + // validate each Resource + for _, r := range in { + if err := validate(r.MustString(), config); err != nil { + meta, merr := r.GetMeta() + if merr != nil { + return nil, merr + } + fmt.Fprintf( + os.Stderr, + "Resource invalid: (Kind: %s, Name: %s)\n", + meta.Kind, meta.Name, + ) + return nil, err + } + } + return in, nil +} + +func validate(r string, config *kubeval.Config) error { + results, err := kubeval.Validate([]byte(r), config) + if err != nil { + return err + } + + return checkResults(results) +} + +func checkResults(results []kubeval.ValidationResult) error { + if len(results) == 0 { + return nil + } + + errs := []string{} + for _, r := range results { + for _, e := range r.Errors { + // Workaround a bug where the + // "config.kubernetes.io/index" annotation value is a + // number (invalid), and is still set on the Resource + // regardless of OmitReaderAnnotations. + // TODO: Remove this once the above issues are + // resolved. + if e.String() == "metadata.annotations: Invalid type. Expected: [string,null], given: integer" { + continue + } + errs = append(errs, e.String()) + } + } + + if len(errs) > 0 { + return errors.New(strings.Join(errs, "\n")) + } + + return nil +} + +// parseAPI parses the functionConfig into an API struct. +func (f *kubevalFilter) parseAPI() API { + // parse the input function config -- TODO: simplify this + var api API + if err := yaml.Unmarshal([]byte(f.rw.FunctionConfig.MustString()), &api); err != nil { + fmt.Fprintf(os.Stderr, "%v\n", err) + os.Exit(1) + } + + return api +} diff --git a/functions/examples/validator-kubeval/local-resource/example-use.yaml b/functions/examples/validator-kubeval/local-resource/example-use.yaml new file mode 100644 index 000000000..3ec77bae7 --- /dev/null +++ b/functions/examples/validator-kubeval/local-resource/example-use.yaml @@ -0,0 +1,33 @@ +# Copyright 2019 The Kubernetes Authors. +# SPDX-License-Identifier: Apache-2.0 + +apiVersion: examples.config.kubernetes.io/v1beta1 +kind: Kubeval +metadata: + configFn: + container: + image: gcr.io/kustomize-functions/example-validator-kubeval:v0.1.0 +spec: + strict: true + ignoreMissingSchemas: true + + # TODO: Remove these once function container network/volumes features are + # stabilized. + # Relevant issues: + # - https://github.com/kubernetes-sigs/kustomize/issues/1901 + # - https://github.com/kubernetes-sigs/kustomize/issues/1902 + kubernetesVersion: "1.16.0" + schemaLocation: "file:///schemas" +--- +apiVersion: v1 +kind: Service +metadata: + name: svc +spec: + clusterIP: None + publishNotReadyAddresses: true + prots: # Fix this typo ("ports") to pass validation. + - port: 2380 + name: etcd-server-ssl + - port: 2379 + name: etcd-client-ssl From 46f1f1b5da2cf441e848ca4a791a949e8cb260f3 Mon Sep 17 00:00:00 2001 From: guoxudong Date: Sat, 14 Dec 2019 11:02:08 +0800 Subject: [PATCH 04/27] fix --- docs/zh/INSTALL.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/zh/INSTALL.md b/docs/zh/INSTALL.md index bb370fcfb..585aa011f 100644 --- a/docs/zh/INSTALL.md +++ b/docs/zh/INSTALL.md @@ -23,9 +23,9 @@ kubernetes-sigs/kustomize/master/hack/install_kustomize.sh" | bash ## 尝试 `go` -这种方式只是为了更好的展示 kustomize 作为 `go` 语言工具的工作原理。开发者应该拉取 repo（详见下一部分），而 CI/CD 脚本中应下载随时可以运行的可执行文件，而不要依赖 `go` 语言工具。 +这种方式只是为了更好的展示如何使用 `go` 语言来安装 kustomize。实际使用中，我们并不推荐此方法。kustomize 的开发者应该拉取此 repo（详见下一部分），而 CI/CD 脚本中应直接下载可执行文件，而不要依赖 `go` 语言工具。 -安装 v3 版本中最新的 kustomize 到 `$GOPATH/bin`: +将 kustomize 的最新版本 v3 安装到 `$GOPATH/bin`: ```bash GO111MODULE=on go install sigs.k8s.io/kustomize/kustomize/v3 @@ -50,7 +50,7 @@ git clone git@github.com:kubernetes-sigs/kustomize.git # 进入目录 cd kustomize -# 如果您不想从头开始构建，则可以选择 checkout 特定的标签 +# 如果您不想从 HEAD 开始构建， 则可以选择切换特定的标签 git checkout kustomize/v3.2.3 # 开始构建 From 60188ebe202414cb1d04dec0650c1a27c94e9a35 Mon Sep 17 00:00:00 2001 From: Phillip Wittrock Date: Mon, 16 Dec 2019 16:30:08 -0800 Subject: [PATCH 05/27] Address issues in release scripts - Merge in changes to branches - Get rid of releaseall.sh -- it is too error prone right now - Support for releasing pluginator - Automatically release binaries for kustomize --- releasing/VERSIONS | 10 +++++++--- releasing/releaseallmodules.sh | 21 --------------------- releasing/releasemodule.sh | 28 +++++++++++++++++++++------- 3 files changed, 28 insertions(+), 31 deletions(-) delete mode 100755 releasing/releaseallmodules.sh diff --git a/releasing/VERSIONS b/releasing/VERSIONS index e35ba377d..7ec4de9e6 100644 --- a/releasing/VERSIONS +++ b/releasing/VERSIONS @@ -6,7 +6,7 @@ # kyaml version export kyaml_major=0 export kyaml_minor=0 -export kyaml_patch=1 +export kyaml_patch=2 # kustomize api version export api_major=0 @@ -16,14 +16,18 @@ export api_patch=0 # cmd/config version export cmd_config_major=0 export cmd_config_minor=0 -export cmd_config_patch=1 +export cmd_config_patch=2 # cmd/kubectl version export cmd_kubectl_major=0 export cmd_kubectl_minor=0 -export cmd_kubectl_patch=1 +export cmd_kubectl_patch=2 # kustomize version export kustomize_major=3 export kustomize_minor=5 export kustomize_patch=1 + +export pluginator_major=2 +export pluginator_minor=1 +export pluginator_patch=0 diff --git a/releasing/releaseallmodules.sh b/releasing/releaseallmodules.sh deleted file mode 100755 index 87a146f60..000000000 --- a/releasing/releaseallmodules.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash -# Copyright 2019 The Kubernetes Authors. -# SPDX-License-Identifier: Apache-2.0 - -set -e - -# fetch upstream once -git fetch upstream -export FETCH="false" - -# release modules without binaries -for module in "kyaml api cmd/config cmd/kubectl" -do - releasing/releasemodule.sh $module -done - -# release modules with binaries -for binary in "kustomize" -do - BINARY=true releasing/releasemodule.sh $binary -done diff --git a/releasing/releasemodule.sh b/releasing/releasemodule.sh index 4d1096d4c..cde4b7a33 100755 --- a/releasing/releasemodule.sh +++ b/releasing/releasemodule.sh @@ -4,11 +4,8 @@ # run this script with releasing/releasemodule.sh MODULE # -- e.g. releasing/releasemodule.sh cmd/config -# to push the latest tag to release a binary, run with BINARY=true -# -- e.g. BINARY=true releasing/releasemodule.sh kustomize # to skip fetch from upstream, run with FETCH=false # -- e.g. FETCH=false releasing/releasemodule.sh kyaml -# for a list of modules see releasing/releaseallmodules.sh set -e # perform release for a module @@ -28,11 +25,14 @@ function releaseModule { # create a temporary workspace for our work wktree=$(mktemp -d /tmp/kustomize-releases-XXXXXX) - git branch -f $branch upstream/master # always release from master + git branch $branch upstream/$branch git worktree add $wktree $branch # create a separate worktree for the branch pushd . cd $wktree/$module # cd into the worktree/module + # merge master changes into the release branch + git merge upstream/master + echo "dir: $wktree" echo "module: $module v$major.$minor.$patch" echo "branch: $branch" @@ -81,8 +81,22 @@ function releaseBinary { fi } +modules="kyaml api cmd/config cmd/kubectl pluginator kustomize" + # configure the branch and tag names -module="${1?must provide the module to release as an argument}" +module="${1?must provide the module to release as an argument: supported modules [$modules]}" + +# verify the module +found=false +for m in $modules; do + if [ "$m" == "$module" ]; then + found=true + fi +done +if [ "$found" != "true" ]; then + echo "unknown module \"$module\", must be one of: [$modules]" + exit 1 +fi # get the release versions source releasing/VERSIONS @@ -98,7 +112,7 @@ fi # release the module releaseModule $module -# release the binary -if [ "$BINARY" == "true" ]; then +# release the kustomize binary if the module is kustomize +if [ "$module" == "kustomize" ]; then releaseBinary $module fi From eaaefc128f2454b1877365a8965738482f64daaf Mon Sep 17 00:00:00 2001 From: Phillip Wittrock Date: Mon, 16 Dec 2019 17:44:45 -0800 Subject: [PATCH 06/27] Using single quote style when setting annotations Annotations must be strings. Use single-quote style so tools don't get confused about the type. --- cmd/config/internal/commands/cat_test.go | 36 ++++---- cmd/config/internal/commands/cmdwrap_test.go | 12 +-- cmd/config/internal/commands/grep_test.go | 16 ++-- cmd/config/internal/generateddocs/api/docs.go | 10 ++- kyaml/fieldmeta/fieldmeta.go | 3 + kyaml/kio/byteio_reader_test.go | 24 ++--- kyaml/kio/example_test.go | 4 +- kyaml/kio/filters/container_test.go | 16 ++-- kyaml/kio/filters/filters_test.go | 24 ++--- kyaml/kio/pkgio_reader_test.go | 90 +++++++++---------- kyaml/yaml/fns_test.go | 4 +- kyaml/yaml/kfns.go | 6 +- 12 files changed, 127 insertions(+), 118 deletions(-) diff --git a/cmd/config/internal/commands/cat_test.go b/cmd/config/internal/commands/cat_test.go index c1c00aae1..c02b8f200 100644 --- a/cmd/config/internal/commands/cat_test.go +++ b/cmd/config/internal/commands/cat_test.go @@ -90,8 +90,8 @@ metadata: name: foo annotations: app: nginx2 - config.kubernetes.io/package: . - config.kubernetes.io/path: f1.yaml + config.kubernetes.io/package: '.' + config.kubernetes.io/path: 'f1.yaml' spec: replicas: 1 --- @@ -100,8 +100,8 @@ metadata: name: foo annotations: app: nginx - config.kubernetes.io/package: . - config.kubernetes.io/path: f1.yaml + config.kubernetes.io/package: '.' + config.kubernetes.io/path: 'f1.yaml' spec: selector: app: nginx @@ -196,8 +196,8 @@ metadata: name: foo annotations: app: nginx2 - config.kubernetes.io/package: . - config.kubernetes.io/path: f1.yaml + config.kubernetes.io/package: '.' + config.kubernetes.io/path: 'f1.yaml' spec: replicas: 1 --- @@ -206,8 +206,8 @@ metadata: name: foo annotations: app: nginx - config.kubernetes.io/package: . - config.kubernetes.io/path: f1.yaml + config.kubernetes.io/package: '.' + config.kubernetes.io/path: 'f1.yaml' spec: selector: app: nginx @@ -233,8 +233,8 @@ metadata: name: bar annotations: app: nginx - config.kubernetes.io/package: . - config.kubernetes.io/path: f2.yaml + config.kubernetes.io/package: '.' + config.kubernetes.io/path: 'f2.yaml' spec: replicas: 3 `, b.String()) { @@ -414,8 +414,8 @@ metadata: name: foo annotations: app: nginx2 - config.kubernetes.io/package: . - config.kubernetes.io/path: f1.yaml + config.kubernetes.io/package: '.' + config.kubernetes.io/path: 'f1.yaml' spec: replicas: 1 --- @@ -424,8 +424,8 @@ metadata: name: foo annotations: app: nginx - config.kubernetes.io/package: . - config.kubernetes.io/path: f1.yaml + config.kubernetes.io/package: '.' + config.kubernetes.io/path: 'f1.yaml' spec: selector: app: nginx @@ -536,8 +536,8 @@ metadata: name: foo annotations: app: nginx2 - config.kubernetes.io/package: . - config.kubernetes.io/path: f1.yaml + config.kubernetes.io/package: '.' + config.kubernetes.io/path: 'f1.yaml' spec: replicas: 1 --- @@ -546,8 +546,8 @@ metadata: name: foo annotations: app: nginx - config.kubernetes.io/package: . - config.kubernetes.io/path: f1.yaml + config.kubernetes.io/package: '.' + config.kubernetes.io/path: 'f1.yaml' spec: selector: app: nginx diff --git a/cmd/config/internal/commands/cmdwrap_test.go b/cmd/config/internal/commands/cmdwrap_test.go index 0424f0120..affa88bf1 100644 --- a/cmd/config/internal/commands/cmdwrap_test.go +++ b/cmd/config/internal/commands/cmdwrap_test.go @@ -78,7 +78,7 @@ items: name: test app: nginx annotations: - config.kubernetes.io/index: 0 + config.kubernetes.io/index: "0" config.kubernetes.io/path: config/test_deployment.yaml spec: replicas: 11 @@ -109,7 +109,7 @@ items: name: test app: nginx annotations: - config.kubernetes.io/index: 0 + config.kubernetes.io/index: "0" config.kubernetes.io/path: config/test_service.yaml spec: selector: @@ -133,7 +133,7 @@ items: name: test app: nginx annotations: - config.kubernetes.io/index: 0 + config.kubernetes.io/index: "0" config.kubernetes.io/path: config/test_deployment.yaml spec: replicas: 11 @@ -161,7 +161,7 @@ items: name: test app: nginx annotations: - config.kubernetes.io/index: 0 + config.kubernetes.io/index: "0" config.kubernetes.io/path: config/test_service.yaml spec: selector: @@ -185,7 +185,7 @@ items: name: test app: nginx annotations: - config.kubernetes.io/index: 0 + config.kubernetes.io/index: "0" config.kubernetes.io/path: config/test_deployment.yaml spec: replicas: 11 @@ -216,7 +216,7 @@ items: name: test app: nginx annotations: - config.kubernetes.io/index: 0 + config.kubernetes.io/index: "0" config.kubernetes.io/path: config/test_service.yaml spec: selector: diff --git a/cmd/config/internal/commands/grep_test.go b/cmd/config/internal/commands/grep_test.go index d1dd521c7..6d869af35 100644 --- a/cmd/config/internal/commands/grep_test.go +++ b/cmd/config/internal/commands/grep_test.go @@ -75,9 +75,9 @@ metadata: name: foo annotations: app: nginx2 - config.kubernetes.io/index: 0 - config.kubernetes.io/package: . - config.kubernetes.io/path: f1.yaml + config.kubernetes.io/index: '0' + config.kubernetes.io/package: '.' + config.kubernetes.io/path: 'f1.yaml' spec: replicas: 1 --- @@ -86,9 +86,9 @@ metadata: name: foo annotations: app: nginx - config.kubernetes.io/index: 1 - config.kubernetes.io/package: . - config.kubernetes.io/path: f1.yaml + config.kubernetes.io/index: '1' + config.kubernetes.io/package: '.' + config.kubernetes.io/path: 'f1.yaml' spec: selector: app: nginx @@ -145,7 +145,7 @@ metadata: name: foo annotations: app: nginx2 - config.kubernetes.io/index: 0 + config.kubernetes.io/index: '0' spec: replicas: 1 --- @@ -154,7 +154,7 @@ metadata: name: foo annotations: app: nginx - config.kubernetes.io/index: 1 + config.kubernetes.io/index: '1' spec: selector: app: nginx diff --git a/cmd/config/internal/generateddocs/api/docs.go b/cmd/config/internal/generateddocs/api/docs.go index d0ea008d5..39907c1b6 100644 --- a/cmd/config/internal/generateddocs/api/docs.go +++ b/cmd/config/internal/generateddocs/api/docs.go @@ -292,16 +292,17 @@ Resource Configuration may be read / written from / to sources such as directori stdin|out or network. Tools may be composed using pipes such that the tools writing Resource Configuration may be a different tool from the one that read the configuration. In order for tools to be composed in this way, while preserving origin information -- -such as the original file, index, etc. +such as the original file, index, etc.: -Tools **SHOULD** write the following annotations when reading from sources, -and **SHOULD** respect the annotations when writing to sinks. +Tools **SHOULD** insert the following annotations when reading from sources, +and **SHOULD** delete the annotations when writing to sinks. ### ` + "`" + `config.kubernetes.io/path` + "`" + ` Records the slash-delimited, OS-agnostic, relative file path to a Resource. This annotation **SHOULD** be set when reading Resources from files. +It **SHOULD** be unset when writing Resources to files. When writing Resources to a directory, the Resource **SHOULD** be written to the corresponding path relative to that directory. @@ -313,10 +314,11 @@ Example: ### ` + "`" + `config.kubernetes.io/index` + "`" + ` -Records the index of a Resource in file. In a multi-object files YAML file, Resources are separated +Records the index of a Resource in file. In a multi-object YAML file, Resources are separated by three dashes (` + "`" + `---` + "`" + `), and the index represents the positon of the Resource starting from zero. This annotation **SHOULD** be set when reading Resources from files. +It **SHOULD** be unset when writing Resources to files. When writing multiple Resources to the same file, the Resource **SHOULD** be written in the relative order matching the index. diff --git a/kyaml/fieldmeta/fieldmeta.go b/kyaml/fieldmeta/fieldmeta.go index 7b47229db..5458b6285 100644 --- a/kyaml/fieldmeta/fieldmeta.go +++ b/kyaml/fieldmeta/fieldmeta.go @@ -1,3 +1,6 @@ +// Copyright 2019 The Kubernetes Authors. +// SPDX-License-Identifier: Apache-2.0 + package fieldmeta import ( diff --git a/kyaml/kio/byteio_reader_test.go b/kyaml/kio/byteio_reader_test.go index 4e0f237d8..e6de127a7 100644 --- a/kyaml/kio/byteio_reader_test.go +++ b/kyaml/kio/byteio_reader_test.go @@ -150,7 +150,7 @@ func TestByteReader_Read(t *testing.T) { c: d metadata: annotations: - config.kubernetes.io/index: 0 + config.kubernetes.io/index: '0' `, `# second resource e: f @@ -158,12 +158,12 @@ g: - h metadata: annotations: - config.kubernetes.io/index: 1 + config.kubernetes.io/index: '1' `, `i: j metadata: annotations: - config.kubernetes.io/index: 2 + config.kubernetes.io/index: '2' `, } for i := range nodes { @@ -230,7 +230,7 @@ func TestByteReader_Read_setAnnotationsOmitReaderAnnotations(t *testing.T) { c: d metadata: annotations: - foo: bar + foo: 'bar' `, `# second resource e: f @@ -238,12 +238,12 @@ g: - h metadata: annotations: - foo: bar + foo: 'bar' `, `i: j metadata: annotations: - foo: bar + foo: 'bar' `, } for i := range nodes { @@ -278,8 +278,8 @@ func TestByteReader_Read_setAnnotations(t *testing.T) { c: d metadata: annotations: - config.kubernetes.io/index: 0 - foo: bar + config.kubernetes.io/index: '0' + foo: 'bar' `, `# second resource e: f @@ -287,14 +287,14 @@ g: - h metadata: annotations: - config.kubernetes.io/index: 1 - foo: bar + config.kubernetes.io/index: '1' + foo: 'bar' `, `i: j metadata: annotations: - config.kubernetes.io/index: 2 - foo: bar + config.kubernetes.io/index: '2' + foo: 'bar' `, } for i := range nodes { diff --git a/kyaml/kio/example_test.go b/kyaml/kio/example_test.go index d0c27ec51..cdf9e7cb6 100644 --- a/kyaml/kio/example_test.go +++ b/kyaml/kio/example_test.go @@ -77,7 +77,7 @@ spec: // labels: // app: nginx // annotations: - // foo: bar + // foo: 'bar' // spec: // replicas: 3 // selector: @@ -99,7 +99,7 @@ spec: // metadata: // name: nginx // annotations: - // foo: bar + // foo: 'bar' // spec: // selector: // app: nginx diff --git a/kyaml/kio/filters/container_test.go b/kyaml/kio/filters/container_test.go index 336ba0e8e..b17769f6f 100644 --- a/kyaml/kio/filters/container_test.go +++ b/kyaml/kio/filters/container_test.go @@ -177,13 +177,13 @@ items: metadata: name: deployment-foo annotations: - config.kubernetes.io/index: 0 + config.kubernetes.io/index: '0' - apiVersion: v1 kind: Service metadata: name: service-foo annotations: - config.kubernetes.io/index: 1 + config.kubernetes.io/index: '1' functionConfig: {apiVersion: apps/v1, kind: Deployment, metadata: {name: foo}} `, s) { t.FailNow() @@ -208,14 +208,14 @@ kind: StatefulSet metadata: name: deployment-foo annotations: - config.kubernetes.io/index: 0 + config.kubernetes.io/index: '0' --- apiVersion: v1 kind: Service metadata: name: service-foo annotations: - config.kubernetes.io/index: 1 + config.kubernetes.io/index: '1' `, b.String()) } @@ -259,13 +259,13 @@ items: metadata: name: deployment-foo annotations: - config.kubernetes.io/index: 0 + config.kubernetes.io/index: '0' - apiVersion: v1 kind: Service metadata: name: service-foo annotations: - config.kubernetes.io/index: 1 + config.kubernetes.io/index: '1' functionConfig: {apiversion: apps/v1, kind: Deployment, metadata: {name: foo}} `, s) { t.FailNow() @@ -290,14 +290,14 @@ kind: Deployment metadata: name: deployment-foo annotations: - config.kubernetes.io/index: 0 + config.kubernetes.io/index: '0' --- apiVersion: v1 kind: Service metadata: name: service-foo annotations: - config.kubernetes.io/index: 1 + config.kubernetes.io/index: '1' `, b.String()) } diff --git a/kyaml/kio/filters/filters_test.go b/kyaml/kio/filters/filters_test.go index 720874572..3e24b20dc 100644 --- a/kyaml/kio/filters/filters_test.go +++ b/kyaml/kio/filters/filters_test.go @@ -53,21 +53,21 @@ metadata: name: foo1 namespace: bar annotations: - config.kubernetes.io/path: foo1_deployment.yaml + config.kubernetes.io/path: 'foo1_deployment.yaml' --- apiVersion: v1 kind: Service metadata: name: foo1 annotations: - config.kubernetes.io/path: foo1_service.yaml + config.kubernetes.io/path: 'foo1_service.yaml' --- apiVersion: apps/v1 kind: Deployment metadata: name: foo2 annotations: - config.kubernetes.io/path: foo2_deployment.yaml + config.kubernetes.io/path: 'foo2_deployment.yaml' --- apiVersion: v1 kind: Service @@ -75,7 +75,7 @@ metadata: name: foo2 namespace: bar annotations: - config.kubernetes.io/path: foo2_service.yaml + config.kubernetes.io/path: 'foo2_service.yaml' `, out.String()) } @@ -97,7 +97,7 @@ kind: Service metadata: name: foo1 annotations: - config.kubernetes.io/path: foo1__service.yaml + config.kubernetes.io/path: 'foo1__service.yaml' --- apiVersion: apps/v1 kind: Deployment @@ -105,14 +105,14 @@ metadata: name: foo1 namespace: bar annotations: - config.kubernetes.io/path: foo1_bar_deployment.yaml + config.kubernetes.io/path: 'foo1_bar_deployment.yaml' --- apiVersion: apps/v1 kind: Deployment metadata: name: foo2 annotations: - config.kubernetes.io/path: foo2__deployment.yaml + config.kubernetes.io/path: 'foo2__deployment.yaml' --- apiVersion: v1 kind: Service @@ -120,7 +120,7 @@ metadata: name: foo2 namespace: bar annotations: - config.kubernetes.io/path: foo2_bar_service.yaml + config.kubernetes.io/path: 'foo2_bar_service.yaml' `, out.String()) } @@ -143,14 +143,14 @@ metadata: name: foo1 namespace: bar annotations: - config.kubernetes.io/path: resource.yaml + config.kubernetes.io/path: 'resource.yaml' --- apiVersion: apps/v1 kind: Deployment metadata: name: foo2 annotations: - config.kubernetes.io/path: resource.yaml + config.kubernetes.io/path: 'resource.yaml' --- apiVersion: v1 kind: Service @@ -158,13 +158,13 @@ metadata: name: foo2 namespace: bar annotations: - config.kubernetes.io/path: resource.yaml + config.kubernetes.io/path: 'resource.yaml' --- apiVersion: v1 kind: Service metadata: name: foo1 annotations: - config.kubernetes.io/path: resource.yaml + config.kubernetes.io/path: 'resource.yaml' `, out.String()) } diff --git a/kyaml/kio/pkgio_reader_test.go b/kyaml/kio/pkgio_reader_test.go index 8a446a601..7ee12c862 100644 --- a/kyaml/kio/pkgio_reader_test.go +++ b/kyaml/kio/pkgio_reader_test.go @@ -107,16 +107,16 @@ func TestLocalPackageReader_Read_pkg(t *testing.T) { `a: b #first metadata: annotations: - config.kubernetes.io/index: 0 - config.kubernetes.io/package: . - config.kubernetes.io/path: a_test.yaml + config.kubernetes.io/index: '0' + config.kubernetes.io/package: '.' + config.kubernetes.io/path: 'a_test.yaml' `, `c: d # second metadata: annotations: - config.kubernetes.io/index: 1 - config.kubernetes.io/package: . - config.kubernetes.io/path: a_test.yaml + config.kubernetes.io/index: '1' + config.kubernetes.io/package: '.' + config.kubernetes.io/path: 'a_test.yaml' `, `# second thing e: f @@ -126,9 +126,9 @@ g: - j metadata: annotations: - config.kubernetes.io/index: 0 - config.kubernetes.io/package: . - config.kubernetes.io/path: b_test.yaml + config.kubernetes.io/index: '0' + config.kubernetes.io/package: '.' + config.kubernetes.io/path: 'b_test.yaml' `, } for i := range nodes { @@ -169,16 +169,16 @@ func TestLocalPackageReader_Read_file(t *testing.T) { `a: b #first metadata: annotations: - config.kubernetes.io/index: 0 - config.kubernetes.io/package: . - config.kubernetes.io/path: a_test.yaml + config.kubernetes.io/index: '0' + config.kubernetes.io/package: '.' + config.kubernetes.io/path: 'a_test.yaml' `, `c: d # second metadata: annotations: - config.kubernetes.io/index: 1 - config.kubernetes.io/package: . - config.kubernetes.io/path: a_test.yaml + config.kubernetes.io/index: '1' + config.kubernetes.io/package: '.' + config.kubernetes.io/path: 'a_test.yaml' `, } for i := range nodes { @@ -268,16 +268,16 @@ func TestLocalPackageReader_Read_nestedDirs(t *testing.T) { `a: b #first metadata: annotations: - config.kubernetes.io/index: 0 - config.kubernetes.io/package: a/b - config.kubernetes.io/path: a/b/a_test.yaml + config.kubernetes.io/index: '0' + config.kubernetes.io/package: 'a/b' + config.kubernetes.io/path: 'a/b/a_test.yaml' `, `c: d # second metadata: annotations: - config.kubernetes.io/index: 1 - config.kubernetes.io/package: a/b - config.kubernetes.io/path: a/b/a_test.yaml + config.kubernetes.io/index: '1' + config.kubernetes.io/package: 'a/b' + config.kubernetes.io/path: 'a/b/a_test.yaml' `, `# second thing e: f @@ -287,9 +287,9 @@ g: - j metadata: annotations: - config.kubernetes.io/index: 0 - config.kubernetes.io/package: a/b - config.kubernetes.io/path: a/b/b_test.yaml + config.kubernetes.io/index: '0' + config.kubernetes.io/package: 'a/b' + config.kubernetes.io/path: 'a/b/b_test.yaml' `, } for i := range nodes { @@ -326,9 +326,9 @@ func TestLocalPackageReader_Read_matchRegex(t *testing.T) { assert.Equal(t, `a: b #first metadata: annotations: - config.kubernetes.io/index: 0 - config.kubernetes.io/package: a/b - config.kubernetes.io/path: a/b/a_test.yaml + config.kubernetes.io/index: '0' + config.kubernetes.io/package: 'a/b' + config.kubernetes.io/path: 'a/b/a_test.yaml' `, val) val, err = nodes[1].String() @@ -336,9 +336,9 @@ metadata: assert.Equal(t, `c: d # second metadata: annotations: - config.kubernetes.io/index: 1 - config.kubernetes.io/package: a/b - config.kubernetes.io/path: a/b/a_test.yaml + config.kubernetes.io/index: '1' + config.kubernetes.io/package: 'a/b' + config.kubernetes.io/path: 'a/b/a_test.yaml' `, val) } @@ -365,9 +365,9 @@ func TestLocalPackageReader_Read_skipSubpackage(t *testing.T) { assert.Equal(t, `a: b #first metadata: annotations: - config.kubernetes.io/index: 0 - config.kubernetes.io/package: a/b - config.kubernetes.io/path: a/b/a_test.yaml + config.kubernetes.io/index: '0' + config.kubernetes.io/package: 'a/b' + config.kubernetes.io/path: 'a/b/a_test.yaml' `, val) val, err = nodes[1].String() @@ -375,9 +375,9 @@ metadata: assert.Equal(t, `c: d # second metadata: annotations: - config.kubernetes.io/index: 1 - config.kubernetes.io/package: a/b - config.kubernetes.io/path: a/b/a_test.yaml + config.kubernetes.io/index: '1' + config.kubernetes.io/package: 'a/b' + config.kubernetes.io/path: 'a/b/a_test.yaml' `, val) } @@ -403,9 +403,9 @@ func TestLocalPackageReader_Read_includeSubpackage(t *testing.T) { assert.Equal(t, `a: b #first metadata: annotations: - config.kubernetes.io/index: 0 - config.kubernetes.io/package: a/b - config.kubernetes.io/path: a/b/a_test.yaml + config.kubernetes.io/index: '0' + config.kubernetes.io/package: 'a/b' + config.kubernetes.io/path: 'a/b/a_test.yaml' `, val) val, err = nodes[1].String() @@ -413,9 +413,9 @@ metadata: assert.Equal(t, `c: d # second metadata: annotations: - config.kubernetes.io/index: 1 - config.kubernetes.io/package: a/b - config.kubernetes.io/path: a/b/a_test.yaml + config.kubernetes.io/index: '1' + config.kubernetes.io/package: 'a/b' + config.kubernetes.io/path: 'a/b/a_test.yaml' `, val) val, err = nodes[2].String() @@ -428,9 +428,9 @@ g: - j metadata: annotations: - config.kubernetes.io/index: 0 - config.kubernetes.io/package: a/c - config.kubernetes.io/path: a/c/c_test.yaml + config.kubernetes.io/index: '0' + config.kubernetes.io/package: 'a/c' + config.kubernetes.io/path: 'a/c/c_test.yaml' `, val) } diff --git a/kyaml/yaml/fns_test.go b/kyaml/yaml/fns_test.go index 3419cf311..1bb05ce4d 100644 --- a/kyaml/yaml/fns_test.go +++ b/kyaml/yaml/fns_test.go @@ -637,12 +637,12 @@ func TestSetAnnotation_Fn(t *testing.T) { kind: Deployment`)) rn := assertNoError(t)(r0.Pipe(SetAnnotation("a.b.c", "d.e.f"))) - assert.Equal(t, "d.e.f\n", assertNoErrorString(t)(rn.String())) + assert.Equal(t, "'d.e.f'\n", assertNoErrorString(t)(rn.String())) assert.Equal(t, `apiVersion: apps/v1 kind: Deployment metadata: annotations: - a.b.c: d.e.f + a.b.c: 'd.e.f' `, assertNoErrorString(t)(r0.String())) } diff --git a/kyaml/yaml/kfns.go b/kyaml/yaml/kfns.go index 60d28b09a..564682c64 100644 --- a/kyaml/yaml/kfns.go +++ b/kyaml/yaml/kfns.go @@ -31,9 +31,13 @@ type AnnotationSetter struct { } func (s AnnotationSetter) Filter(rn *RNode) (*RNode, error) { + // some tools get confused about the type if annotations are not quoted + v := NewScalarRNode(s.Value) + v.YNode().Tag = "!!str" + v.YNode().Style = yaml.SingleQuotedStyle return rn.Pipe( PathGetter{Path: []string{"metadata", "annotations"}, Create: yaml.MappingNode}, - FieldSetter{Name: s.Key, Value: NewScalarRNode(s.Value)}) + FieldSetter{Name: s.Key, Value: v}) } func SetAnnotation(key, value string) AnnotationSetter { From 0b9aa418c036ed367f54cb33b0077a4641c2cabf Mon Sep 17 00:00:00 2001 From: Jeffrey Regan Date: Mon, 16 Dec 2019 19:10:20 -0800 Subject: [PATCH 07/27] Pin kustomize to specific versions of local deps. --- kustomize/go.mod | 12 +++--------- kustomize/go.sum | 6 ++++++ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/kustomize/go.mod b/kustomize/go.mod index 282ab940e..da1af47ab 100644 --- a/kustomize/go.mod +++ b/kustomize/go.mod @@ -7,18 +7,12 @@ require ( github.com/spf13/cobra v0.0.5 github.com/spf13/pflag v1.0.5 sigs.k8s.io/kustomize/api v0.3.0 - sigs.k8s.io/kustomize/cmd/config v0.0.0 - sigs.k8s.io/kustomize/cmd/kubectl v0.0.0 - sigs.k8s.io/kustomize/kyaml v0.0.0 + sigs.k8s.io/kustomize/cmd/config v0.0.2 + sigs.k8s.io/kustomize/cmd/kubectl v0.0.2 + sigs.k8s.io/kustomize/kyaml v0.0.2 sigs.k8s.io/yaml v1.1.0 ) -replace ( - sigs.k8s.io/kustomize/cmd/config v0.0.0 => ../cmd/config - sigs.k8s.io/kustomize/cmd/kubectl v0.0.0 => ../cmd/kubectl - sigs.k8s.io/kustomize/kyaml v0.0.0 => ../kyaml -) - exclude ( github.com/russross/blackfriday v2.0.0+incompatible sigs.k8s.io/kustomize/api v0.2.0 diff --git a/kustomize/go.sum b/kustomize/go.sum index 02dc9555c..c780d3be1 100644 --- a/kustomize/go.sum +++ b/kustomize/go.sum @@ -527,6 +527,12 @@ sigs.k8s.io/kustomize v2.0.3+incompatible h1:JUufWFNlI44MdtnjUqVnvh29rR37PQFzPbL sigs.k8s.io/kustomize v2.0.3+incompatible/go.mod h1:MkjgH3RdOWrievjo6c9T245dYlB5QeXV4WCbnt/PEpU= sigs.k8s.io/kustomize/api v0.3.0 h1:e7Erw2n8lT8+IWUukktozF0bgWwH2fFC+qsXP0gabg0= sigs.k8s.io/kustomize/api v0.3.0/go.mod h1:4jaPCtRzxfQLFdYq4gYo40dBGW1hyPp/f4AuiZB5dAQ= +sigs.k8s.io/kustomize/cmd/config v0.0.2 h1:FphfIoGJ0jGGJJXq9WoG5sqqEIuTeDGx58E5NWHV8Hc= +sigs.k8s.io/kustomize/cmd/config v0.0.2/go.mod h1:c6IBoPpAAm5a2aD+0iH8IfeyCF5GPsY5Ws57Dwpcvg0= +sigs.k8s.io/kustomize/cmd/kubectl v0.0.2 h1:MxUAU5ie0tqx2MuDrUlcAL+Mgt8LVFcXc2scinSD8/w= +sigs.k8s.io/kustomize/cmd/kubectl v0.0.2/go.mod h1:SbNCE1g937W1yvaQrZbvPNT3aDRdicdeW2qXLTa+YiM= +sigs.k8s.io/kustomize/kyaml v0.0.2 h1:Rl/wMrnpZzZjsVeFIIOAb92Kz/UfLrTUEXjiHW6oS0o= +sigs.k8s.io/kustomize/kyaml v0.0.2/go.mod h1:rywm/rcR5LmCBghz9956tE45OdUPChFoXVVs+WmhMTI= sigs.k8s.io/kustomize/pluginator/v2 v2.0.0/go.mod h1:zrXhTv8BAKt0egmZX/8AtMOSFUSWM9YuoHvvqz8/eHE= sigs.k8s.io/structured-merge-diff v0.0.0-20190525122527-15d366b2352e/go.mod h1:wWxsB5ozmmv/SG7nM11ayaAW51xMvak/t1r0CSlcokI= sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs= From 7a384bc0d85cc8ab10a912fbd2382d8d4852dd6d Mon Sep 17 00:00:00 2001 From: Naoki Oketani Date: Tue, 17 Dec 2019 18:22:49 +0900 Subject: [PATCH 08/27] Fixed a wrong path and run `make all` --- cmd/config/docs/tutorials/function-basics.md | 4 ++-- cmd/config/internal/generateddocs/api/docs.go | 10 ++++++---- cmd/config/internal/generateddocs/tutorials/docs.go | 4 ++-- functions/examples/template-go-nginx/README.md | 2 +- .../examples/template-heredoc-cockroachdb/README.md | 2 +- 5 files changed, 12 insertions(+), 10 deletions(-) diff --git a/cmd/config/docs/tutorials/function-basics.md b/cmd/config/docs/tutorials/function-basics.md index 63702e97a..39cf67989 100644 --- a/cmd/config/docs/tutorials/function-basics.md +++ b/cmd/config/docs/tutorials/function-basics.md @@ -39,7 +39,7 @@ #### 2. Modify the Generated Resources - modify the generated Resources by adding an annotation, sidecar container, etc. - - modify the `local-resources/example-use.yaml` by changing the replicas + - modify the `local-resource/example-use.yaml` by changing the replicas re-run `run`. this will apply the updated replicas to the generated Resources, but keep the fields that you manually added to the generated Resource configuration. @@ -86,7 +86,7 @@ #### 2. Modify the Generated Resources - modify the generated Resources by adding an annotation, sidecar container, etc. - - modify the `local-resources/example-use.yaml` by changing the replicas + - modify the `local-resource/example-use.yaml` by changing the replicas re-run `run`. this will apply the updated replicas to the generated Resources, but keep the fields that you manually added to the generated Resource configuration. diff --git a/cmd/config/internal/generateddocs/api/docs.go b/cmd/config/internal/generateddocs/api/docs.go index d0ea008d5..39907c1b6 100644 --- a/cmd/config/internal/generateddocs/api/docs.go +++ b/cmd/config/internal/generateddocs/api/docs.go @@ -292,16 +292,17 @@ Resource Configuration may be read / written from / to sources such as directori stdin|out or network. Tools may be composed using pipes such that the tools writing Resource Configuration may be a different tool from the one that read the configuration. In order for tools to be composed in this way, while preserving origin information -- -such as the original file, index, etc. +such as the original file, index, etc.: -Tools **SHOULD** write the following annotations when reading from sources, -and **SHOULD** respect the annotations when writing to sinks. +Tools **SHOULD** insert the following annotations when reading from sources, +and **SHOULD** delete the annotations when writing to sinks. ### ` + "`" + `config.kubernetes.io/path` + "`" + ` Records the slash-delimited, OS-agnostic, relative file path to a Resource. This annotation **SHOULD** be set when reading Resources from files. +It **SHOULD** be unset when writing Resources to files. When writing Resources to a directory, the Resource **SHOULD** be written to the corresponding path relative to that directory. @@ -313,10 +314,11 @@ Example: ### ` + "`" + `config.kubernetes.io/index` + "`" + ` -Records the index of a Resource in file. In a multi-object files YAML file, Resources are separated +Records the index of a Resource in file. In a multi-object YAML file, Resources are separated by three dashes (` + "`" + `---` + "`" + `), and the index represents the positon of the Resource starting from zero. This annotation **SHOULD** be set when reading Resources from files. +It **SHOULD** be unset when writing Resources to files. When writing multiple Resources to the same file, the Resource **SHOULD** be written in the relative order matching the index. diff --git a/cmd/config/internal/generateddocs/tutorials/docs.go b/cmd/config/internal/generateddocs/tutorials/docs.go index 3f5e81c9e..8ebbeb87e 100644 --- a/cmd/config/internal/generateddocs/tutorials/docs.go +++ b/cmd/config/internal/generateddocs/tutorials/docs.go @@ -316,7 +316,7 @@ var FunctionBasicsLong = ` #### 2. Modify the Generated Resources - modify the generated Resources by adding an annotation, sidecar container, etc. - - modify the ` + "`" + `local-resources/example-use.yaml` + "`" + ` by changing the replicas + - modify the ` + "`" + `local-resource/example-use.yaml` + "`" + ` by changing the replicas re-run ` + "`" + `run` + "`" + `. this will apply the updated replicas to the generated Resources, but keep the fields that you manually added to the generated Resource configuration. @@ -363,7 +363,7 @@ var FunctionBasicsLong = ` #### 2. Modify the Generated Resources - modify the generated Resources by adding an annotation, sidecar container, etc. - - modify the ` + "`" + `local-resources/example-use.yaml` + "`" + ` by changing the replicas + - modify the ` + "`" + `local-resource/example-use.yaml` + "`" + ` by changing the replicas re-run ` + "`" + `run` + "`" + `. this will apply the updated replicas to the generated Resources, but keep the fields that you manually added to the generated Resource configuration. diff --git a/functions/examples/template-go-nginx/README.md b/functions/examples/template-go-nginx/README.md index ad6bc133d..94e529328 100644 --- a/functions/examples/template-go-nginx/README.md +++ b/functions/examples/template-go-nginx/README.md @@ -29,7 +29,7 @@ with `metadata.configFn` and running: kustomize config run local-resource/ -This generates the `local-resources/config` directory containing the template output. +This generates the `local-resource/config` directory containing the template output. - the template output may be modified by adding fields -- such as initContainers, sidecarConatiners, cpu resource limits, etc -- and these fields will be retained diff --git a/functions/examples/template-heredoc-cockroachdb/README.md b/functions/examples/template-heredoc-cockroachdb/README.md index 90c39cfc8..da390b012 100644 --- a/functions/examples/template-heredoc-cockroachdb/README.md +++ b/functions/examples/template-heredoc-cockroachdb/README.md @@ -24,7 +24,7 @@ with `metadata.configFn` and running: kustomize config run local-resource/ -This generates the `local-resources/config` directory containing the template output. +This generates the `local-resource/config` directory containing the template output. - the template output may be modified by adding fields -- such as initContainers, sidecarConatiners, cpu resource limits, etc -- and these fields will be retained From ae2bfc8ee6f8637de1d93e782ec1dcc7ece690db Mon Sep 17 00:00:00 2001 From: Phillip Wittrock Date: Tue, 17 Dec 2019 09:28:44 -0800 Subject: [PATCH 09/27] Fix travis kyaml exit and targets - properly exit non-0 if there are any uncommitted files - make functions/examples targes - add missing licenses - refactor into loop --- .../template-heredoc-cockroachdb/Makefile | 2 ++ .../validator-kubeval/image/Dockerfile | 3 +++ travis/consider-early-travis-exit.sh | 3 +++ travis/kyaml-pre-commit.sh | 25 +++++++++++-------- 4 files changed, 23 insertions(+), 10 deletions(-) diff --git a/functions/examples/template-heredoc-cockroachdb/Makefile b/functions/examples/template-heredoc-cockroachdb/Makefile index fabbaeb52..0badb79a7 100644 --- a/functions/examples/template-heredoc-cockroachdb/Makefile +++ b/functions/examples/template-heredoc-cockroachdb/Makefile @@ -7,6 +7,8 @@ license: (which $(GOPATH)/bin/addlicense || go get github.com/google/addlicense) $(GOPATH)/bin/addlicense -y 2019 -c "The Kubernetes Authors." -f LICENSE_TEMPLATE . +all: license + image: docker build image -t gcr.io/kustomize-functions/example-cockroachdb:v0.1.0 docker push gcr.io/kustomize-functions/example-cockroachdb:v0.1.0 diff --git a/functions/examples/validator-kubeval/image/Dockerfile b/functions/examples/validator-kubeval/image/Dockerfile index 7181aa545..e7d0fd1c2 100644 --- a/functions/examples/validator-kubeval/image/Dockerfile +++ b/functions/examples/validator-kubeval/image/Dockerfile @@ -1,3 +1,6 @@ +# Copyright 2019 The Kubernetes Authors. +# SPDX-License-Identifier: Apache-2.0 + FROM alpine:latest as schemas RUN apk --no-cache add git RUN git clone --depth 1 https://github.com/instrumenta/kubernetes-json-schema.git diff --git a/travis/consider-early-travis-exit.sh b/travis/consider-early-travis-exit.sh index eea852bcf..b979c8eea 100644 --- a/travis/consider-early-travis-exit.sh +++ b/travis/consider-early-travis-exit.sh @@ -1,3 +1,6 @@ +# Copyright 2019 The Kubernetes Authors. +# SPDX-License-Identifier: Apache-2.0 + # Exits with status 0 if it can be determined that the # current PR should not trigger all travis checks. # diff --git a/travis/kyaml-pre-commit.sh b/travis/kyaml-pre-commit.sh index 28211cf2f..43594a821 100755 --- a/travis/kyaml-pre-commit.sh +++ b/travis/kyaml-pre-commit.sh @@ -1,16 +1,21 @@ #!/bin/bash +# Copyright 2019 The Kubernetes Authors. +# SPDX-License-Identifier: Apache-2.0 + set -e -cd kyaml -make all - -cd ../cmd/config -make all - -cd ../kubectl -make all +# run all tests for kyaml and related commands +targets="kyaml cmd/config cmd/kubectl functions/examples/injection-tshirt-sizes functions/examples/template-go-nginx functions/examples/template-heredoc-cockroachdb functions/examples/validator-kubeval functions/examples/validator-resource-requests" +for target in $targets; do + pushd . + cd $target + make all + popd +done # make sure no files were generated or changed by make -cd ../.. +# ignore changes to go.mod and go.sum -- they are too flaky +find . -name go.mod | xargs git checkout -- +find . -name go.sum | xargs git checkout -- git add . -git diff-index HEAD -- +git diff-index HEAD --exit-code From 502f86a9825c91765f3264d44449726d83895f61 Mon Sep 17 00:00:00 2001 From: Phillip Wittrock Date: Tue, 17 Dec 2019 09:34:44 -0800 Subject: [PATCH 10/27] rename ownedBy json value to setBy --- cmd/config/docs/commands/sub.md | 4 ++-- cmd/config/docs/commands/subset.md | 2 +- cmd/config/internal/generateddocs/commands/docs.go | 6 +++--- kyaml/fieldmeta/fieldmeta.go | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/cmd/config/docs/commands/sub.md b/cmd/config/docs/commands/sub.md index 38540fe9f..eb5b7fbc8 100644 --- a/cmd/config/docs/commands/sub.md +++ b/cmd/config/docs/commands/sub.md @@ -80,12 +80,12 @@ To create a substitution for a field see: `kustomize help config set create` # dir/resources.yaml ... metadata: - name: test-app1 # {"substitutions":[{"name":"prefix","marker":"PREFIX-","value":"test-"}],"ownedBy":"dev","description":"test environment"} + name: test-app1 # {"substitutions":[{"name":"prefix","marker":"PREFIX-","value":"test-"}],"setBy":"dev","description":"test environment"} ... --- ... metadata: - name: test-app2 # {"substitutions":[{"name":"prefix","marker":"PREFIX-","value":"test-"}],"ownedBy":"dev","description":"test environment"} + name: test-app2 # {"substitutions":[{"name":"prefix","marker":"PREFIX-","value":"test-"}],"setBy":"dev","description":"test environment"} ... Revert substitution: diff --git a/cmd/config/docs/commands/subset.md b/cmd/config/docs/commands/subset.md index 1a3f5ca00..f5ee8b91d 100644 --- a/cmd/config/docs/commands/subset.md +++ b/cmd/config/docs/commands/subset.md @@ -144,7 +144,7 @@ FieldMeta Schema read by `sub`: "type": "string", "description": "A description of the field's current value. Optional." }, - "ownedBy": { + "setBy": { "type": "string", "description": "The current owner of the field. Optional." }, diff --git a/cmd/config/internal/generateddocs/commands/docs.go b/cmd/config/internal/generateddocs/commands/docs.go index 598a8c676..ef0013e83 100644 --- a/cmd/config/internal/generateddocs/commands/docs.go +++ b/cmd/config/internal/generateddocs/commands/docs.go @@ -262,12 +262,12 @@ var SubExamples = ` # dir/resources.yaml ... metadata: - name: test-app1 # {"substitutions":[{"name":"prefix","marker":"PREFIX-","value":"test-"}],"ownedBy":"dev","description":"test environment"} + name: test-app1 # {"substitutions":[{"name":"prefix","marker":"PREFIX-","value":"test-"}],"setBy":"dev","description":"test environment"} ... --- ... metadata: - name: test-app2 # {"substitutions":[{"name":"prefix","marker":"PREFIX-","value":"test-"}],"ownedBy":"dev","description":"test environment"} + name: test-app2 # {"substitutions":[{"name":"prefix","marker":"PREFIX-","value":"test-"}],"setBy":"dev","description":"test environment"} ... Revert substitution: @@ -421,7 +421,7 @@ FieldMeta Schema read by ` + "`" + `sub` + "`" + `: "type": "string", "description": "A description of the field's current value. Optional." }, - "ownedBy": { + "setBy": { "type": "string", "description": "The current owner of the field. Optional." }, diff --git a/kyaml/fieldmeta/fieldmeta.go b/kyaml/fieldmeta/fieldmeta.go index 5458b6285..9447c80af 100644 --- a/kyaml/fieldmeta/fieldmeta.go +++ b/kyaml/fieldmeta/fieldmeta.go @@ -18,7 +18,7 @@ type FieldMeta struct { // Substitutions are substitutions that may be performed against this field Substitutions []Substitution `yaml:"substitutions,omitempty" json:"substitutions,omitempty"` // OwnedBy records the owner of this field - OwnedBy string `yaml:"ownedBy,omitempty" json:"ownedBy,omitempty"` + OwnedBy string `yaml:"setBy,omitempty" json:"setBy,omitempty"` // DefaultedBy records that this field was default, but may be changed by other owners DefaultedBy string `yaml:"defaultedBy,omitempty" json:"defaultedBy,omitempty"` // Description is a description of the current field value, e.g. why it was set From de824c2e4de9e8abeb9949d6f3c3f1540a9cc966 Mon Sep 17 00:00:00 2001 From: Phillip Wittrock Date: Tue, 17 Dec 2019 13:48:29 -0800 Subject: [PATCH 11/27] Drop mdrip dependency from api/ because it has conflicting deps with kubectl --- api/go.mod | 1 - api/go.sum | 62 ++------------------------------ api/internal/tools/tools.go | 2 -- kustomize/go.sum | 70 +++++++++++++++++++++++++++++++++++++ 4 files changed, 72 insertions(+), 63 deletions(-) diff --git a/api/go.mod b/api/go.mod index cdf82ed3d..4fc747aef 100644 --- a/api/go.mod +++ b/api/go.mod @@ -7,7 +7,6 @@ require ( github.com/go-openapi/spec v0.19.4 github.com/golangci/golangci-lint v1.21.0 github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 - github.com/monopole/mdrip v1.0.1 github.com/pkg/errors v0.8.1 golang.org/x/tools v0.0.0-20191010075000-0337d82405ff gopkg.in/yaml.v2 v2.2.4 diff --git a/api/go.sum b/api/go.sum index b3a0e5e61..c5aa6187d 100644 --- a/api/go.sum +++ b/api/go.sum @@ -2,14 +2,10 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI= -github.com/Azure/go-autorest/autorest v0.9.2/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI= github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0= -github.com/Azure/go-autorest/autorest/adal v0.8.0/go.mod h1:Z6vX6WXXuyieHAXwMj0S6HY6e6wcHn37qQMBQlvY3lc= github.com/Azure/go-autorest/autorest/date v0.1.0/go.mod h1:plvfp3oPSKwf2DNjlBjWF/7vwR+cUD/ELuzDCXwHUVA= -github.com/Azure/go-autorest/autorest/date v0.2.0/go.mod h1:vcORJHLJEh643/Ioh9+vPmf1Ij9AEBM5FuBIXLmIy0g= github.com/Azure/go-autorest/autorest/mocks v0.1.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= github.com/Azure/go-autorest/autorest/mocks v0.2.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= -github.com/Azure/go-autorest/autorest/mocks v0.3.0/go.mod h1:a8FDP3DYzQ4RYfVAxAN3SVSiiO77gL2j2ronKKP0syM= github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc= github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= @@ -49,14 +45,9 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= -github.com/docker/spdystream v0.0.0-20181023171402-6480d4af844c/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= github.com/elazarl/goproxy v0.0.0-20170405201442-c4fc26588b6e/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= -github.com/elazarl/goproxy v0.0.0-20191011121108-aa519ddbe484/go.mod h1:Ro8st/ElPeALwNFlcTpWmkr6IoMFfkjXAvTHpevnDsM= -github.com/elazarl/goproxy/ext v0.0.0-20190711103511-473e67f1d7d2/go.mod h1:gNh8nYJoAm43RfaxurUnxr+N1PwuFV3ZMl/efxlIlY8= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633 h1:H2pdYOb3KQ1/YsqVWoWNLQO+fusocsw354rqGTZtAgw= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= -github.com/emicklei/go-restful v2.9.6+incompatible h1:tfrHha8zJ01ywiOEC1miGY8st1/igzWB8OmvPgoYX7w= -github.com/emicklei/go-restful v2.9.6+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch v4.5.0+incompatible h1:ouOWdg56aJriqS0huScTkVXPC5IcNrDCXZ6OoTAWu7M= github.com/evanphx/json-patch v4.5.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= @@ -119,13 +110,10 @@ github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7a github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d h1:3PaI8p3seN09VjbTYC/QWlUZdZ1qS1zGjy7LH2Wt07I= github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= -github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= -github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20191027212112-611e8accdfc9/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v0.0.0-20161109072736-4bd1920723d7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -145,15 +133,12 @@ github.com/golangci/goconst v0.0.0-20180610141641-041c5f2b40f3 h1:pe9JHs3cHHDQgO github.com/golangci/goconst v0.0.0-20180610141641-041c5f2b40f3/go.mod h1:JXrF4TWy4tXYn62/9x8Wm/K/dm06p8tCKwFRDPZG/1o= github.com/golangci/gocyclo v0.0.0-20180528134321-2becd97e67ee h1:J2XAy40+7yz70uaOiMbNnluTg7gyQhtGqLQncQh+4J8= github.com/golangci/gocyclo v0.0.0-20180528134321-2becd97e67ee/go.mod h1:ozx7R9SIwqmqf5pRP90DhR2Oay2UIjGuKheCBCNwAYU= -github.com/golangci/gofmt v0.0.0-20181222123516-0b8337e80d98/go.mod h1:9qCChq59u/eW8im404Q2WWTrnBUQKjpNYKMbU4M7EFU= github.com/golangci/gofmt v0.0.0-20190930125516-244bba706f1a h1:iR3fYXUjHCR97qWS8ch1y9zPNsgXThGwjKPrYfqMPks= github.com/golangci/gofmt v0.0.0-20190930125516-244bba706f1a/go.mod h1:9qCChq59u/eW8im404Q2WWTrnBUQKjpNYKMbU4M7EFU= -github.com/golangci/golangci-lint v1.19.1/go.mod h1:2CEc4Fxx3vxDv7g8DyXkHCBF73AOzAymcJAprs2vCps= github.com/golangci/golangci-lint v1.21.0 h1:HxAxpR8Z0M8omihvQdsD3PF0qPjlqYqp2vMJzstoKeI= github.com/golangci/golangci-lint v1.21.0/go.mod h1:phxpHK52q7SE+5KpPnti4oZTdFCEsn/tKN+nFvCKXfk= github.com/golangci/ineffassign v0.0.0-20190609212857-42439a7714cc h1:gLLhTLMk2/SutryVJ6D4VZCU3CUqr8YloG7FPIBWFpI= github.com/golangci/ineffassign v0.0.0-20190609212857-42439a7714cc/go.mod h1:e5tpTHCfVze+7EpLEozzMB3eafxo2KT5veNg1k6byQU= -github.com/golangci/lint-1 v0.0.0-20190420132249-ee948d087217/go.mod h1:66R6K6P6VWk9I95jvqGxkqJxVWGFy9XlDwLwVz1RCFg= github.com/golangci/lint-1 v0.0.0-20191013205115-297bf364a8e0 h1:MfyDlzVjl1hoaPzPD4Gpb/QgoRfSBR0jdhwGyAWwMSA= github.com/golangci/lint-1 v0.0.0-20191013205115-297bf364a8e0/go.mod h1:66R6K6P6VWk9I95jvqGxkqJxVWGFy9XlDwLwVz1RCFg= github.com/golangci/maligned v0.0.0-20180506175553-b1d89398deca h1:kNY3/svz5T29MYHubXix4aDDuE3RWHkPvopM/EDv/MA= @@ -169,9 +154,8 @@ github.com/golangci/unconvert v0.0.0-20180507085042-28b1c447d1f4/go.mod h1:Izgrg github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg= -github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/gofuzz v0.0.0-20161122191042-44d81051d367/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI= github.com/google/gofuzz v1.0.0 h1:A8PeW59pxE9IoFRqBp37U+mSNaQoZ46F1f0f863XSXw= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -184,31 +168,16 @@ github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d h1:7XGaL1e6bYS1yIonGp9761ExpPPV1ui0SAC59Yube9k= github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= -github.com/googleapis/gnostic v0.3.0 h1:CcQijm0XKekKjP/YCz28LXVSpgguuB+nCxaSjCe09y0= -github.com/googleapis/gnostic v0.3.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= github.com/gophercloud/gophercloud v0.1.0/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEoIEcSTewFxm1c5g8= -github.com/gophercloud/gophercloud v0.6.0/go.mod h1:GICNByuaEBibcjmjvI7QvYJSZEbGkcYwAR7EZK2WMqM= -github.com/gorilla/context v0.0.0-20160226214623-1ea25387ff6f h1:9oNbS1z4rVpbnkHBdPZU4jo9bSmrLpII768arSyMFgk= -github.com/gorilla/context v0.0.0-20160226214623-1ea25387ff6f/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= -github.com/gorilla/mux v1.6.0 h1:UykbtMB/w5No2LmE16gINgLj+r/vbziTgaoERQv6U+0= -github.com/gorilla/mux v1.6.0/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= -github.com/gorilla/securecookie v0.0.0-20160422134519-667fe4e3466a h1:YH0IojQwndMQdeRWdw1aPT8bkbiWaYR3WD+Zf5e09DU= -github.com/gorilla/securecookie v0.0.0-20160422134519-667fe4e3466a/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= -github.com/gorilla/sessions v0.0.0-20160922145804-ca9ada445741 h1:OuuPl66BpF1q3OEkaPpp+VfzxrBBY62ATGdWqql/XX8= -github.com/gorilla/sessions v0.0.0-20160922145804-ca9ada445741/go.mod h1:+WVp8kdw6VhyKExm03PAMRn2ZxnPtm58pV0dBVPdhHE= -github.com/gorilla/websocket v1.2.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= -github.com/gorilla/websocket v1.4.0 h1:WDFjx/TMzVgy9VdMMQi2K2Emtwi2QcUQsztZ/zLaH/Q= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gostaticanalysis/analysisutil v0.0.0-20190318220348-4088753ea4d3 h1:JVnpOZS+qxli+rgVl98ILOXVNbW+kb5wcxeGx8ShUIw= github.com/gostaticanalysis/analysisutil v0.0.0-20190318220348-4088753ea4d3/go.mod h1:eEOZF4jCKGi+aprrirO9e7WKB3beBRtWgqGunKl6pKE= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= -github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.3/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= @@ -248,10 +217,8 @@ github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e h1:hB2xlXdHp/pmPZq0y3QnmWAArdw9PqbmotexnWx/FU8= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/matoous/godox v0.0.0-20190910121045-032ad8106c86/go.mod h1:1BELzlh859Sh1c6+90blK8lbYy0kwQf1bYlBhBysy1s= github.com/matoous/godox v0.0.0-20190911065817-5d6d842e92eb h1:RHba4YImhrUVQDHUCe2BNSOz4tVy2yGyXhvYDvxGgeE= github.com/matoous/godox v0.0.0-20190911065817-5d6d842e92eb/go.mod h1:1BELzlh859Sh1c6+90blK8lbYy0kwQf1bYlBhBysy1s= -github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.4 h1:snbPLB8fVfU9iwbbo30TPtbLRzwWu6aJS6Xh4eaaviA= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE= @@ -270,10 +237,6 @@ github.com/modern-go/reflect2 v0.0.0-20180320133207-05fbef0ca5da/go.mod h1:bx2lN github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/monopole/mdrip v1.0.0 h1:RFDBa+tab6mW+gX4Ww2SZDc4kS6p01FwnLtgz64Il+I= -github.com/monopole/mdrip v1.0.0/go.mod h1:N1/ppRG9CaPeUKAUHZ3dUlfOT81lTpKZLkyhCvTETwM= -github.com/monopole/mdrip v1.0.1 h1:LzwNWb8ge3+4iBZmxIE6VfUR5KIxhF7DKdf85t8Yvos= -github.com/monopole/mdrip v1.0.1/go.mod h1:/7E04hlzRG9Jrp6WILZfYYm/REoJWL2l+MlsCO1eH74= github.com/mozilla/tls-observatory v0.0.0-20190404164649-a3c1b6cfecfd/go.mod h1:SrKMQvPiws7F7iqYp8/TX+IhxCYhzr6N/1yb8cwHsGk= github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= @@ -308,12 +271,8 @@ github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7z github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/quasilyte/go-consistent v0.0.0-20190521200055-c6f3937de18c/go.mod h1:5STLWrekHfjyYwxBRVRXNOSewLJ3PWfDJd1VyTS21fI= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= -github.com/rogpeppe/go-charset v0.0.0-20180617210344-2471d30d28b4/go.mod h1:qgYeAmZ5ZIpBWTGllZSQnw97Dj+woV0toclVaRGI8pc= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= -github.com/russross/blackfriday v2.0.0+incompatible h1:cBXrhZNUf9C+La9/YpS+UHpUT8YD6Td9ZMSU9APFcsk= -github.com/russross/blackfriday v2.0.0+incompatible/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= -github.com/securego/gosec v0.0.0-20190912120752-140048b2a218/go.mod h1:q6oYAujd2qyeU4cJqIri4LBIgdHXGvxWHZ1E29HNFRE= github.com/securego/gosec v0.0.0-20191002120514-e680875ea14d h1:BzRvVq1EHuIjxpijCEKpAxzKUUMurOQ4sknehIATRh8= github.com/securego/gosec v0.0.0-20191002120514-e680875ea14d/go.mod h1:w5+eXa0mYznDkHaMCXA4XYffjlH+cy1oyKbfzJXa2Do= github.com/shirou/gopsutil v0.0.0-20190901111213-e4ec7b275ada/go.mod h1:WWnYX4lzhCH5h/3YBfyVA3VbLYjlMZZAQcW9ojMexNc= @@ -322,8 +281,6 @@ github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e h1:MZM7FHLqUHYI0Y/mQAt github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk= github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041 h1:llrF3Fs4018ePo4+G/HV/uQUqEI1HMDjCeOf2V6puPc= github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041/go.mod h1:N5mDOmsrJOB+vfqUK+7DmDyjhSLIIBnXo9lvZJj3MWQ= -github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= -github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= @@ -356,7 +313,6 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/timakin/bodyclose v0.0.0-20190721030226-87058b9bfcec/go.mod h1:Qimiffbc6q9tBWlVV6x0P9sat/ao1xEkREYPPj9hphk= github.com/timakin/bodyclose v0.0.0-20190930140734-f7f2e9bca95e h1:RumXZ56IrCj4CL+g1b9OL/oH0QnsF976bC8xQFYUD5Q= github.com/timakin/bodyclose v0.0.0-20190930140734-f7f2e9bca95e/go.mod h1:Qimiffbc6q9tBWlVV6x0P9sat/ao1xEkREYPPj9hphk= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= @@ -364,7 +320,6 @@ github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGr github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/ultraware/funlen v0.0.2 h1:Av96YVBwwNSe4MLR7iI/BIa3VyI7/djnto/pK3Uxbdo= github.com/ultraware/funlen v0.0.2/go.mod h1:Dp4UiAus7Wdb9KUZsYWZEWiRzGuM2kXM1lPbfaF6xhA= -github.com/ultraware/whitespace v0.0.3/go.mod h1:aVMh/gQve5Maj9hQ/hg+F75lr/X5A89uZnzAmWSineA= github.com/ultraware/whitespace v0.0.4 h1:If7Va4cM03mpgrNH9k49/VOicWpGoG70XPBFFODYDsg= github.com/ultraware/whitespace v0.0.4/go.mod h1:aVMh/gQve5Maj9hQ/hg+F75lr/X5A89uZnzAmWSineA= github.com/uudashr/gocognit v0.0.0-20190926065955-1655d0de0517 h1:ChMKTho2hWKpks/nD/FL2KqM1wuVt62oJeiE8+eFpGs= @@ -387,7 +342,6 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -410,8 +364,6 @@ golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190909003024-a7b16738d86b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190923162816-aa69164e4478 h1:l5EDrHhldLYb3ZRHDUhXF7Om7MvYXnkV9/iQNo1lX6g= golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191004110552-13f9640d40b9 h1:rjwSpXsdiK0dV8/Naq3kAw9ymfAeJIyd0upUIElB+lI= golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -437,7 +389,6 @@ golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190911201528-7ad0cfa0b7b5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190922100055-0a153f010e69 h1:rOhMmluY6kLMhdnrivzec6lLgaVbMHMn2ISQXJeJ5EM= golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -447,7 +398,6 @@ golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -462,12 +412,11 @@ golang.org/x/tools v0.0.0-20190311215038-5c2858a9cfe5/go.mod h1:LCzVGOaR6xXOjkQ3 golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190322203728-c1a832b0ad89/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190521203540-521d6ed310dd/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59 h1:QjA/9ArTfVTLfEhClDCG7SGrZkZixxWpwNCDiwJfh88= golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190719005602-e377ae9d6386/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI= golang.org/x/tools v0.0.0-20190910044552-dd2b5c81c578/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190911230505-6bfd74cf029c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190912215617-3720d1ec3678/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190930201159-7c411dea38b0/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191010075000-0337d82405ff h1:XdBG6es/oFDr1HwaxkxgVve7NB281QhxgK/i4voubFs= golang.org/x/tools v0.0.0-20191010075000-0337d82405ff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -492,8 +441,6 @@ gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMy gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= -gopkg.in/russross/blackfriday.v2 v2.0.0 h1:+FlnIV8DSQnT7NZ43hcVKcdJdzZoeCmJj4Ql8gq5keA= -gopkg.in/russross/blackfriday.v2 v2.0.0/go.mod h1:6sSBNz/GtOm/pJTuh5UmBK2ZHfmnxGbl2NZg1UliSOI= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= @@ -518,7 +465,6 @@ k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8= k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= k8s.io/kube-openapi v0.0.0-20191107075043-30be4d16710a h1:UcxjrRMyNx/i/y8G7kPvLyy7rfbeuf1PYyBf973pgyU= k8s.io/kube-openapi v0.0.0-20191107075043-30be4d16710a/go.mod h1:1TqjTSzOxsLGIKfj0lK8EeCP7K1iUG65v09OM0/WG5E= -k8s.io/utils v0.0.0-20191030222137-2b95a09bc58d/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= k8s.io/utils v0.0.0-20191114184206-e782cd3c129f/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed h1:WX1yoOaKQfddO/mLzdV4wptyWgoH/6hwLs7QHTixo0I= mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed/go.mod h1:Xkxe497xwlCKkIaQYRfC7CSLworTXY9RMqwhhCm+8Nc= @@ -526,10 +472,6 @@ mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b h1:DxJ5nJdkhDlLok9K6qO+5290kphD mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b/go.mod h1:2odslEg/xrtNQqCYg2/jCoyKnw3vv5biOc3JnIcYfL4= mvdan.cc/unparam v0.0.0-20190720180237-d51796306d8f h1:Cq7MalBHYACRd6EesksG1Q8EoIAKOsiZviGKbOLIej4= mvdan.cc/unparam v0.0.0-20190720180237-d51796306d8f/go.mod h1:4G1h5nDURzA3bwVMZIVpwbkw+04kSxk3rAtzlimaUJw= -sigs.k8s.io/kustomize/api v0.2.0/go.mod h1:zVtMg179jW1gr74jo9fc2Ac9dLYLTZZThc3DDb9lDW4= -sigs.k8s.io/kustomize/pluginator/v2 v2.0.0 h1:sES7e09G19Q0VjRp4ATSYKpTXoWaX8WMSHfw6u3G2Ok= -sigs.k8s.io/kustomize/pluginator/v2 v2.0.0/go.mod h1:zrXhTv8BAKt0egmZX/8AtMOSFUSWM9YuoHvvqz8/eHE= -sigs.k8s.io/kustomize/pseudo/k8s v0.1.0/go.mod h1:bl/gVJgYYhJZCZdYU2BfnaKYAlqFkgbJEkpl302jEss= sigs.k8s.io/structured-merge-diff v0.0.0-20190525122527-15d366b2352e/go.mod h1:wWxsB5ozmmv/SG7nM11ayaAW51xMvak/t1r0CSlcokI= sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= diff --git a/api/internal/tools/tools.go b/api/internal/tools/tools.go index 302d2b4d7..ac7ae28e9 100644 --- a/api/internal/tools/tools.go +++ b/api/internal/tools/tools.go @@ -15,8 +15,6 @@ import ( _ "golang.org/x/tools/cmd/stringer" // for lint checks _ "github.com/golangci/golangci-lint/cmd/golangci-lint" - // for integration tests driven by the examples - _ "github.com/monopole/mdrip" // REMOVED pluginator from this process, and leaving // this note to discourage its reintroduction, // because pluginator depends on the api, forcing diff --git a/kustomize/go.sum b/kustomize/go.sum index c780d3be1..f542a9868 100644 --- a/kustomize/go.sum +++ b/kustomize/go.sum @@ -17,12 +17,14 @@ github.com/Azure/go-autorest/logger v0.1.0 h1:ruG4BSDXONFRrZZJ2GUXDiUyVpayPmb1Gn github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc= github.com/Azure/go-autorest/tracing v0.5.0 h1:TRn4WjSnkcSy5AEG3pnbtFSwNtwzjr4VYyQflFE619k= github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk= +github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/MakeNowJust/heredoc v0.0.0-20170808103936-bb23615498cd h1:sjQovDkwrZp8u+gxLtPgKGjk5hCxuy2hrRejBTA9xFU= github.com/MakeNowJust/heredoc v0.0.0-20170808103936-bb23615498cd/go.mod h1:64YHyfSL2R96J44Nlwm39UHepQbyR5q10x7iYa1ks2E= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/OpenPeeDeeP/depguard v1.0.1 h1:VlW4R6jmBIv3/u1JNlawEvJMM4J+dPORPaZasQee8Us= github.com/OpenPeeDeeP/depguard v1.0.1/go.mod h1:xsIw86fROiiwelg+jB2uM9PiKihMMmUx/1V+TNhjQvM= github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI= @@ -37,6 +39,7 @@ github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= +github.com/bombsimon/wsl v1.2.5 h1:9gTOkIwVtoDZywvX802SDHokeX4kW1cKnV8ZTVAPkRs= github.com/bombsimon/wsl v1.2.5/go.mod h1:43lEF/i0kpXbLCeDXL9LMT8c92HyBywXb0AsgMHYngM= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/chai2010/gettext-go v0.0.0-20160711120539-c6fed771bfd5 h1:7aWHqerlJ41y6FOsEUvknqgXnGmJyJSbjhAWq5pO4F8= @@ -74,16 +77,19 @@ github.com/evanphx/json-patch v4.5.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLi github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d h1:105gxyaGwCFad8crR9dcMQWvV9Hvulu6hwUh4tWPJnM= github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d/go.mod h1:ZZMPRZwes7CROmyNKgQzC3XPs6L/G2EJLHddWejkmf4= github.com/fatih/camelcase v1.0.0/go.mod h1:yN2Sb0lFhZJUdVvtELVWefmrXpuZESvPmqwoZc+/fpc= +github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-critic/go-critic v0.3.5-0.20190904082202-d79a9f0c64db h1:GYXWx7Vr3+zv833u+8IoXbNnQY0AdXsxAgI0kX7xcwA= github.com/go-critic/go-critic v0.3.5-0.20190904082202-d79a9f0c64db/go.mod h1:+sE8vrLDS2M0pZkBk0wy6+nLdKexVDrl/jBqQOTDThA= github.com/go-errors/errors v1.0.1 h1:LUHzmkK3GUKUrL/1gfBUxAHzcev3apQlezX/+O7ma6w= github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-lintpack/lintpack v0.5.2 h1:DI5mA3+eKdWeJ40nU4d6Wc26qmdG8RCi/btYq0TuRN0= github.com/go-lintpack/lintpack v0.5.2/go.mod h1:NwZuYi2nUHho8XEIZ6SIxihrnPoqBTDqfpXvXAN0sXM= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= @@ -106,25 +112,36 @@ github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh github.com/go-openapi/swag v0.19.5 h1:lTz6Ys4CmqqCQmZPBlbQENR1/GucA2bzYTE12Pw4tFY= github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/go-toolsmith/astcast v1.0.0 h1:JojxlmI6STnFVG9yOImLeGREv8W2ocNUM+iOhR6jE7g= github.com/go-toolsmith/astcast v1.0.0/go.mod h1:mt2OdQTeAQcY4DQgPSArJjHCcOwlX+Wl/kwN+LbLGQ4= +github.com/go-toolsmith/astcopy v1.0.0 h1:OMgl1b1MEpjFQ1m5ztEO06rz5CUd3oBv9RF7+DyvdG8= github.com/go-toolsmith/astcopy v1.0.0/go.mod h1:vrgyG+5Bxrnz4MZWPF+pI4R8h3qKRjjyvV/DSez4WVQ= github.com/go-toolsmith/astequal v0.0.0-20180903214952-dcb477bfacd6/go.mod h1:H+xSiq0+LtiDC11+h1G32h7Of5O3CYFJ99GVbS5lDKY= +github.com/go-toolsmith/astequal v1.0.0 h1:4zxD8j3JRFNyLN46lodQuqz3xdKSrur7U/sr0SDS/gQ= github.com/go-toolsmith/astequal v1.0.0/go.mod h1:H+xSiq0+LtiDC11+h1G32h7Of5O3CYFJ99GVbS5lDKY= github.com/go-toolsmith/astfmt v0.0.0-20180903215011-8f8ee99c3086/go.mod h1:mP93XdblcopXwlyN4X4uodxXQhldPGZbcEJIimQHrkg= +github.com/go-toolsmith/astfmt v1.0.0 h1:A0vDDXt+vsvLEdbMFJAUBI/uTbRw1ffOPnxsILnFL6k= github.com/go-toolsmith/astfmt v1.0.0/go.mod h1:cnWmsOAuq4jJY6Ct5YWlVLmcmLMn1JUPuQIHCY7CJDw= github.com/go-toolsmith/astinfo v0.0.0-20180906194353-9809ff7efb21/go.mod h1:dDStQCHtmZpYOmjRP/8gHHnCCch3Zz3oEgCdZVdtweU= github.com/go-toolsmith/astp v0.0.0-20180903215135-0af7e3c24f30/go.mod h1:SV2ur98SGypH1UjcPpCatrV5hPazG6+IfNHbkDXBRrk= +github.com/go-toolsmith/astp v1.0.0 h1:alXE75TXgcmupDsMK1fRAy0YUzLzqPVvBKoyWV+KPXg= github.com/go-toolsmith/astp v1.0.0/go.mod h1:RSyrtpVlfTFGDYRbrjyWP1pYu//tSFcvdYrA8meBmLI= github.com/go-toolsmith/pkgload v0.0.0-20181119091011-e9e65178eee8/go.mod h1:WoMrjiy4zvdS+Bg6z9jZH82QXwkcgCBX6nOfnmdaHks= +github.com/go-toolsmith/pkgload v1.0.0 h1:4DFWWMXVfbcN5So1sBNW9+yeiMqLFGl1wFLTL5R0Tgg= github.com/go-toolsmith/pkgload v1.0.0/go.mod h1:5eFArkbO80v7Z0kdngIxsRXRMTaX4Ilcwuh3clNrQJc= +github.com/go-toolsmith/strparse v1.0.0 h1:Vcw78DnpCAKlM20kSbAyO4mPfJn/lyYA4BJUDxe2Jb4= github.com/go-toolsmith/strparse v1.0.0/go.mod h1:YI2nUKP9YGZnL/L1/DLFBfixrcjslWct4wyljWhSRy8= +github.com/go-toolsmith/typep v1.0.0 h1:zKymWyA1TRYvqYrYDrfEMZULyrhcnGY3x7LDKU2XQaA= github.com/go-toolsmith/typep v1.0.0/go.mod h1:JSQCQMUPdRlMZFswiq3TGpNp1GMktqkR2Ns5AIQkATU= +github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= +github.com/gofrs/flock v0.0.0-20190320160742-5135e617513b h1:ekuhfTjngPhisSjOJ0QWKpPQE8/rbknHaes6WVJj5Hw= github.com/gofrs/flock v0.0.0-20190320160742-5135e617513b/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d h1:3PaI8p3seN09VjbTYC/QWlUZdZ1qS1zGjy7LH2Wt07I= github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef h1:veQD95Isof8w9/WXiA+pa3tz3fJXkt5B7QaRBrM62gk= @@ -136,20 +153,35 @@ github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golangci/check v0.0.0-20180506172741-cfe4005ccda2 h1:23T5iq8rbUYlhpt5DB4XJkc6BU31uODLD1o1gKvZmD0= github.com/golangci/check v0.0.0-20180506172741-cfe4005ccda2/go.mod h1:k9Qvh+8juN+UKMCS/3jFtGICgW8O96FVaZsaxdzDkR4= +github.com/golangci/dupl v0.0.0-20180902072040-3e9179ac440a h1:w8hkcTqaFpzKqonE9uMCefW1WDie15eSP/4MssdenaM= github.com/golangci/dupl v0.0.0-20180902072040-3e9179ac440a/go.mod h1:ryS0uhF+x9jgbj/N71xsEqODy9BN81/GonCZiOzirOk= +github.com/golangci/errcheck v0.0.0-20181223084120-ef45e06d44b6 h1:YYWNAGTKWhKpcLLt7aSj/odlKrSrelQwlovBpDuf19w= github.com/golangci/errcheck v0.0.0-20181223084120-ef45e06d44b6/go.mod h1:DbHgvLiFKX1Sh2T1w8Q/h4NAI8MHIpzCdnBUDTXU3I0= +github.com/golangci/go-misc v0.0.0-20180628070357-927a3d87b613 h1:9kfjN3AdxcbsZBf8NjltjWihK2QfBBBZuv91cMFfDHw= github.com/golangci/go-misc v0.0.0-20180628070357-927a3d87b613/go.mod h1:SyvUF2NxV+sN8upjjeVYr5W7tyxaT1JVtvhKhOn2ii8= +github.com/golangci/goconst v0.0.0-20180610141641-041c5f2b40f3 h1:pe9JHs3cHHDQgOFXJJdYkK6fLz2PWyYtP4hthoCMvs8= github.com/golangci/goconst v0.0.0-20180610141641-041c5f2b40f3/go.mod h1:JXrF4TWy4tXYn62/9x8Wm/K/dm06p8tCKwFRDPZG/1o= +github.com/golangci/gocyclo v0.0.0-20180528134321-2becd97e67ee h1:J2XAy40+7yz70uaOiMbNnluTg7gyQhtGqLQncQh+4J8= github.com/golangci/gocyclo v0.0.0-20180528134321-2becd97e67ee/go.mod h1:ozx7R9SIwqmqf5pRP90DhR2Oay2UIjGuKheCBCNwAYU= +github.com/golangci/gofmt v0.0.0-20190930125516-244bba706f1a h1:iR3fYXUjHCR97qWS8ch1y9zPNsgXThGwjKPrYfqMPks= github.com/golangci/gofmt v0.0.0-20190930125516-244bba706f1a/go.mod h1:9qCChq59u/eW8im404Q2WWTrnBUQKjpNYKMbU4M7EFU= +github.com/golangci/golangci-lint v1.21.0 h1:HxAxpR8Z0M8omihvQdsD3PF0qPjlqYqp2vMJzstoKeI= github.com/golangci/golangci-lint v1.21.0/go.mod h1:phxpHK52q7SE+5KpPnti4oZTdFCEsn/tKN+nFvCKXfk= +github.com/golangci/ineffassign v0.0.0-20190609212857-42439a7714cc h1:gLLhTLMk2/SutryVJ6D4VZCU3CUqr8YloG7FPIBWFpI= github.com/golangci/ineffassign v0.0.0-20190609212857-42439a7714cc/go.mod h1:e5tpTHCfVze+7EpLEozzMB3eafxo2KT5veNg1k6byQU= +github.com/golangci/lint-1 v0.0.0-20191013205115-297bf364a8e0 h1:MfyDlzVjl1hoaPzPD4Gpb/QgoRfSBR0jdhwGyAWwMSA= github.com/golangci/lint-1 v0.0.0-20191013205115-297bf364a8e0/go.mod h1:66R6K6P6VWk9I95jvqGxkqJxVWGFy9XlDwLwVz1RCFg= +github.com/golangci/maligned v0.0.0-20180506175553-b1d89398deca h1:kNY3/svz5T29MYHubXix4aDDuE3RWHkPvopM/EDv/MA= github.com/golangci/maligned v0.0.0-20180506175553-b1d89398deca/go.mod h1:tvlJhZqDe4LMs4ZHD0oMUlt9G2LWuDGoisJTBzLMV9o= +github.com/golangci/misspell v0.0.0-20180809174111-950f5d19e770 h1:EL/O5HGrF7Jaq0yNhBLucz9hTuRzj2LdwGBOaENgxIk= github.com/golangci/misspell v0.0.0-20180809174111-950f5d19e770/go.mod h1:dEbvlSfYbMQDtrpRMQU675gSDLDNa8sCPPChZ7PhiVA= +github.com/golangci/prealloc v0.0.0-20180630174525-215b22d4de21 h1:leSNB7iYzLYSSx3J/s5sVf4Drkc68W2wm4Ixh/mr0us= github.com/golangci/prealloc v0.0.0-20180630174525-215b22d4de21/go.mod h1:tf5+bzsHdTM0bsB7+8mt0GUMvjCgwLpTapNZHU8AajI= +github.com/golangci/revgrep v0.0.0-20180526074752-d9c87f5ffaf0 h1:HVfrLniijszjS1aiNg8JbBMO2+E1WIQ+j/gL4SQqGPg= github.com/golangci/revgrep v0.0.0-20180526074752-d9c87f5ffaf0/go.mod h1:qOQCunEYvmd/TLamH+7LlVccLvUH5kZNhbCgTHoBbp4= +github.com/golangci/unconvert v0.0.0-20180507085042-28b1c447d1f4 h1:zwtduBRr5SSWhqsYNgcuWO2kFlpdOZbP0+yRjmvPGys= github.com/golangci/unconvert v0.0.0-20180507085042-28b1c447d1f4/go.mod h1:Izgrg8RkN3rCIMLGE9CyYmU9pY2Jer6DgANEnZ/L/cQ= github.com/golangplus/bytes v0.0.0-20160111154220-45c989fe5450/go.mod h1:Bk6SMAONeMXrxql8uvOKuAZSu8aM5RUGv+1C6IJaEho= github.com/golangplus/fmt v0.0.0-20150411045040-2a5d6d7d2995/go.mod h1:lJgMEyOkYFkPcDKwRXegd+iM6E7matEszMG5HhwytU8= @@ -175,12 +207,18 @@ github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d h1:7XGaL1e6bYS1 github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= github.com/gophercloud/gophercloud v0.1.0 h1:P/nh25+rzXouhytV2pUHBb65fnds26Ghl8/391+sT5o= github.com/gophercloud/gophercloud v0.1.0/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEoIEcSTewFxm1c5g8= +github.com/gorilla/context v0.0.0-20160226214623-1ea25387ff6f h1:9oNbS1z4rVpbnkHBdPZU4jo9bSmrLpII768arSyMFgk= github.com/gorilla/context v0.0.0-20160226214623-1ea25387ff6f/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= +github.com/gorilla/mux v1.6.0 h1:UykbtMB/w5No2LmE16gINgLj+r/vbziTgaoERQv6U+0= github.com/gorilla/mux v1.6.0/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/gorilla/securecookie v0.0.0-20160422134519-667fe4e3466a h1:YH0IojQwndMQdeRWdw1aPT8bkbiWaYR3WD+Zf5e09DU= github.com/gorilla/securecookie v0.0.0-20160422134519-667fe4e3466a/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= +github.com/gorilla/sessions v0.0.0-20160922145804-ca9ada445741 h1:OuuPl66BpF1q3OEkaPpp+VfzxrBBY62ATGdWqql/XX8= github.com/gorilla/sessions v0.0.0-20160922145804-ca9ada445741/go.mod h1:+WVp8kdw6VhyKExm03PAMRn2ZxnPtm58pV0dBVPdhHE= github.com/gorilla/websocket v1.2.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= +github.com/gorilla/websocket v1.4.0 h1:WDFjx/TMzVgy9VdMMQi2K2Emtwi2QcUQsztZ/zLaH/Q= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= +github.com/gostaticanalysis/analysisutil v0.0.0-20190318220348-4088753ea4d3 h1:JVnpOZS+qxli+rgVl98ILOXVNbW+kb5wcxeGx8ShUIw= github.com/gostaticanalysis/analysisutil v0.0.0-20190318220348-4088753ea4d3/go.mod h1:eEOZF4jCKGi+aprrirO9e7WKB3beBRtWgqGunKl6pKE= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 h1:pdN6V1QBWetyv/0+wjACpqVH+eVULgEjkurDLq3goeM= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= @@ -194,6 +232,7 @@ github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHh github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= @@ -211,6 +250,7 @@ github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= +github.com/kisielk/gotool v1.0.0 h1:AV2c/EiW3KqPNT9ZKl07ehoAGi4C5/01Cfbblndcapg= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.4.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= @@ -231,23 +271,29 @@ github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de h1:9TO3cAIGXtEhn github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de/go.mod h1:zAbeS9B/r2mtpb6U+EI2rYA5OAXxsYw6wTamcNW+zcE= github.com/lithammer/dedent v1.1.0/go.mod h1:jrXYCQtgg0nJiN+StA2KgR7w6CiQNv9Fd/Z9BP0jIOc= github.com/logrusorgru/aurora v0.0.0-20181002194514-a7b3b318ed4e/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= +github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDePerRcY= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.7.0 h1:aizVhC/NAAcKWb+5QsU1iNOZb4Yws5UO2I+aIprQITM= github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= +github.com/matoous/godox v0.0.0-20190911065817-5d6d842e92eb h1:RHba4YImhrUVQDHUCe2BNSOz4tVy2yGyXhvYDvxGgeE= github.com/matoous/godox v0.0.0-20190911065817-5d6d842e92eb/go.mod h1:1BELzlh859Sh1c6+90blK8lbYy0kwQf1bYlBhBysy1s= +github.com/mattn/go-colorable v0.1.4 h1:snbPLB8fVfU9iwbbo30TPtbLRzwWu6aJS6Xh4eaaviA= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-runewidth v0.0.7 h1:Ei8KR0497xHyKJPAv59M1dkC+rOZCMBJ+t3fZ+twI54= github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-ps v0.0.0-20190716172923-621e5597135b/go.mod h1:r1VsdOzOPt1ZSrGZWFoNhsAedKnEd6r9Np1+5blZCWk= github.com/mitchellh/go-wordwrap v1.0.0 h1:6GlHJ/LTGMrIJbwgdqdl2eEH8o+Exx/0m8ir9Gns0u4= github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo= +github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= @@ -256,11 +302,13 @@ github.com/modern-go/reflect2 v0.0.0-20180320133207-05fbef0ca5da/go.mod h1:bx2lN github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/monopole/mdrip v1.0.1 h1:LzwNWb8ge3+4iBZmxIE6VfUR5KIxhF7DKdf85t8Yvos= github.com/monopole/mdrip v1.0.1/go.mod h1:/7E04hlzRG9Jrp6WILZfYYm/REoJWL2l+MlsCO1eH74= github.com/mozilla/tls-observatory v0.0.0-20190404164649-a3c1b6cfecfd/go.mod h1:SrKMQvPiws7F7iqYp8/TX+IhxCYhzr6N/1yb8cwHsGk= github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= +github.com/nbutton23/zxcvbn-go v0.0.0-20180912185939-ae427f1e4c1d h1:AREM5mwr4u1ORQBMvzfzBgpsctsbQikCVpvC+tX285E= github.com/nbutton23/zxcvbn-go v0.0.0-20180912185939-ae427f1e4c1d/go.mod h1:o96djdrsSGy3AWPyBgZMAGfxZNfgntdJG+11KU4QvbU= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/olekukonko/tablewriter v0.0.4 h1:vHD/YYe1Wolo78koG299f7V/VAS08c6IpCLn+Ejf/w8= @@ -273,6 +321,7 @@ github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGV github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME= github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= +github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= @@ -304,44 +353,58 @@ github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6So github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/russross/blackfriday v1.5.2 h1:HyvC0ARfnZBqnXwABFeSZHpKvJHJJfPz81GNueLj0oo= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= +github.com/securego/gosec v0.0.0-20191002120514-e680875ea14d h1:BzRvVq1EHuIjxpijCEKpAxzKUUMurOQ4sknehIATRh8= github.com/securego/gosec v0.0.0-20191002120514-e680875ea14d/go.mod h1:w5+eXa0mYznDkHaMCXA4XYffjlH+cy1oyKbfzJXa2Do= github.com/shirou/gopsutil v0.0.0-20190901111213-e4ec7b275ada/go.mod h1:WWnYX4lzhCH5h/3YBfyVA3VbLYjlMZZAQcW9ojMexNc= github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc= +github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e h1:MZM7FHLqUHYI0Y/mQAt3d2aYa0SiNms/hFqC9qJYolM= github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk= +github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041 h1:llrF3Fs4018ePo4+G/HV/uQUqEI1HMDjCeOf2V6puPc= github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041/go.mod h1:N5mDOmsrJOB+vfqUK+7DmDyjhSLIIBnXo9lvZJj3MWQ= +github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= +github.com/sourcegraph/go-diff v0.5.1 h1:gO6i5zugwzo1RVTvgvfwCOSVegNuvnNi6bAD1QCmkHs= github.com/sourcegraph/go-diff v0.5.1/go.mod h1:j2dHj3m8aZgQO8lMTcTnBcXkRRRqi34cd2MNlA9u1mE= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/afero v1.2.2 h1:5jhuqJyZCZf2JRofRvN/nIFgIWNzPa3/Vz8mYylgbWc= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= +github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.5 h1:f0B+LkLX6DtmRH1isoNA9VTtNUK9K8xYd28JNNfOv/s= github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= +github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= +github.com/spf13/viper v1.4.0 h1:yXHLWeravcrgGyFSyCgdYpXQ9dR9c/WED3pg1RhxqEU= github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/testify v0.0.0-20151208002404-e3a8ff8ce365/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/timakin/bodyclose v0.0.0-20190930140734-f7f2e9bca95e h1:RumXZ56IrCj4CL+g1b9OL/oH0QnsF976bC8xQFYUD5Q= github.com/timakin/bodyclose v0.0.0-20190930140734-f7f2e9bca95e/go.mod h1:Qimiffbc6q9tBWlVV6x0P9sat/ao1xEkREYPPj9hphk= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= +github.com/ultraware/funlen v0.0.2 h1:Av96YVBwwNSe4MLR7iI/BIa3VyI7/djnto/pK3Uxbdo= github.com/ultraware/funlen v0.0.2/go.mod h1:Dp4UiAus7Wdb9KUZsYWZEWiRzGuM2kXM1lPbfaF6xhA= +github.com/ultraware/whitespace v0.0.4 h1:If7Va4cM03mpgrNH9k49/VOicWpGoG70XPBFFODYDsg= github.com/ultraware/whitespace v0.0.4/go.mod h1:aVMh/gQve5Maj9hQ/hg+F75lr/X5A89uZnzAmWSineA= +github.com/uudashr/gocognit v0.0.0-20190926065955-1655d0de0517 h1:ChMKTho2hWKpks/nD/FL2KqM1wuVt62oJeiE8+eFpGs= github.com/uudashr/gocognit v0.0.0-20190926065955-1655d0de0517/go.mod h1:j44Ayx2KW4+oB6SWMv8KsmHzZrOInQav7D3cQMJ5JUM= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasthttp v1.2.0/go.mod h1:4vX61m6KN+xDduDNwXrhIAVZaZaZiQ1luJk8LWSxF3s= @@ -450,6 +513,7 @@ golang.org/x/tools v0.0.0-20190719005602-e377ae9d6386/go.mod h1:jcCCGcm9btYwXyDq golang.org/x/tools v0.0.0-20190910044552-dd2b5c81c578/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190920225731-5eefd052ad72/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190930201159-7c411dea38b0/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191010075000-0337d82405ff h1:XdBG6es/oFDr1HwaxkxgVve7NB281QhxgK/i4voubFs= golang.org/x/tools v0.0.0-20191010075000-0337d82405ff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gonum.org/v1/gonum v0.0.0-20190331200053-3d26580ed485/go.mod h1:2ltnJ7xHfj0zHS40VVPYEAAMTa3ZGguvHGBSJeRWqE0= @@ -476,6 +540,7 @@ gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMy gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= +gopkg.in/russross/blackfriday.v2 v2.0.0 h1:+FlnIV8DSQnT7NZ43hcVKcdJdzZoeCmJj4Ql8gq5keA= gopkg.in/russross/blackfriday.v2 v2.0.0/go.mod h1:6sSBNz/GtOm/pJTuh5UmBK2ZHfmnxGbl2NZg1UliSOI= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= @@ -490,6 +555,7 @@ gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3 h1:3JgtbtFHMiCmsznwGVTUWbgGov+pVqnlf1dEJTNAXeM= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= k8s.io/api v0.17.0 h1:H9d/lw+VkZKEVIUc8F3wgiQ+FUXTTr21M87jXLU7yqM= k8s.io/api v0.17.0/go.mod h1:npsyOePkeP0CPwyGfXDHxvypiYMJxBWAMpQxCaJ4ZxI= @@ -520,8 +586,11 @@ modernc.org/golex v1.0.0/go.mod h1:b/QX9oBD/LhixY6NDh+IdGv17hgB+51fET1i2kPSmvk= modernc.org/mathutil v1.0.0/go.mod h1:wU0vUrJsVWBZ4P6e7xtFJEhFSNsfRLJ8H458uRjg03k= modernc.org/strutil v1.0.0/go.mod h1:lstksw84oURvj9y3tn8lGvRxyRC1S2+g5uuIzNfIOBs= modernc.org/xc v1.0.0/go.mod h1:mRNCo0bvLjGhHO9WsyuKVU4q0ceiDDDoEeWDJHrNx8I= +mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed h1:WX1yoOaKQfddO/mLzdV4wptyWgoH/6hwLs7QHTixo0I= mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed/go.mod h1:Xkxe497xwlCKkIaQYRfC7CSLworTXY9RMqwhhCm+8Nc= +mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b h1:DxJ5nJdkhDlLok9K6qO+5290kphDJbHOQO1DFFFTeBo= mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b/go.mod h1:2odslEg/xrtNQqCYg2/jCoyKnw3vv5biOc3JnIcYfL4= +mvdan.cc/unparam v0.0.0-20190720180237-d51796306d8f h1:Cq7MalBHYACRd6EesksG1Q8EoIAKOsiZviGKbOLIej4= mvdan.cc/unparam v0.0.0-20190720180237-d51796306d8f/go.mod h1:4G1h5nDURzA3bwVMZIVpwbkw+04kSxk3rAtzlimaUJw= sigs.k8s.io/kustomize v2.0.3+incompatible h1:JUufWFNlI44MdtnjUqVnvh29rR37PQFzPbLXqhyOyX0= sigs.k8s.io/kustomize v2.0.3+incompatible/go.mod h1:MkjgH3RdOWrievjo6c9T245dYlB5QeXV4WCbnt/PEpU= @@ -537,5 +606,6 @@ sigs.k8s.io/kustomize/pluginator/v2 v2.0.0/go.mod h1:zrXhTv8BAKt0egmZX/8AtMOSFUS sigs.k8s.io/structured-merge-diff v0.0.0-20190525122527-15d366b2352e/go.mod h1:wWxsB5ozmmv/SG7nM11ayaAW51xMvak/t1r0CSlcokI= sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= +sourcegraph.com/sqs/pbtypes v0.0.0-20180604144634-d3ebe8f20ae4 h1:JPJh2pk3+X4lXAkZIk2RuE/7/FoK9maXw+TNPJhVS/c= sourcegraph.com/sqs/pbtypes v0.0.0-20180604144634-d3ebe8f20ae4/go.mod h1:ketZ/q3QxT9HOBeFhu6RdvsftgpsbFHBF5Cas6cDKZ0= vbom.ml/util v0.0.0-20160121211510-db5cfe13f5cc/go.mod h1:so/NYdZXCz+E3ZpW0uAoCj6uzU2+8OWDFv/HxUSs7kI= From c66dd497c3ac737665d9a324debc64178b51858c Mon Sep 17 00:00:00 2001 From: Phillip Wittrock Date: Tue, 17 Dec 2019 14:01:00 -0800 Subject: [PATCH 12/27] Update api to 0.3.1 --- kustomize/go.mod | 2 +- kustomize/go.sum | 1 + releasing/VERSIONS | 4 ++-- releasing/releasemodule.sh | 24 +++++++----------------- 4 files changed, 11 insertions(+), 20 deletions(-) diff --git a/kustomize/go.mod b/kustomize/go.mod index da1af47ab..3ff28c01a 100644 --- a/kustomize/go.mod +++ b/kustomize/go.mod @@ -6,7 +6,7 @@ require ( github.com/pkg/errors v0.8.1 github.com/spf13/cobra v0.0.5 github.com/spf13/pflag v1.0.5 - sigs.k8s.io/kustomize/api v0.3.0 + sigs.k8s.io/kustomize/api v0.3.1 sigs.k8s.io/kustomize/cmd/config v0.0.2 sigs.k8s.io/kustomize/cmd/kubectl v0.0.2 sigs.k8s.io/kustomize/kyaml v0.0.2 diff --git a/kustomize/go.sum b/kustomize/go.sum index f542a9868..79a60710e 100644 --- a/kustomize/go.sum +++ b/kustomize/go.sum @@ -596,6 +596,7 @@ sigs.k8s.io/kustomize v2.0.3+incompatible h1:JUufWFNlI44MdtnjUqVnvh29rR37PQFzPbL sigs.k8s.io/kustomize v2.0.3+incompatible/go.mod h1:MkjgH3RdOWrievjo6c9T245dYlB5QeXV4WCbnt/PEpU= sigs.k8s.io/kustomize/api v0.3.0 h1:e7Erw2n8lT8+IWUukktozF0bgWwH2fFC+qsXP0gabg0= sigs.k8s.io/kustomize/api v0.3.0/go.mod h1:4jaPCtRzxfQLFdYq4gYo40dBGW1hyPp/f4AuiZB5dAQ= +sigs.k8s.io/kustomize/api v0.3.1/go.mod h1:A+ATnlHqzictQfQC1q3KB/T6MSr0UWQsrrLxMWkge2E= sigs.k8s.io/kustomize/cmd/config v0.0.2 h1:FphfIoGJ0jGGJJXq9WoG5sqqEIuTeDGx58E5NWHV8Hc= sigs.k8s.io/kustomize/cmd/config v0.0.2/go.mod h1:c6IBoPpAAm5a2aD+0iH8IfeyCF5GPsY5Ws57Dwpcvg0= sigs.k8s.io/kustomize/cmd/kubectl v0.0.2 h1:MxUAU5ie0tqx2MuDrUlcAL+Mgt8LVFcXc2scinSD8/w= diff --git a/releasing/VERSIONS b/releasing/VERSIONS index 7ec4de9e6..35521dd0c 100644 --- a/releasing/VERSIONS +++ b/releasing/VERSIONS @@ -11,7 +11,7 @@ export kyaml_patch=2 # kustomize api version export api_major=0 export api_minor=3 -export api_patch=0 +export api_patch=1 # cmd/config version export cmd_config_major=0 @@ -26,7 +26,7 @@ export cmd_kubectl_patch=2 # kustomize version export kustomize_major=3 export kustomize_minor=5 -export kustomize_patch=1 +export kustomize_patch=3 export pluginator_major=2 export pluginator_minor=1 diff --git a/releasing/releasemodule.sh b/releasing/releasemodule.sh index cde4b7a33..888c18b8c 100755 --- a/releasing/releasemodule.sh +++ b/releasing/releasemodule.sh @@ -63,24 +63,10 @@ function releaseModule { rm -rf $wktree git worktree prune git branch -D $branch + echo "$module complete" } -function releaseBinary { - # move the latest tag for the binary to trigger cloudbuild - binary=$1 - echo "binary: $binary" - - if [ "$NO_DRY_RUN" == "true" ]; then - git tag -d latest_$binary - git push upstream :latest_$binary - git tag -a latest_$binary - git push upstream latest_$binary - else - echo "Skipping push binary $binary -- run with NO_DRY_RUN=true to push the release." - fi -} - modules="kyaml api cmd/config cmd/kubectl pluginator kustomize" # configure the branch and tag names @@ -112,7 +98,11 @@ fi # release the module releaseModule $module -# release the kustomize binary if the module is kustomize if [ "$module" == "kustomize" ]; then - releaseBinary $module + # TODO: Do this for all modules + pushd . + getter=$(mktemp -d /tmp/kustomize-releases-XXXXXX) + cd $getter + go get sigs.k8s.io/kustomize/$module/v3 + popd fi From cc8b100331d8ae34bf861aa4ab24acafc3b8516a Mon Sep 17 00:00:00 2001 From: Ilia Zlobin Date: Wed, 18 Dec 2019 17:04:43 +0300 Subject: [PATCH 13/27] Handle variables in annotations --- api/konfig/builtinpluginconsts/varreference.go | 2 ++ api/krusty/variableref_test.go | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/api/konfig/builtinpluginconsts/varreference.go b/api/konfig/builtinpluginconsts/varreference.go index 823745716..dbf2b45be 100644 --- a/api/konfig/builtinpluginconsts/varreference.go +++ b/api/konfig/builtinpluginconsts/varreference.go @@ -190,5 +190,7 @@ varReference: kind: StatefulSet - path: metadata/labels + +- path: metadata/annotations ` ) diff --git a/api/krusty/variableref_test.go b/api/krusty/variableref_test.go index 8ca85f9fc..37b9bf082 100644 --- a/api/krusty/variableref_test.go +++ b/api/krusty/variableref_test.go @@ -1157,6 +1157,8 @@ vars: name: my-deployment labels: my-label: $(NAMESPACE) + annotations: + my-annotation: $(NAMESPACE) spec: template: spec: @@ -1176,6 +1178,8 @@ vars: apiVersion: apps/v1 kind: Deployment metadata: + annotations: + my-annotation: my-namespace labels: my-label: my-namespace name: my-deployment From e44d1298dfc49eb9415f59d56860d48e429f0144 Mon Sep 17 00:00:00 2001 From: Haiyan Meng Date: Mon, 16 Dec 2019 10:51:40 -0800 Subject: [PATCH 14/27] Return errors if http Client.Do resp status code is not 2xx --- api/internal/crawl/crawler/github/crawler.go | 26 +++++++++++++------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/api/internal/crawl/crawler/github/crawler.go b/api/internal/crawl/crawler/github/crawler.go index 66fa9b20c..78e9e8411 100644 --- a/api/internal/crawl/crawler/github/crawler.go +++ b/api/internal/crawl/crawler/github/crawler.go @@ -115,18 +115,18 @@ func (gc githubCrawler) FetchDocument(_ context.Context, d *doc.Document) error d.FilePath = d.FilePath + path return nil } + return err } - resp, err := gc.client.GetRawUserContent(url) - if err := handle(resp, err, ""); err == nil { + resp, errGetRawUserContent := gc.client.GetRawUserContent(url) + if err := handle(resp, errGetRawUserContent, ""); err == nil { return nil } for _, file := range konfig.RecognizedKustomizationFileNames() { - resp, err = gc.client.GetRawUserContent(url + "/" + file) - err := handle(resp, err, "/"+file) - if err != nil { - continue + resp, errGetRawUserContent = gc.client.GetRawUserContent(url + "/" + file) + if err = handle(resp, errGetRawUserContent, "/"+file); err == nil { + return nil } } return fmt.Errorf("file not found: %s, error: %v", url, err) @@ -559,7 +559,15 @@ func (gcl GhClient) Do(query string) (*http.Response, error) { return nil, err } req.Header.Add("Authorization", fmt.Sprintf("token %s", gcl.accessToken)) - return gcl.client.Do(req) + + // gcl.client.Do: a non-2xx status code doesn't cause an error. + // See https://golang.org/pkg/net/http/#Client.Do for more info. + resp, err := gcl.client.Do(req) + if resp.StatusCode != http.StatusOK { + err = fmt.Errorf("GhClient.Do(%s) failed with response code: %d", + query, resp.StatusCode) + } + return resp, err } func (gcl GhClient) getWithRetry( @@ -574,8 +582,8 @@ func (gcl GhClient) getWithRetry( retryCount > 0 { retryTime := resp.Header.Get("Retry-After") - i, err := strconv.Atoi(retryTime) - if err != nil { + i, errAtoi := strconv.Atoi(retryTime) + if errAtoi != nil { return resp, fmt.Errorf( "query '%s' forbidden without 'Retry-After'", query) } From 12fc8f41c74ff7794491de4e26acf60901549263 Mon Sep 17 00:00:00 2001 From: Haiyan Meng Date: Mon, 16 Dec 2019 11:08:38 -0800 Subject: [PATCH 15/27] Add support for github paths starting with "git@github.com:" --- api/internal/crawl/doc/docname.go | 12 +++++++++--- api/internal/crawl/doc/docname_test.go | 6 ++++++ 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/api/internal/crawl/doc/docname.go b/api/internal/crawl/doc/docname.go index ee05b2d3f..5afca1290 100644 --- a/api/internal/crawl/doc/docname.go +++ b/api/internal/crawl/doc/docname.go @@ -78,11 +78,17 @@ func (doc *Document) ID() string { } func (doc *Document) RepositoryFullName() string { - doc.RepositoryURL = strings.TrimRight(doc.RepositoryURL, "/") - sections := strings.Split(doc.RepositoryURL, "/") + url := strings.TrimRight(doc.RepositoryURL, "/") + + gitPrefix := "git@github.com:" + if strings.HasPrefix(url, gitPrefix) { + url = url[len(gitPrefix):] + } + + sections := strings.Split(url, "/") l := len(sections) if l < 2 { - return doc.RepositoryURL + return url } return path.Join(sections[l-2], sections[l-1]) } diff --git a/api/internal/crawl/doc/docname_test.go b/api/internal/crawl/doc/docname_test.go index afcd702cb..f1b65dc8f 100644 --- a/api/internal/crawl/doc/docname_test.go +++ b/api/internal/crawl/doc/docname_test.go @@ -92,6 +92,12 @@ func TestDocument_RepositoryFullName(t *testing.T) { }, expectedRepositoryFullName: "", }, + { + doc: Document{ + RepositoryURL: "git@github.com:user/repo", + }, + expectedRepositoryFullName: "user/repo", + }, } for _, tc := range testCases { From 8c89f0946c452aacb2b0308b858ae7dcbda48788 Mon Sep 17 00:00:00 2001 From: Haiyan Meng Date: Mon, 16 Dec 2019 11:39:54 -0800 Subject: [PATCH 16/27] Avoid to index a document if FetchDcoument or SetCreated fails --- api/internal/crawl/crawler/crawler.go | 36 +++++++++++--------- api/internal/crawl/crawler/github/crawler.go | 3 ++ api/internal/crawl/doc/doc.go | 2 ++ 3 files changed, 25 insertions(+), 16 deletions(-) diff --git a/api/internal/crawl/crawler/crawler.go b/api/internal/crawl/crawler/crawler.go index f57f247e5..410c3454b 100644 --- a/api/internal/crawl/crawler/crawler.go +++ b/api/internal/crawl/crawler/crawler.go @@ -72,17 +72,17 @@ func addBranches(cdoc CrawledDocument, match Crawler, indx IndexFunc, seen[cdoc.ID()] = struct{}{} // Insert into index - err := indx(cdoc, match) - logIfErr(err) - if err != nil { + if err := indx(cdoc, match); err != nil { + logger.Println("Failed to index: ", err) return } deps, err := cdoc.GetResources() - logIfErr(err) if err != nil { + logger.Println(err) return } + for _, dep := range deps { if _, ok := seen[dep.ID()]; ok { continue @@ -107,29 +107,33 @@ func doCrawl(ctx context.Context, docsPtr *CrawlSeed, crawlers []Crawler, conv C } docCount++ + if tail.WasCached() { + logger.Printf("%s %s is cached already", tail.RepositoryURL, tail.FilePath) + continue + } + match := findMatch(tail, crawlers) if match == nil { - logIfErr(fmt.Errorf( - "%v could not match any crawler", tail)) + logIfErr(fmt.Errorf("%v could not match any crawler", tail)) continue } logger.Println("Crawling ", tail.RepositoryURL, tail.FilePath) - err := match.FetchDocument(ctx, tail) - logIfErr(err) - // If there was no change or there is an error, we don't have - // to branch out, since the dependencies are already in the - // index, or we cannot find the document. - if err != nil || tail.WasCached() { - if tail.WasCached() { - logger.Println(tail.RepositoryURL, tail.FilePath, "is cached already") - } + if err := match.FetchDocument(ctx, tail); err != nil { + logger.Printf("FetchDocument failed on %s %s: %v", + tail.RepositoryURL, tail.FilePath, err) continue } - logIfErr(match.SetCreated(ctx, tail)) + if err := match.SetCreated(ctx, tail); err != nil { + logger.Printf("SetCreated failed on %s %s: %v", + tail.RepositoryURL, tail.FilePath, err) + continue + } cdoc, err := conv(tail) + // If conv returns an error, cdoc can still be added into the index so that + // cdoc.Document can be searched. logIfErr(err) addBranches(cdoc, match, indx, seen, stack) diff --git a/api/internal/crawl/crawler/github/crawler.go b/api/internal/crawl/crawler/github/crawler.go index 78e9e8411..83a118553 100644 --- a/api/internal/crawl/crawler/github/crawler.go +++ b/api/internal/crawl/crawler/github/crawler.go @@ -93,6 +93,9 @@ func (gc githubCrawler) Crawl( return nil } +// FetchDocument first tries to fetch the document with d.FilePath. If it fails, +// it will try to add each string in konfig.RecognizedKustomizationFileNames() to +// d.FilePath, and try to fetch the document again. func (gc githubCrawler) FetchDocument(_ context.Context, d *doc.Document) error { repoURL := d.RepositoryURL + "/" + d.FilePath + "?ref=" + d.DefaultBranch repoSpec, err := git.NewRepoSpecFromUrl(repoURL) diff --git a/api/internal/crawl/doc/doc.go b/api/internal/crawl/doc/doc.go index 5e4af4fc4..50241e5ad 100644 --- a/api/internal/crawl/doc/doc.go +++ b/api/internal/crawl/doc/doc.go @@ -116,6 +116,8 @@ func (doc *KustomizationDocument) readBytes() ([]map[string]interface{}, error) return configs, nil } +// ParseYAML parses doc.Document and sets the following fields of doc: +// Kinds, Values, Identifiers. func (doc *KustomizationDocument) ParseYAML() error { doc.Identifiers = make([]string, 0) doc.Values = make([]string, 0) From 5598d35e4b8bc54a66ad9606345df1808e0c6390 Mon Sep 17 00:00:00 2001 From: Haiyan Meng Date: Mon, 16 Dec 2019 12:13:27 -0800 Subject: [PATCH 17/27] Add a summary for doCrawl --- api/internal/crawl/crawler/crawler.go | 32 +++++++++++++++++++++++---- 1 file changed, 28 insertions(+), 4 deletions(-) diff --git a/api/internal/crawl/crawler/crawler.go b/api/internal/crawl/crawler/crawler.go index 410c3454b..30d95ec0f 100644 --- a/api/internal/crawl/crawler/crawler.go +++ b/api/internal/crawl/crawler/crawler.go @@ -93,7 +93,15 @@ func addBranches(cdoc CrawledDocument, match Crawler, indx IndexFunc, func doCrawl(ctx context.Context, docsPtr *CrawlSeed, crawlers []Crawler, conv Converter, indx IndexFunc, seen map[string]struct{}, stack *CrawlSeed) { - docCount := 0 + + UpdatedDocCount := 0 + seenDocCount := 0 + cachedDocCount := 0 + findMatchErrCount := 0 + FetchDocumentErrCount := 0 + SetCreatedErrCount := 0 + convErrCount := 0 + // During the execution of the for loop, more Documents may be added into (*docsPtr). for len(*docsPtr) > 0 { // get the last Document in (*docPtr), which will be crawled in this iteration. @@ -103,18 +111,20 @@ func doCrawl(ctx context.Context, docsPtr *CrawlSeed, crawlers []Crawler, conv C *docsPtr = (*docsPtr)[:(len(*docsPtr) - 1)] if _, ok := seen[tail.ID()]; ok { + seenDocCount++ continue } - docCount++ if tail.WasCached() { logger.Printf("%s %s is cached already", tail.RepositoryURL, tail.FilePath) + cachedDocCount++ continue } match := findMatch(tail, crawlers) if match == nil { logIfErr(fmt.Errorf("%v could not match any crawler", tail)) + findMatchErrCount++ continue } @@ -122,23 +132,37 @@ func doCrawl(ctx context.Context, docsPtr *CrawlSeed, crawlers []Crawler, conv C if err := match.FetchDocument(ctx, tail); err != nil { logger.Printf("FetchDocument failed on %s %s: %v", tail.RepositoryURL, tail.FilePath, err) + FetchDocumentErrCount++ continue } if err := match.SetCreated(ctx, tail); err != nil { logger.Printf("SetCreated failed on %s %s: %v", tail.RepositoryURL, tail.FilePath, err) + SetCreatedErrCount++ continue } cdoc, err := conv(tail) // If conv returns an error, cdoc can still be added into the index so that // cdoc.Document can be searched. - logIfErr(err) + if err != nil { + logger.Printf("conv failed on %s %s: %v", + tail.RepositoryURL, tail.FilePath, err) + convErrCount++ + } + UpdatedDocCount++ addBranches(cdoc, match, indx, seen, stack) } - logger.Printf("%d documents were crawled by doCrawl\n", docCount) + logger.Printf("Summary of doCrawl:\n") + logger.Printf("\t%d documents were updated\n", UpdatedDocCount) + logger.Printf("\t%d documents were seen by the crawler already and skipped\n", seenDocCount) + logger.Printf("\t%d documents were cached already and skipped\n", cachedDocCount) + logger.Printf("\t%d documents didn't have a matching crawler and skipped\n", findMatchErrCount) + logger.Printf("\t%d documents cannot be fetched and skipped\n", FetchDocumentErrCount) + logger.Printf("\t%d documents cannot update its creation time and skipped\n", SetCreatedErrCount) + logger.Printf("\t%d documents cannot be converted and skipped\n", convErrCount) } // CrawlFromSeed updates all the documents in seed, and crawls all the new From 272b7a6fcd7fd06fe4a9da5bbcbca4e63df9d4b7 Mon Sep 17 00:00:00 2001 From: Haiyan Meng Date: Mon, 16 Dec 2019 12:39:13 -0800 Subject: [PATCH 18/27] Use `UpdateRequest` to insert/update a document Currently, `IndexRequest` is used to insert/update a document, which increases the version of the document every time IndexRequest.Do is called. --- api/internal/crawl/index/elasticsearch.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/api/internal/crawl/index/elasticsearch.go b/api/internal/crawl/index/elasticsearch.go index 8893dc2e8..2696d8dfd 100644 --- a/api/internal/crawl/index/elasticsearch.go +++ b/api/internal/crawl/index/elasticsearch.go @@ -180,12 +180,15 @@ func (idx *index) DeleteIndex() error { // Insert or update the document by ID. func (idx *index) Put(uniqueID string, doc interface{}) (string, error) { - body, err := json.Marshal(doc) + docBytes, err := json.Marshal(doc) if err != nil { return "", err } + body := byteJoin(`{"doc":`, docBytes, `}`) - req := esapi.IndexRequest{ + // Use `UpdateRequest` here instead of `IndexRequest`. + // For a document with a given id, every call of IndexRequest.Do will increase the version of a document. + req := esapi.UpdateRequest{ Index: idx.name, Body: bytes.NewReader(body), DocumentID: uniqueID, From 1eb713157ccc0fee511aa2089ba5cd5e32854770 Mon Sep 17 00:00:00 2001 From: Haiyan Meng Date: Mon, 16 Dec 2019 15:13:59 -0800 Subject: [PATCH 19/27] Sort the string slice fields of a document to avoid updating the index unnecessarily --- api/internal/crawl/doc/doc.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/api/internal/crawl/doc/doc.go b/api/internal/crawl/doc/doc.go index 50241e5ad..6ab581458 100644 --- a/api/internal/crawl/doc/doc.go +++ b/api/internal/crawl/doc/doc.go @@ -2,6 +2,7 @@ package doc import ( "fmt" + "sort" "strings" "sigs.k8s.io/kustomize/api/k8sdeps/kunstruct" @@ -161,6 +162,13 @@ func (doc *KustomizationDocument) ParseYAML() error { doc.Identifiers = append(doc.Identifiers, key) } + // Without sorting these fields, every time when the string order in these fields changes, + // the document in the index will be updated. + // Sorting these fields are necessary to avoid a document being updated unnecessarily. + sort.Strings(doc.Kinds) + sort.Strings(doc.Values) + sort.Strings(doc.Identifiers) + return nil } From 2c2aa928cc17c9fa2b4a17dcc757b72840760683 Mon Sep 17 00:00:00 2001 From: Haiyan Meng Date: Tue, 17 Dec 2019 09:32:11 -0800 Subject: [PATCH 20/27] Delete non-existing documents from the index --- api/internal/crawl/cmd/crawler/crawler.go | 14 +++++++--- api/internal/crawl/crawler/crawler.go | 27 +++++++++++++++----- api/internal/crawl/crawler/crawler_test.go | 3 ++- api/internal/crawl/crawler/github/crawler.go | 5 +--- api/internal/crawl/index/kustomize.go | 11 ++++++++ 5 files changed, 44 insertions(+), 16 deletions(-) diff --git a/api/internal/crawl/cmd/crawler/crawler.go b/api/internal/crawl/cmd/crawler/crawler.go index 7e14f7072..c4e913706 100644 --- a/api/internal/crawl/cmd/crawler/crawler.go +++ b/api/internal/crawl/cmd/crawler/crawler.go @@ -65,12 +65,18 @@ func main() { } // Index updates the value in the index. - index := func(cdoc crawler.CrawledDocument, crwlr crawler.Crawler) error { + index := func(cdoc crawler.CrawledDocument, crwlr crawler.Crawler, mode index.Mode) error { switch d := cdoc.(type) { case *doc.KustomizationDocument: - fmt.Println("Inserting: ", d) - _, err := idx.Put(d.ID(), d) - return err + switch mode { + case index.Delete: + fmt.Println("Deleting: ", d) + return idx.Delete(d.ID()) + default: + fmt.Println("Inserting: ", d) + _, err := idx.Put(d.ID(), d) + return err + } default: return fmt.Errorf("type %T not supported", d) } diff --git a/api/internal/crawl/crawler/crawler.go b/api/internal/crawl/crawler/crawler.go index 30d95ec0f..49c2403d1 100644 --- a/api/internal/crawl/crawler/crawler.go +++ b/api/internal/crawl/crawler/crawler.go @@ -8,6 +8,7 @@ import ( "fmt" "log" "os" + "sigs.k8s.io/kustomize/api/internal/crawl/index" "sync" _ "github.com/gomodule/redigo/redis" @@ -47,7 +48,7 @@ type CrawledDocument interface { type CrawlSeed []*doc.Document -type IndexFunc func(CrawledDocument, Crawler) error +type IndexFunc func(CrawledDocument, Crawler, index.Mode) error type Converter func(*doc.Document) (CrawledDocument, error) func logIfErr(err error) { @@ -72,8 +73,9 @@ func addBranches(cdoc CrawledDocument, match Crawler, indx IndexFunc, seen[cdoc.ID()] = struct{}{} // Insert into index - if err := indx(cdoc, match); err != nil { - logger.Println("Failed to index: ", err) + if err := indx(cdoc, match, index.InsertOrUpdate); err != nil { + logger.Printf("Failed to insert or update %s %s: %v", + cdoc.GetDocument().RepositoryURL, cdoc.GetDocument().FilePath, err) return } @@ -101,6 +103,7 @@ func doCrawl(ctx context.Context, docsPtr *CrawlSeed, crawlers []Crawler, conv C FetchDocumentErrCount := 0 SetCreatedErrCount := 0 convErrCount := 0 + deleteDocCount := 0 // During the execution of the for loop, more Documents may be added into (*docsPtr). for len(*docsPtr) > 0 { @@ -133,6 +136,16 @@ func doCrawl(ctx context.Context, docsPtr *CrawlSeed, crawlers []Crawler, conv C logger.Printf("FetchDocument failed on %s %s: %v", tail.RepositoryURL, tail.FilePath, err) FetchDocumentErrCount++ + // delete the document from the index + cdoc := &doc.KustomizationDocument{ + Document: *tail, + } + seen[cdoc.ID()] = struct{}{} + if err := indx(cdoc, match, index.Delete); err != nil { + logger.Printf("Failed to delete %s %s: %v", + cdoc.RepositoryURL, cdoc.FilePath, err) + } + deleteDocCount++ continue } @@ -140,7 +153,6 @@ func doCrawl(ctx context.Context, docsPtr *CrawlSeed, crawlers []Crawler, conv C logger.Printf("SetCreated failed on %s %s: %v", tail.RepositoryURL, tail.FilePath, err) SetCreatedErrCount++ - continue } cdoc, err := conv(tail) @@ -160,9 +172,10 @@ func doCrawl(ctx context.Context, docsPtr *CrawlSeed, crawlers []Crawler, conv C logger.Printf("\t%d documents were seen by the crawler already and skipped\n", seenDocCount) logger.Printf("\t%d documents were cached already and skipped\n", cachedDocCount) logger.Printf("\t%d documents didn't have a matching crawler and skipped\n", findMatchErrCount) - logger.Printf("\t%d documents cannot be fetched and skipped\n", FetchDocumentErrCount) - logger.Printf("\t%d documents cannot update its creation time and skipped\n", SetCreatedErrCount) - logger.Printf("\t%d documents cannot be converted and skipped\n", convErrCount) + logger.Printf("\t%d documents cannot be fetched, %d out of them are deleted\n", + FetchDocumentErrCount, deleteDocCount) + logger.Printf("\t%d documents cannot update its creation time but still were inserted or updated in the index\n", SetCreatedErrCount) + logger.Printf("\t%d documents cannot be converted but still were inserted or updated in the index\n", convErrCount) } // CrawlFromSeed updates all the documents in seed, and crawls all the new diff --git a/api/internal/crawl/crawler/crawler_test.go b/api/internal/crawl/crawler/crawler_test.go index 64887b1fc..ec479facf 100644 --- a/api/internal/crawl/crawler/crawler_test.go +++ b/api/internal/crawl/crawler/crawler_test.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "reflect" + "sigs.k8s.io/kustomize/api/internal/crawl/index" "sort" "strings" "sync" @@ -316,7 +317,7 @@ resources: Document: *d, }, nil }, - func(d CrawledDocument, cr Crawler) error { + func(d CrawledDocument, cr Crawler, mode index.Mode) error { visited[d.ID()]++ return nil }, diff --git a/api/internal/crawl/crawler/github/crawler.go b/api/internal/crawl/crawler/github/crawler.go index 83a118553..fec1628b0 100644 --- a/api/internal/crawl/crawler/github/crawler.go +++ b/api/internal/crawl/crawler/github/crawler.go @@ -580,10 +580,7 @@ func (gcl GhClient) getWithRetry( retryCount := gcl.retryCount - for err == nil && - resp.StatusCode == http.StatusForbidden && - retryCount > 0 { - + for resp.StatusCode == http.StatusForbidden && retryCount > 0 { retryTime := resp.Header.Get("Retry-After") i, errAtoi := strconv.Atoi(retryTime) if errAtoi != nil { diff --git a/api/internal/crawl/index/kustomize.go b/api/internal/crawl/index/kustomize.go index 79833d3fd..430142fc3 100644 --- a/api/internal/crawl/index/kustomize.go +++ b/api/internal/crawl/index/kustomize.go @@ -16,6 +16,12 @@ const ( AggregationKeyword = "aggs" ) +type Mode int +const ( + InsertOrUpdate = iota + Delete +) + // Redefinition of Hits structure. Must match the json string of // KustomizeResult.Hits.Hits. Declared as a convenience for iteration. type KustomizeHits []struct { @@ -301,6 +307,11 @@ func (ki *KustomizeIndex) Put(id string, doc *doc.KustomizationDocument) (string return id, nil } +// Delete a document with a given id from the kustomize index. +func (ki *KustomizeIndex) Delete(id string) error { + return ki.index.Delete(id) +} + // Kustomize search options: What metrics should be returned? Kind Aggregation, // TimeseriesAggregation, etc. Also embedds the SearchOptions field to specify // the position in the sorted list of results and the number of results to return. From bef157d6b38d15917b81567429f20718b995a880 Mon Sep 17 00:00:00 2001 From: Haiyan Meng Date: Tue, 17 Dec 2019 12:14:41 -0800 Subject: [PATCH 21/27] Fix insert/updating document logic --- api/internal/crawl/cmd/crawler/crawler.go | 9 ++- api/internal/crawl/index/elasticsearch.go | 82 ++++++++++++++--------- api/internal/crawl/index/kustomize.go | 8 +-- 3 files changed, 57 insertions(+), 42 deletions(-) diff --git a/api/internal/crawl/cmd/crawler/crawler.go b/api/internal/crawl/cmd/crawler/crawler.go index c4e913706..ea73efd04 100644 --- a/api/internal/crawl/cmd/crawler/crawler.go +++ b/api/internal/crawl/cmd/crawler/crawler.go @@ -65,7 +65,7 @@ func main() { } // Index updates the value in the index. - index := func(cdoc crawler.CrawledDocument, crwlr crawler.Crawler, mode index.Mode) error { + indexFunc := func(cdoc crawler.CrawledDocument, crwlr crawler.Crawler, mode index.Mode) error { switch d := cdoc.(type) { case *doc.KustomizationDocument: switch mode { @@ -74,8 +74,7 @@ func main() { return idx.Delete(d.ID()) default: fmt.Println("Inserting: ", d) - _, err := idx.Put(d.ID(), d) - return err + return idx.Put(d.ID(), d) } default: return fmt.Errorf("type %T not supported", d) @@ -123,6 +122,6 @@ func main() { } crawlers := []crawler.Crawler{ghCrawler} - crawler.CrawlFromSeed(ctx, seedDocs, crawlers, docConverter, index, seen) - crawler.CrawlGithub(ctx, crawlers, docConverter, index, seen) + crawler.CrawlFromSeed(ctx, seedDocs, crawlers, docConverter, indexFunc, seen) + crawler.CrawlGithub(ctx, crawlers, docConverter, indexFunc, seen) } diff --git a/api/internal/crawl/index/elasticsearch.go b/api/internal/crawl/index/elasticsearch.go index 2696d8dfd..b80d7f901 100644 --- a/api/internal/crawl/index/elasticsearch.go +++ b/api/internal/crawl/index/elasticsearch.go @@ -6,7 +6,6 @@ import ( "encoding/json" "fmt" "io" - "io/ioutil" "time" es "github.com/elastic/go-elasticsearch/v6" @@ -179,47 +178,47 @@ func (idx *index) DeleteIndex() error { } // Insert or update the document by ID. -func (idx *index) Put(uniqueID string, doc interface{}) (string, error) { - docBytes, err := json.Marshal(doc) +func (idx *index) Put(uniqueID string, doc interface{}) error { + exists, err := idx.Exists(uniqueID) if err != nil { - return "", err + return err } - body := byteJoin(`{"doc":`, docBytes, `}`) - // Use `UpdateRequest` here instead of `IndexRequest`. - // For a document with a given id, every call of IndexRequest.Do will increase the version of a document. - req := esapi.UpdateRequest{ - Index: idx.name, - Body: bytes.NewReader(body), - DocumentID: uniqueID, - } - res, err := req.Do(idx.ctx, idx.client) - - var id string - readId := func(reader io.Reader) error { - type InsertResult struct { - ID string `json:"_id,omitempty"` + if exists { + docBytes, err := json.Marshal(doc) + if err != nil { + return err } - var ir InsertResult - data, err := ioutil.ReadAll(reader) + body := byteJoin(`{"doc":`, docBytes, `}`) + + // For a document with a given id, every call of IndexRequest.Do will increase the version of a document. + // To avoid increasing the document version unnecessarily, use UpdateRequest here. + req := esapi.UpdateRequest{ + Index: idx.name, + Body: bytes.NewReader(body), + DocumentID: uniqueID, + } + res, err := req.Do(idx.ctx, idx.client) + + err = idx.responseErrorOrNil("could not update document", + res, err, ignoreResponseBody) + } else { + body, err := json.Marshal(doc) if err != nil { return err } - err = json.Unmarshal(data, &ir) - if err != nil { - return err + req := esapi.IndexRequest{ + Index: idx.name, + Body: bytes.NewReader(body), + DocumentID: uniqueID, } - id = ir.ID + res, err := req.Do(idx.ctx, idx.client) - return nil + err = idx.responseErrorOrNil("could not insert document", + res, err, ignoreResponseBody) } - - // populates the id field. - err = idx.responseErrorOrNil("could not insert document", - res, err, readId) - - return id, err + return err } type scrollUpdater func(string, readerFunc) error @@ -299,3 +298,24 @@ func (idx *index) Delete(id string) error { fmt.Sprintf("could not delete id(%s) from index(%s)", id, idx.name), res, err, ignoreResponseBody) } + +// Check whether a given document id is in the index +func (idx *index) Exists(id string) (bool, error) { + op := idx.client.Exists + res, err := op( + idx.name, + id, + op.WithContext(idx.ctx), + op.WithPretty(), + ) + + if !res.IsError() { + return true, nil + } else if res.StatusCode == 404 { + return false, nil + } else { + return false, idx.responseErrorOrNil( + fmt.Sprintf("could not check the existence of id(%s) from index(%s)", id, idx.name), + res, err, ignoreResponseBody) + } +} diff --git a/api/internal/crawl/index/kustomize.go b/api/internal/crawl/index/kustomize.go index 430142fc3..cedea28bb 100644 --- a/api/internal/crawl/index/kustomize.go +++ b/api/internal/crawl/index/kustomize.go @@ -299,12 +299,8 @@ func (ki *KustomizeIndex) IterateQuery(query []byte, batchSize int, } // type specific Put for inserting structured kustomization documents. -func (ki *KustomizeIndex) Put(id string, doc *doc.KustomizationDocument) (string, error) { - id, err := ki.index.Put(id, doc) - if err != nil { - return id, fmt.Errorf("could not insert in elastic: %v", err) - } - return id, nil +func (ki *KustomizeIndex) Put(id string, doc *doc.KustomizationDocument) error { + return ki.index.Put(id, doc) } // Delete a document with a given id from the kustomize index. From a35f00213914802c6e53b3d464640b3485597d3c Mon Sep 17 00:00:00 2001 From: Haiyan Meng Date: Tue, 17 Dec 2019 12:31:23 -0800 Subject: [PATCH 22/27] Run `goimports` --- api/internal/crawl/crawler/crawler.go | 5 +++-- api/internal/crawl/crawler/crawler_test.go | 3 ++- api/internal/crawl/doc/doc.go | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/api/internal/crawl/crawler/crawler.go b/api/internal/crawl/crawler/crawler.go index 49c2403d1..d46cf161d 100644 --- a/api/internal/crawl/crawler/crawler.go +++ b/api/internal/crawl/crawler/crawler.go @@ -8,9 +8,10 @@ import ( "fmt" "log" "os" - "sigs.k8s.io/kustomize/api/internal/crawl/index" "sync" + "sigs.k8s.io/kustomize/api/internal/crawl/index" + _ "github.com/gomodule/redigo/redis" "sigs.k8s.io/kustomize/api/internal/crawl/doc" @@ -138,7 +139,7 @@ func doCrawl(ctx context.Context, docsPtr *CrawlSeed, crawlers []Crawler, conv C FetchDocumentErrCount++ // delete the document from the index cdoc := &doc.KustomizationDocument{ - Document: *tail, + Document: *tail, } seen[cdoc.ID()] = struct{}{} if err := indx(cdoc, match, index.Delete); err != nil { diff --git a/api/internal/crawl/crawler/crawler_test.go b/api/internal/crawl/crawler/crawler_test.go index ec479facf..00a619c46 100644 --- a/api/internal/crawl/crawler/crawler_test.go +++ b/api/internal/crawl/crawler/crawler_test.go @@ -5,13 +5,14 @@ import ( "errors" "fmt" "reflect" - "sigs.k8s.io/kustomize/api/internal/crawl/index" "sort" "strings" "sync" "testing" "time" + "sigs.k8s.io/kustomize/api/internal/crawl/index" + "sigs.k8s.io/kustomize/api/internal/crawl/doc" "sigs.k8s.io/kustomize/api/konfig" ) diff --git a/api/internal/crawl/doc/doc.go b/api/internal/crawl/doc/doc.go index 6ab581458..953f8d4b4 100644 --- a/api/internal/crawl/doc/doc.go +++ b/api/internal/crawl/doc/doc.go @@ -47,7 +47,7 @@ type set map[string]struct{} func (doc *KustomizationDocument) String() string { return fmt.Sprintf("%s %s %s %v %v %v len(identifiers):%v len(values):%v", doc.RepositoryURL, doc.FilePath, doc.DefaultBranch, doc.CreationTime, - doc.IsSame, doc.Kinds, len(doc.Identifiers), len(doc.Values)) + doc.IsSame, doc.Kinds, len(doc.Identifiers), len(doc.Values)) } // Implements the CrawlerDocument interface. From f5ff254203320e2f02ed8f917ff1f432539cfedf Mon Sep 17 00:00:00 2001 From: Haiyan Meng Date: Tue, 17 Dec 2019 12:37:34 -0800 Subject: [PATCH 23/27] Update deps --- api/internal/crawl/go.mod | 4 +-- api/internal/crawl/go.sum | 57 +++++++++------------------------------ 2 files changed, 14 insertions(+), 47 deletions(-) diff --git a/api/internal/crawl/go.mod b/api/internal/crawl/go.mod index cabc30d5b..b8ddb8140 100644 --- a/api/internal/crawl/go.mod +++ b/api/internal/crawl/go.mod @@ -1,9 +1,9 @@ module sigs.k8s.io/kustomize/api/internal/crawl -go 1.13 +go 1.12 require ( - github.com/elastic/go-elasticsearch/v6 v6.8.2 + github.com/elastic/go-elasticsearch/v6 v6.8.5 github.com/gomodule/redigo v2.0.0+incompatible github.com/gorilla/mux v1.7.3 github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 diff --git a/api/internal/crawl/go.sum b/api/internal/crawl/go.sum index 16214211e..9fb6aa083 100644 --- a/api/internal/crawl/go.sum +++ b/api/internal/crawl/go.sum @@ -2,14 +2,10 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI= -github.com/Azure/go-autorest/autorest v0.9.2/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI= github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0= -github.com/Azure/go-autorest/autorest/adal v0.8.0/go.mod h1:Z6vX6WXXuyieHAXwMj0S6HY6e6wcHn37qQMBQlvY3lc= github.com/Azure/go-autorest/autorest/date v0.1.0/go.mod h1:plvfp3oPSKwf2DNjlBjWF/7vwR+cUD/ELuzDCXwHUVA= -github.com/Azure/go-autorest/autorest/date v0.2.0/go.mod h1:vcORJHLJEh643/Ioh9+vPmf1Ij9AEBM5FuBIXLmIy0g= github.com/Azure/go-autorest/autorest/mocks v0.1.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= github.com/Azure/go-autorest/autorest/mocks v0.2.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= -github.com/Azure/go-autorest/autorest/mocks v0.3.0/go.mod h1:a8FDP3DYzQ4RYfVAxAN3SVSiiO77gL2j2ronKKP0syM= github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc= github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= @@ -44,14 +40,10 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= -github.com/docker/spdystream v0.0.0-20181023171402-6480d4af844c/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= -github.com/elastic/go-elasticsearch/v6 v6.8.2 h1:rp5DGrd63V5c6nHLjF6QEXUpZSvs0+QM3ld7m9VhV2g= -github.com/elastic/go-elasticsearch/v6 v6.8.2/go.mod h1:UwaDJsD3rWLM5rKNFzv9hgox93HoX8utj1kxD9aFUcI= +github.com/elastic/go-elasticsearch/v6 v6.8.5 h1:U2HtkBseC1FNBmDr0TR2tKltL6FxoY+niDAlj5M8TK8= +github.com/elastic/go-elasticsearch/v6 v6.8.5/go.mod h1:UwaDJsD3rWLM5rKNFzv9hgox93HoX8utj1kxD9aFUcI= github.com/elazarl/goproxy v0.0.0-20170405201442-c4fc26588b6e/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= -github.com/elazarl/goproxy v0.0.0-20191011121108-aa519ddbe484/go.mod h1:Ro8st/ElPeALwNFlcTpWmkr6IoMFfkjXAvTHpevnDsM= -github.com/elazarl/goproxy/ext v0.0.0-20190711103511-473e67f1d7d2/go.mod h1:gNh8nYJoAm43RfaxurUnxr+N1PwuFV3ZMl/efxlIlY8= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= -github.com/emicklei/go-restful v2.9.6+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch v4.5.0+incompatible h1:ouOWdg56aJriqS0huScTkVXPC5IcNrDCXZ6OoTAWu7M= github.com/evanphx/json-patch v4.5.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= @@ -95,13 +87,11 @@ github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJA github.com/gofrs/flock v0.0.0-20190320160742-5135e617513b/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= +github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d h1:3PaI8p3seN09VjbTYC/QWlUZdZ1qS1zGjy7LH2Wt07I= github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= -github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= -github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20191027212112-611e8accdfc9/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v0.0.0-20161109072736-4bd1920723d7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -115,12 +105,9 @@ github.com/golangci/errcheck v0.0.0-20181223084120-ef45e06d44b6/go.mod h1:DbHgvL github.com/golangci/go-misc v0.0.0-20180628070357-927a3d87b613/go.mod h1:SyvUF2NxV+sN8upjjeVYr5W7tyxaT1JVtvhKhOn2ii8= github.com/golangci/goconst v0.0.0-20180610141641-041c5f2b40f3/go.mod h1:JXrF4TWy4tXYn62/9x8Wm/K/dm06p8tCKwFRDPZG/1o= github.com/golangci/gocyclo v0.0.0-20180528134321-2becd97e67ee/go.mod h1:ozx7R9SIwqmqf5pRP90DhR2Oay2UIjGuKheCBCNwAYU= -github.com/golangci/gofmt v0.0.0-20181222123516-0b8337e80d98/go.mod h1:9qCChq59u/eW8im404Q2WWTrnBUQKjpNYKMbU4M7EFU= github.com/golangci/gofmt v0.0.0-20190930125516-244bba706f1a/go.mod h1:9qCChq59u/eW8im404Q2WWTrnBUQKjpNYKMbU4M7EFU= -github.com/golangci/golangci-lint v1.19.1/go.mod h1:2CEc4Fxx3vxDv7g8DyXkHCBF73AOzAymcJAprs2vCps= github.com/golangci/golangci-lint v1.21.0/go.mod h1:phxpHK52q7SE+5KpPnti4oZTdFCEsn/tKN+nFvCKXfk= github.com/golangci/ineffassign v0.0.0-20190609212857-42439a7714cc/go.mod h1:e5tpTHCfVze+7EpLEozzMB3eafxo2KT5veNg1k6byQU= -github.com/golangci/lint-1 v0.0.0-20190420132249-ee948d087217/go.mod h1:66R6K6P6VWk9I95jvqGxkqJxVWGFy9XlDwLwVz1RCFg= github.com/golangci/lint-1 v0.0.0-20191013205115-297bf364a8e0/go.mod h1:66R6K6P6VWk9I95jvqGxkqJxVWGFy9XlDwLwVz1RCFg= github.com/golangci/maligned v0.0.0-20180506175553-b1d89398deca/go.mod h1:tvlJhZqDe4LMs4ZHD0oMUlt9G2LWuDGoisJTBzLMV9o= github.com/golangci/misspell v0.0.0-20180809174111-950f5d19e770/go.mod h1:dEbvlSfYbMQDtrpRMQU675gSDLDNa8sCPPChZ7PhiVA= @@ -132,9 +119,8 @@ github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg= -github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/gofuzz v0.0.0-20161122191042-44d81051d367/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI= github.com/google/gofuzz v1.0.0 h1:A8PeW59pxE9IoFRqBp37U+mSNaQoZ46F1f0f863XSXw= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -144,11 +130,9 @@ github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm4 github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d h1:7XGaL1e6bYS1yIonGp9761ExpPPV1ui0SAC59Yube9k= github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= -github.com/googleapis/gnostic v0.3.0 h1:CcQijm0XKekKjP/YCz28LXVSpgguuB+nCxaSjCe09y0= -github.com/googleapis/gnostic v0.3.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= github.com/gophercloud/gophercloud v0.1.0/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEoIEcSTewFxm1c5g8= -github.com/gophercloud/gophercloud v0.6.0/go.mod h1:GICNByuaEBibcjmjvI7QvYJSZEbGkcYwAR7EZK2WMqM= github.com/gorilla/context v0.0.0-20160226214623-1ea25387ff6f/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= github.com/gorilla/mux v1.6.0/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/mux v1.7.3 h1:gnP5JzjVOuiZD07fKKToCAOjS0yOpj/qPETTXCCS6hw= @@ -166,7 +150,6 @@ github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgf github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.3/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= @@ -200,9 +183,7 @@ github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czP github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/matoous/godox v0.0.0-20190910121045-032ad8106c86/go.mod h1:1BELzlh859Sh1c6+90blK8lbYy0kwQf1bYlBhBysy1s= github.com/matoous/godox v0.0.0-20190911065817-5d6d842e92eb/go.mod h1:1BELzlh859Sh1c6+90blK8lbYy0kwQf1bYlBhBysy1s= -github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw= @@ -217,7 +198,6 @@ github.com/modern-go/reflect2 v0.0.0-20180320133207-05fbef0ca5da/go.mod h1:bx2lN github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/monopole/mdrip v1.0.0/go.mod h1:N1/ppRG9CaPeUKAUHZ3dUlfOT81lTpKZLkyhCvTETwM= github.com/monopole/mdrip v1.0.1/go.mod h1:/7E04hlzRG9Jrp6WILZfYYm/REoJWL2l+MlsCO1eH74= github.com/mozilla/tls-observatory v0.0.0-20190404164649-a3c1b6cfecfd/go.mod h1:SrKMQvPiws7F7iqYp8/TX+IhxCYhzr6N/1yb8cwHsGk= github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= @@ -251,13 +231,10 @@ github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7z github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/quasilyte/go-consistent v0.0.0-20190521200055-c6f3937de18c/go.mod h1:5STLWrekHfjyYwxBRVRXNOSewLJ3PWfDJd1VyTS21fI= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= -github.com/rogpeppe/go-charset v0.0.0-20180617210344-2471d30d28b4/go.mod h1:qgYeAmZ5ZIpBWTGllZSQnw97Dj+woV0toclVaRGI8pc= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik= github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= -github.com/russross/blackfriday v2.0.0+incompatible/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= -github.com/securego/gosec v0.0.0-20190912120752-140048b2a218/go.mod h1:q6oYAujd2qyeU4cJqIri4LBIgdHXGvxWHZ1E29HNFRE= github.com/securego/gosec v0.0.0-20191002120514-e680875ea14d/go.mod h1:w5+eXa0mYznDkHaMCXA4XYffjlH+cy1oyKbfzJXa2Do= github.com/shirou/gopsutil v0.0.0-20190901111213-e4ec7b275ada/go.mod h1:WWnYX4lzhCH5h/3YBfyVA3VbLYjlMZZAQcW9ojMexNc= github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc= @@ -288,13 +265,11 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/timakin/bodyclose v0.0.0-20190721030226-87058b9bfcec/go.mod h1:Qimiffbc6q9tBWlVV6x0P9sat/ao1xEkREYPPj9hphk= github.com/timakin/bodyclose v0.0.0-20190930140734-f7f2e9bca95e/go.mod h1:Qimiffbc6q9tBWlVV6x0P9sat/ao1xEkREYPPj9hphk= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/ultraware/funlen v0.0.2/go.mod h1:Dp4UiAus7Wdb9KUZsYWZEWiRzGuM2kXM1lPbfaF6xhA= -github.com/ultraware/whitespace v0.0.3/go.mod h1:aVMh/gQve5Maj9hQ/hg+F75lr/X5A89uZnzAmWSineA= github.com/ultraware/whitespace v0.0.4/go.mod h1:aVMh/gQve5Maj9hQ/hg+F75lr/X5A89uZnzAmWSineA= github.com/uudashr/gocognit v0.0.0-20190926065955-1655d0de0517/go.mod h1:j44Ayx2KW4+oB6SWMv8KsmHzZrOInQav7D3cQMJ5JUM= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= @@ -315,7 +290,6 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -338,9 +312,8 @@ golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190909003024-a7b16738d86b h1:XfVGCX+0T4WOStkaOsJRllbsiImhB2jgVBGc9L0lPGc= -golang.org/x/net v0.0.0-20190909003024-a7b16738d86b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191004110552-13f9640d40b9 h1:rjwSpXsdiK0dV8/Naq3kAw9ymfAeJIyd0upUIElB+lI= golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -364,8 +337,7 @@ golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190911201528-7ad0cfa0b7b5 h1:SW/0nsKCUaozCUtZTakri5laocGx/5bkDSSLrFUsa5s= -golang.org/x/sys v0.0.0-20190911201528-7ad0cfa0b7b5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190922100055-0a153f010e69 h1:rOhMmluY6kLMhdnrivzec6lLgaVbMHMn2ISQXJeJ5EM= golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -374,7 +346,6 @@ golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -393,8 +364,6 @@ golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190719005602-e377ae9d6386/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI= golang.org/x/tools v0.0.0-20190910044552-dd2b5c81c578/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190911230505-6bfd74cf029c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190912215617-3720d1ec3678/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190930201159-7c411dea38b0/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191010075000-0337d82405ff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -429,8 +398,11 @@ gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +k8s.io/api v0.17.0 h1:H9d/lw+VkZKEVIUc8F3wgiQ+FUXTTr21M87jXLU7yqM= k8s.io/api v0.17.0/go.mod h1:npsyOePkeP0CPwyGfXDHxvypiYMJxBWAMpQxCaJ4ZxI= +k8s.io/apimachinery v0.17.0 h1:xRBnuie9rXcPxUkDizUsGvPf1cnlZCFu210op7J7LJo= k8s.io/apimachinery v0.17.0/go.mod h1:b9qmWdKlLuU9EBh+06BtLcSf/Mu89rWL33naRxs1uZg= +k8s.io/client-go v0.17.0 h1:8QOGvUGdqDMFrm9sD6IUFl256BcffynGoe80sxgTEDg= k8s.io/client-go v0.17.0/go.mod h1:TYgR6EUHs6k45hb6KWjVD6jFZvJV4gHDikv/It0xz+k= k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= @@ -439,17 +411,12 @@ k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8= k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= k8s.io/kube-openapi v0.0.0-20191107075043-30be4d16710a h1:UcxjrRMyNx/i/y8G7kPvLyy7rfbeuf1PYyBf973pgyU= k8s.io/kube-openapi v0.0.0-20191107075043-30be4d16710a/go.mod h1:1TqjTSzOxsLGIKfj0lK8EeCP7K1iUG65v09OM0/WG5E= -k8s.io/utils v0.0.0-20191030222137-2b95a09bc58d/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= k8s.io/utils v0.0.0-20191114184206-e782cd3c129f/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed/go.mod h1:Xkxe497xwlCKkIaQYRfC7CSLworTXY9RMqwhhCm+8Nc= mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b/go.mod h1:2odslEg/xrtNQqCYg2/jCoyKnw3vv5biOc3JnIcYfL4= mvdan.cc/unparam v0.0.0-20190720180237-d51796306d8f/go.mod h1:4G1h5nDURzA3bwVMZIVpwbkw+04kSxk3rAtzlimaUJw= -sigs.k8s.io/kustomize/api v0.2.0 h1:e++6JpysnnlUbHmFrv6jvfF5rFlgQ103bS1DO7r5bWA= -sigs.k8s.io/kustomize/api v0.2.0/go.mod h1:zVtMg179jW1gr74jo9fc2Ac9dLYLTZZThc3DDb9lDW4= -sigs.k8s.io/kustomize/api v0.3.0/go.mod h1:4jaPCtRzxfQLFdYq4gYo40dBGW1hyPp/f4AuiZB5dAQ= -sigs.k8s.io/kustomize/pluginator/v2 v2.0.0/go.mod h1:zrXhTv8BAKt0egmZX/8AtMOSFUSWM9YuoHvvqz8/eHE= -sigs.k8s.io/kustomize/pseudo/k8s v0.1.0 h1:otg4dLFc03c3gzl+2CV8GPGcd1kk8wjXwD+UhhcCn5I= -sigs.k8s.io/kustomize/pseudo/k8s v0.1.0/go.mod h1:bl/gVJgYYhJZCZdYU2BfnaKYAlqFkgbJEkpl302jEss= +sigs.k8s.io/kustomize/api v0.3.0 h1:riR/YsL75nGb+aIPFdIRiqu21+OZbAXQybDS7+FUYRg= +sigs.k8s.io/kustomize/api v0.3.0/go.mod h1:DWNMJBV1xvLruMpihGgnIPznMwHpwUSrxz6v3gnw5kw= sigs.k8s.io/structured-merge-diff v0.0.0-20190525122527-15d366b2352e/go.mod h1:wWxsB5ozmmv/SG7nM11ayaAW51xMvak/t1r0CSlcokI= sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= From 127541f61056f8b2b244c18e8cdf6a9ad2b13efa Mon Sep 17 00:00:00 2001 From: Haiyan Meng Date: Tue, 17 Dec 2019 14:35:44 -0800 Subject: [PATCH 24/27] Support diffrent modes of running the crawler --- api/internal/crawl/cmd/crawler/crawler.go | 32 ++++++++++++++- .../crawl/config/crawler/job/README.md | 41 +++++++++++++++++++ .../crawl/config/crawler/job/job.yaml | 2 +- 3 files changed, 72 insertions(+), 3 deletions(-) create mode 100644 api/internal/crawl/config/crawler/job/README.md diff --git a/api/internal/crawl/cmd/crawler/crawler.go b/api/internal/crawl/cmd/crawler/crawler.go index ea73efd04..7c62522af 100644 --- a/api/internal/crawl/cmd/crawler/crawler.go +++ b/api/internal/crawl/cmd/crawler/crawler.go @@ -3,6 +3,7 @@ package main import ( "context" "fmt" + "log" "net/http" "os" "time" @@ -23,11 +24,31 @@ const ( retryCount = 3 githubUserEnv = "GITHUB_USER" githubRepoEnv = "GITHUB_REPO" + crawlIndexOnlyEnv = "CRAWL_INDEX_ONLY" + crawlGithubOnlyEnv = "CRAWL_GITHUB_ONLY" ) +// countEnvs count the environment variables whose values are not empty. +func countEnvs(envs ...string) int { + count := 0 + for _, env := range envs { + if env != "" { + count++ + } + } + return count +} + func main() { githubUser := os.Getenv(githubUserEnv) githubRepo := os.Getenv(githubRepoEnv) + crawlIndexOnly := os.Getenv(crawlIndexOnlyEnv) + crawlGithubOnly := os.Getenv(crawlGithubOnlyEnv) + + if countEnvs(githubUser, githubRepo, crawlIndexOnly, crawlGithubOnly) > 1 { + log.Fatalf("only one of [%s, %s, %s, %s] should be set", + githubUserEnv, githubRepoEnv, crawlIndexOnlyEnv, crawlGithubOnlyEnv) + } githubToken := os.Getenv(githubAccessTokenVar) if githubToken == "" { @@ -122,6 +143,13 @@ func main() { } crawlers := []crawler.Crawler{ghCrawler} - crawler.CrawlFromSeed(ctx, seedDocs, crawlers, docConverter, indexFunc, seen) - crawler.CrawlGithub(ctx, crawlers, docConverter, indexFunc, seen) + + if crawlGithubOnly == "true" || githubRepo != "" || githubUser != "" { + crawler.CrawlGithub(ctx, crawlers, docConverter, indexFunc, seen) + } else if crawlIndexOnly == "true" { + crawler.CrawlFromSeed(ctx, seedDocs, crawlers, docConverter, indexFunc, seen) + } else { + crawler.CrawlFromSeed(ctx, seedDocs, crawlers, docConverter, indexFunc, seen) + crawler.CrawlGithub(ctx, crawlers, docConverter, indexFunc, seen) + } } diff --git a/api/internal/crawl/config/crawler/job/README.md b/api/internal/crawl/config/crawler/job/README.md new file mode 100644 index 000000000..3fc3e665b --- /dev/null +++ b/api/internal/crawl/config/crawler/job/README.md @@ -0,0 +1,41 @@ +There are three ways of running the crawler job. + +# Crawling all the documents in the index and crawling all the kustomization files on Github + +This is the default setting of the crawler job. + +# Crawling all the documents in the index + +Set the environment variable `CRAWL_INDEX_ONLY` to `true` like this: + +``` + - name: CRAWL_INDEX_ONLY + value: true +``` + +# Crawling all the kustomization files on Github + +Set the environment variable `CRAWL_GITHUB_ONLY` to `true` like this: + +``` + - name: CRAWL_GITHUB_ONLY + value: true +``` + +# Crawling all the kustomization files in a Github repo + +Add the environment variable `GITHUB_REPO` into the crawler container. For example: + +``` + - name: GITHUB_REPO + value: kubernetes-sigs/kustomize +``` + +# Crawling all the kustomization files in all the repositories of a Github user + +Add the environment variable `GITHUB_USER` into the crawler container. For example: + +``` + - name: GITHUB_USER + value: kubernetes-sigs +``` diff --git a/api/internal/crawl/config/crawler/job/job.yaml b/api/internal/crawl/config/crawler/job/job.yaml index dde0de398..6dd8d4c97 100644 --- a/api/internal/crawl/config/crawler/job/job.yaml +++ b/api/internal/crawl/config/crawler/job/job.yaml @@ -8,7 +8,7 @@ spec: restartPolicy: OnFailure containers: - name: crawler - image: gcr.io/kustomize-search/crawler:latest + image: gcr.io/haiyanmeng-gke-dev/crawler:v1 imagePullPolicy: Always env: - name: GITHUB_ACCESS_TOKEN From be2e03681d5151c39cb1764290cfd7a98ae69f9d Mon Sep 17 00:00:00 2001 From: Haiyan Meng Date: Tue, 17 Dec 2019 14:56:52 -0800 Subject: [PATCH 25/27] Remove unused param from IndexFunc --- api/internal/crawl/cmd/crawler/crawler.go | 157 ++++++++++++------ .../crawl/config/crawler/job/README.md | 39 +++-- .../crawl/config/crawler/job/job.yaml | 2 + api/internal/crawl/crawler/crawler.go | 6 +- api/internal/crawl/crawler/crawler_test.go | 2 +- 5 files changed, 135 insertions(+), 71 deletions(-) diff --git a/api/internal/crawl/cmd/crawler/crawler.go b/api/internal/crawl/cmd/crawler/crawler.go index 7c62522af..fdbafeafd 100644 --- a/api/internal/crawl/cmd/crawler/crawler.go +++ b/api/internal/crawl/cmd/crawler/crawler.go @@ -22,34 +22,53 @@ const ( redisCacheURL = "REDIS_CACHE_URL" redisKeyURL = "REDIS_KEY_URL" retryCount = 3 - githubUserEnv = "GITHUB_USER" - githubRepoEnv = "GITHUB_REPO" - crawlIndexOnlyEnv = "CRAWL_INDEX_ONLY" - crawlGithubOnlyEnv = "CRAWL_GITHUB_ONLY" ) -// countEnvs count the environment variables whose values are not empty. -func countEnvs(envs ...string) int { - count := 0 - for _, env := range envs { - if env != "" { - count++ - } +type CrawlMode int +const ( + CrawlUnknown CrawlMode = iota + // Crawl all the kustomization files in all the repositories of a Github user + CrawlUser + // Crawl all the kustomization files in a Github repo + CrawlRepo + // Crawl all the documents in the index + CrawlIndex + // Crawl all the kustomization files on Github + CrawlGithub + // Crawl all the documents in the index and crawling all the kustomization files on Github + CrawlIndexAndGithub +) + +func NewCrawlMode(s string) CrawlMode { + switch s { + case "github-user": + return CrawlUser + case "github-repo": + return CrawlRepo + case "": + return CrawlIndexAndGithub + case "index": + return CrawlIndex + case "github": + return CrawlGithub + default: + return CrawlUnknown } - return count +} + +func Usage() { + fmt.Printf("Usage: %s [mode] [githubUser|githubRepo]\n", os.Args[0]) + fmt.Printf("\tmode can be one of [github-user, github-repo, index, github]\n") + fmt.Printf("%s: crawl all the documents in the index and crawling all the kustomization files on Github\n", os.Args[0]) + fmt.Printf("%s index: crawl all the documents in the index\n", os.Args[0]) + fmt.Printf("%s gihub: crawl all the kustomization files on Github\n", os.Args[0]) + fmt.Printf("%s github-user : Crawl all the kustomization files in all the repositories of a Github user\n", os.Args[0]) + fmt.Printf("\tFor example, %s github-user kubernetes-sigs\n", os.Args[0]) + fmt.Printf("%s github-repo : Crawl all the kustomization files in a Github repo\n", os.Args[0]) + fmt.Printf("\tFor example, %s github-repo kubernetes-sigs/kustomize\n", os.Args[0]) } func main() { - githubUser := os.Getenv(githubUserEnv) - githubRepo := os.Getenv(githubRepoEnv) - crawlIndexOnly := os.Getenv(crawlIndexOnlyEnv) - crawlGithubOnly := os.Getenv(crawlGithubOnlyEnv) - - if countEnvs(githubUser, githubRepo, crawlIndexOnly, crawlGithubOnly) > 1 { - log.Fatalf("only one of [%s, %s, %s, %s] should be set", - githubUserEnv, githubRepoEnv, crawlIndexOnlyEnv, crawlGithubOnlyEnv) - } - githubToken := os.Getenv(githubAccessTokenVar) if githubToken == "" { fmt.Printf("Must set the variable '%s' to make github requests.\n", @@ -64,8 +83,6 @@ func main() { return } - seedDocs := make(crawler.CrawlSeed, 0) - cacheURL := os.Getenv(redisCacheURL) cache, err := redis.DialURL(cacheURL) clientCache := &http.Client{} @@ -86,7 +103,7 @@ func main() { } // Index updates the value in the index. - indexFunc := func(cdoc crawler.CrawledDocument, crwlr crawler.Crawler, mode index.Mode) error { + indexFunc := func(cdoc crawler.CrawledDocument, mode index.Mode) error { switch d := cdoc.(type) { case *doc.KustomizationDocument: switch mode { @@ -106,30 +123,41 @@ func main() { // This helps avoid indexing a given document multiple times. seen := make(map[string]struct{}) - var ghCrawler crawler.Crawler - - if githubRepo != "" { - ghCrawler = github.NewCrawler(githubToken, retryCount, clientCache, - github.QueryWith( - github.Filename("kustomization.yaml"), - github.Filename("kustomization.yml"), - github.Repo(githubRepo)), - ) - } else if githubUser != "" { - ghCrawler = github.NewCrawler(githubToken, retryCount, clientCache, - github.QueryWith( - github.Filename("kustomization.yaml"), - github.Filename("kustomization.yml"), - github.User(githubUser)), - ) + var mode CrawlMode + if len(os.Args) == 1 { + mode = CrawlIndexAndGithub } else { - ghCrawler = github.NewCrawler(githubToken, retryCount, clientCache, - github.QueryWith( - github.Filename("kustomization.yaml"), - github.Filename("kustomization.yml")), - ) + mode = NewCrawlMode(os.Args[1]) + } - // get all the documents in the index + ghCrawlerConstructor := func(user, repo string) crawler.Crawler { + if user != "" { + return github.NewCrawler(githubToken, retryCount, clientCache, + github.QueryWith( + github.Filename("kustomization.yaml"), + github.Filename("kustomization.yml"), + github.User(user)), + ) + } else if repo != "" { + return github.NewCrawler(githubToken, retryCount, clientCache, + github.QueryWith( + github.Filename("kustomization.yaml"), + github.Filename("kustomization.yml"), + github.Repo(repo)), + ) + } else { + return github.NewCrawler(githubToken, retryCount, clientCache, + github.QueryWith( + github.Filename("kustomization.yaml"), + github.Filename("kustomization.yml")), + ) + } + } + + seedDocs := make(crawler.CrawlSeed, 0) + + // get all the documents in the index + getSeedDocsFunc := func() { query := []byte(`{ "query":{ "match_all":{} } }`) it := idx.IterateQuery(query, 10000, 60*time.Second) for it.Next() { @@ -142,14 +170,35 @@ func main() { } } - crawlers := []crawler.Crawler{ghCrawler} - - if crawlGithubOnly == "true" || githubRepo != "" || githubUser != "" { - crawler.CrawlGithub(ctx, crawlers, docConverter, indexFunc, seen) - } else if crawlIndexOnly == "true" { - crawler.CrawlFromSeed(ctx, seedDocs, crawlers, docConverter, indexFunc, seen) - } else { + switch mode { + case CrawlIndexAndGithub: + getSeedDocsFunc() + crawlers := []crawler.Crawler{ghCrawlerConstructor("", "")} crawler.CrawlFromSeed(ctx, seedDocs, crawlers, docConverter, indexFunc, seen) crawler.CrawlGithub(ctx, crawlers, docConverter, indexFunc, seen) + case CrawlIndex: + getSeedDocsFunc() + crawlers := []crawler.Crawler{ghCrawlerConstructor("", "")} + crawler.CrawlFromSeed(ctx, seedDocs, crawlers, docConverter, indexFunc, seen) + case CrawlGithub: + crawlers := []crawler.Crawler{ghCrawlerConstructor("", "")} + crawler.CrawlGithub(ctx, crawlers, docConverter, indexFunc, seen) + case CrawlUser: + if len(os.Args) < 3 { + Usage() + log.Fatalf("Please specify a github user!") + } + crawlers := []crawler.Crawler{ghCrawlerConstructor(os.Args[2], "")} + crawler.CrawlGithub(ctx, crawlers, docConverter, indexFunc, seen) + case CrawlRepo: + if len(os.Args) < 3 { + Usage() + log.Fatalf("Please specify a github repo!") + } + crawlers := []crawler.Crawler{ghCrawlerConstructor("", os.Args[2])} + crawler.CrawlGithub(ctx, crawlers, docConverter, indexFunc, seen) + case CrawlUnknown: + Usage() + log.Fatalf("The crawler mode must be one of [github-user, github-repo, index, github]") } } diff --git a/api/internal/crawl/config/crawler/job/README.md b/api/internal/crawl/config/crawler/job/README.md index 3fc3e665b..3570f27ee 100644 --- a/api/internal/crawl/config/crawler/job/README.md +++ b/api/internal/crawl/config/crawler/job/README.md @@ -2,40 +2,53 @@ There are three ways of running the crawler job. # Crawling all the documents in the index and crawling all the kustomization files on Github -This is the default setting of the crawler job. +This is the default setting of the crawler job. The `command` and `args` field +of the container should be: + +``` + command: ["/crawler"] + args: [] +``` + +Or + +``` + command: ["/crawler"] + args: [""] +``` # Crawling all the documents in the index -Set the environment variable `CRAWL_INDEX_ONLY` to `true` like this: +The `command` and `args` field of the container should be: ``` - - name: CRAWL_INDEX_ONLY - value: true + command: ["/crawler"] + args: ["index"] ``` # Crawling all the kustomization files on Github -Set the environment variable `CRAWL_GITHUB_ONLY` to `true` like this: +The `command` and `args` field of the container should be: ``` - - name: CRAWL_GITHUB_ONLY - value: true + command: ["/crawler"] + args: ["github"] ``` # Crawling all the kustomization files in a Github repo -Add the environment variable `GITHUB_REPO` into the crawler container. For example: +The `command` and `args` field of the container should be like: ``` - - name: GITHUB_REPO - value: kubernetes-sigs/kustomize + command: ["/crawler"] + args: ["github-repo", "kubernetes-sigs/kustomize"] ``` # Crawling all the kustomization files in all the repositories of a Github user -Add the environment variable `GITHUB_USER` into the crawler container. For example: +The `command` and `args` field of the container should be like: ``` - - name: GITHUB_USER - value: kubernetes-sigs + command: ["/crawler"] + args: ["github-user", "kubernetes-sigs"] ``` diff --git a/api/internal/crawl/config/crawler/job/job.yaml b/api/internal/crawl/config/crawler/job/job.yaml index 6dd8d4c97..28e36bcb8 100644 --- a/api/internal/crawl/config/crawler/job/job.yaml +++ b/api/internal/crawl/config/crawler/job/job.yaml @@ -10,6 +10,8 @@ spec: - name: crawler image: gcr.io/haiyanmeng-gke-dev/crawler:v1 imagePullPolicy: Always + command: ["/crawler"] + args: ["github-repo", "kubernetes-sigs/kustomize"] env: - name: GITHUB_ACCESS_TOKEN valueFrom: diff --git a/api/internal/crawl/crawler/crawler.go b/api/internal/crawl/crawler/crawler.go index d46cf161d..31cabc2b7 100644 --- a/api/internal/crawl/crawler/crawler.go +++ b/api/internal/crawl/crawler/crawler.go @@ -49,7 +49,7 @@ type CrawledDocument interface { type CrawlSeed []*doc.Document -type IndexFunc func(CrawledDocument, Crawler, index.Mode) error +type IndexFunc func(CrawledDocument, index.Mode) error type Converter func(*doc.Document) (CrawledDocument, error) func logIfErr(err error) { @@ -74,7 +74,7 @@ func addBranches(cdoc CrawledDocument, match Crawler, indx IndexFunc, seen[cdoc.ID()] = struct{}{} // Insert into index - if err := indx(cdoc, match, index.InsertOrUpdate); err != nil { + if err := indx(cdoc, index.InsertOrUpdate); err != nil { logger.Printf("Failed to insert or update %s %s: %v", cdoc.GetDocument().RepositoryURL, cdoc.GetDocument().FilePath, err) return @@ -142,7 +142,7 @@ func doCrawl(ctx context.Context, docsPtr *CrawlSeed, crawlers []Crawler, conv C Document: *tail, } seen[cdoc.ID()] = struct{}{} - if err := indx(cdoc, match, index.Delete); err != nil { + if err := indx(cdoc, index.Delete); err != nil { logger.Printf("Failed to delete %s %s: %v", cdoc.RepositoryURL, cdoc.FilePath, err) } diff --git a/api/internal/crawl/crawler/crawler_test.go b/api/internal/crawl/crawler/crawler_test.go index 00a619c46..41a848612 100644 --- a/api/internal/crawl/crawler/crawler_test.go +++ b/api/internal/crawl/crawler/crawler_test.go @@ -318,7 +318,7 @@ resources: Document: *d, }, nil }, - func(d CrawledDocument, cr Crawler, mode index.Mode) error { + func(d CrawledDocument, mode index.Mode) error { visited[d.ID()]++ return nil }, From 49f17586ca03a5527602f74fa60856dfa7a419b8 Mon Sep 17 00:00:00 2001 From: Sean Sullivan Date: Thu, 19 Dec 2019 11:20:12 -0800 Subject: [PATCH 26/27] Update go.mod to point to most recent version of k8s.io/kubectl --- cmd/kubectl/go.mod | 3 ++- cmd/kubectl/go.sum | 15 +++++++++++---- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/cmd/kubectl/go.mod b/cmd/kubectl/go.mod index 3b053c3cf..259bd5a59 100644 --- a/cmd/kubectl/go.mod +++ b/cmd/kubectl/go.mod @@ -6,7 +6,8 @@ require ( github.com/spf13/cobra v0.0.5 k8s.io/cli-runtime v0.17.0 k8s.io/client-go v0.17.0 - k8s.io/kubectl v0.17.0 + k8s.io/component-base v0.17.0 // indirect + k8s.io/kubectl v0.0.0-20191219154910-1528d4eea6dd sigs.k8s.io/kustomize/kyaml v0.0.0 ) diff --git a/cmd/kubectl/go.sum b/cmd/kubectl/go.sum index b00f96ff9..dac361f72 100644 --- a/cmd/kubectl/go.sum +++ b/cmd/kubectl/go.sum @@ -344,15 +344,22 @@ gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +k8s.io/api v0.0.0-20191214185829-ca1d04f8b0d3/go.mod h1:itOjKREfmUTvcjantxOsyYU5mbFsU7qUnyUuRfF5+5M= k8s.io/api v0.17.0 h1:H9d/lw+VkZKEVIUc8F3wgiQ+FUXTTr21M87jXLU7yqM= k8s.io/api v0.17.0/go.mod h1:npsyOePkeP0CPwyGfXDHxvypiYMJxBWAMpQxCaJ4ZxI= +k8s.io/apimachinery v0.0.0-20191214185652-442f8fb2f03a/go.mod h1:Ng1IY8TS7sC44KJxT/WUR6qFRfWwahYYYpNXyYRKOCY= +k8s.io/apimachinery v0.0.0-20191216025728-0ee8b4573e3a/go.mod h1:Ng1IY8TS7sC44KJxT/WUR6qFRfWwahYYYpNXyYRKOCY= k8s.io/apimachinery v0.17.0 h1:xRBnuie9rXcPxUkDizUsGvPf1cnlZCFu210op7J7LJo= k8s.io/apimachinery v0.17.0/go.mod h1:b9qmWdKlLuU9EBh+06BtLcSf/Mu89rWL33naRxs1uZg= +k8s.io/cli-runtime v0.0.0-20191214191754-e6dc6d5c8724/go.mod h1:wzlq80lvjgHW9if6MlE4OIGC86MDKsy5jtl9nxz/IYY= k8s.io/cli-runtime v0.17.0 h1:XEuStbJBHCQlEKFyTQmceDKEWOSYHZkcYWKp3SsQ9Hk= k8s.io/cli-runtime v0.17.0/go.mod h1:1E5iQpMODZq2lMWLUJELwRu2MLWIzwvMgDBpn3Y81Qo= +k8s.io/client-go v0.0.0-20191214190045-a32a6f7a3052/go.mod h1:tAaoc/sYuIL0+njJefSAmE28CIcxyaFV4kbIujBlY2s= +k8s.io/client-go v0.0.0-20191219150334-0b8da7416048/go.mod h1:ZEe8ZASDUAuqVGJ+UN0ka0PfaR+b6a6E1PGsSNZRui8= k8s.io/client-go v0.17.0 h1:8QOGvUGdqDMFrm9sD6IUFl256BcffynGoe80sxgTEDg= k8s.io/client-go v0.17.0/go.mod h1:TYgR6EUHs6k45hb6KWjVD6jFZvJV4gHDikv/It0xz+k= -k8s.io/code-generator v0.17.0/go.mod h1:DVmfPQgxQENqDIzVR2ddLXMH34qeszkKSdH/N+s+38s= +k8s.io/code-generator v0.0.0-20191214185510-0b9b3c99f9f2/go.mod h1:BjGKcoq1MRUmcssvHiSxodCco1T6nVIt4YeCT5CMSao= +k8s.io/component-base v0.0.0-20191214190519-d868452632e2/go.mod h1:wupxkh1T/oUDqyTtcIjiEfpbmIHGm8By/vqpSKC6z8c= k8s.io/component-base v0.17.0 h1:BnDFcmBDq+RPpxXjmuYnZXb59XNN9CaFrX8ba9+3xrA= k8s.io/component-base v0.17.0/go.mod h1:rKuRAokNMY2nn2A6LP/MiwpoaMRHpfRnrPaUJJj1Yoc= k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= @@ -363,9 +370,9 @@ k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8= k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= k8s.io/kube-openapi v0.0.0-20191107075043-30be4d16710a h1:UcxjrRMyNx/i/y8G7kPvLyy7rfbeuf1PYyBf973pgyU= k8s.io/kube-openapi v0.0.0-20191107075043-30be4d16710a/go.mod h1:1TqjTSzOxsLGIKfj0lK8EeCP7K1iUG65v09OM0/WG5E= -k8s.io/kubectl v0.17.0 h1:xD4EWlL+epc/JTO1gvSjmV9yiYF0Z2wiHK2DIek6URY= -k8s.io/kubectl v0.17.0/go.mod h1:jIPrUAW656Vzn9wZCCe0PC+oTcu56u2HgFD21Xbfk1s= -k8s.io/metrics v0.17.0/go.mod h1:EH1D3YAwN6d7bMelrElnLhLg72l/ERStyv2SIQVt6Do= +k8s.io/kubectl v0.0.0-20191219154910-1528d4eea6dd h1:nZX5+wEqTu/EBIYjrZlFOA63z4+Zcy96lDkCZPU9a9c= +k8s.io/kubectl v0.0.0-20191219154910-1528d4eea6dd/go.mod h1:9ehGcuUGjXVZh0qbYSB0vvofQw2JQe6c6cO0k4wu/Oo= +k8s.io/metrics v0.0.0-20191214191643-6b1944c9f765/go.mod h1:5V7rewilItwK0cz4nomU0b3XCcees2Ka5EBYWS1HBeM= k8s.io/utils v0.0.0-20191114184206-e782cd3c129f h1:GiPwtSzdP43eI1hpPCbROQCCIgCuiMMNF8YUVLF3vJo= k8s.io/utils v0.0.0-20191114184206-e782cd3c129f/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= modernc.org/cc v1.0.0/go.mod h1:1Sk4//wdnYJiUIxnW8ddKpaOJCF37yAdqYnkxUpaYxw= From 98431f6a00bb8bb191cd1c03bcd798c48d5e86e4 Mon Sep 17 00:00:00 2001 From: Phillip Wittrock Date: Thu, 19 Dec 2019 15:05:49 -0800 Subject: [PATCH 27/27] fix kyaml issue where dropping Style created issues dropping the node style creates a compatibility issue where quotes around "on" are dropped because yaml.v3 interprets it as a string. other yaml parsers interpret on as a bool value, and parse it as a bool rather than string. fix: retain the original style so it is kept as quoted. - fmt: don't drop the styles - merge2: keep the style when merging elements - setting a field: if changing the value of a scalar field, retain its style by default --- cmd/config/internal/commands/cat_test.go | 20 +++++------ cmd/config/internal/commands/cmdwrap_test.go | 24 +++++++------- kyaml/kio/filters/fmtr.go | 1 - kyaml/kio/filters/fmtr_test.go | 33 ++++++++++++++++-- kyaml/kio/filters/testyaml/testyaml.go | 4 +++ kyaml/kio/pkgio_writer_test.go | 6 ++-- kyaml/yaml/fns.go | 20 +++++++++-- kyaml/yaml/fns_test.go | 19 +++++++++++ kyaml/yaml/merge2/element_test.go | 35 +++++++------------- kyaml/yaml/merge2/merge2.go | 28 ++++++++++++++++ kyaml/yaml/merge2/merge2_old_test.go | 25 +++----------- kyaml/yaml/merge3/element_test.go | 12 +++---- kyaml/yaml/types.go | 5 +-- 13 files changed, 147 insertions(+), 85 deletions(-) diff --git a/cmd/config/internal/commands/cat_test.go b/cmd/config/internal/commands/cat_test.go index c02b8f200..d9441713d 100644 --- a/cmd/config/internal/commands/cat_test.go +++ b/cmd/config/internal/commands/cat_test.go @@ -114,8 +114,8 @@ metadata: app: nginx annotations: app: nginx - config.kubernetes.io/package: . - config.kubernetes.io/path: f2.yaml + config.kubernetes.io/package: '.' + config.kubernetes.io/path: 'f2.yaml' spec: replicas: 3 `, b.String()) { @@ -218,8 +218,8 @@ metadata: name: foo annotations: config.kubernetes.io/local-config: "true" - config.kubernetes.io/package: . - config.kubernetes.io/path: f2.yaml + config.kubernetes.io/package: '.' + config.kubernetes.io/path: 'f2.yaml' configFn: container: image: gcr.io/example/image:version @@ -314,8 +314,8 @@ metadata: name: foo annotations: config.kubernetes.io/local-config: "true" - config.kubernetes.io/package: . - config.kubernetes.io/path: f2.yaml + config.kubernetes.io/package: '.' + config.kubernetes.io/path: 'f2.yaml' configFn: container: image: gcr.io/example/reconciler:v1 @@ -438,8 +438,8 @@ metadata: app: nginx annotations: app: nginx - config.kubernetes.io/package: . - config.kubernetes.io/path: f2.yaml + config.kubernetes.io/package: '.' + config.kubernetes.io/path: 'f2.yaml' spec: replicas: 3 `, string(actual)) { @@ -560,8 +560,8 @@ metadata: app: nginx annotations: app: nginx - config.kubernetes.io/package: . - config.kubernetes.io/path: f2.yaml + config.kubernetes.io/package: '.' + config.kubernetes.io/path: 'f2.yaml' spec: replicas: 3 `, string(actual)) { diff --git a/cmd/config/internal/commands/cmdwrap_test.go b/cmd/config/internal/commands/cmdwrap_test.go index affa88bf1..a55a951e5 100644 --- a/cmd/config/internal/commands/cmdwrap_test.go +++ b/cmd/config/internal/commands/cmdwrap_test.go @@ -78,8 +78,8 @@ items: name: test app: nginx annotations: - config.kubernetes.io/index: "0" - config.kubernetes.io/path: config/test_deployment.yaml + config.kubernetes.io/index: '0' + config.kubernetes.io/path: 'config/test_deployment.yaml' spec: replicas: 11 selector: @@ -109,8 +109,8 @@ items: name: test app: nginx annotations: - config.kubernetes.io/index: "0" - config.kubernetes.io/path: config/test_service.yaml + config.kubernetes.io/index: '0' + config.kubernetes.io/path: 'config/test_service.yaml' spec: selector: name: test @@ -133,8 +133,8 @@ items: name: test app: nginx annotations: - config.kubernetes.io/index: "0" - config.kubernetes.io/path: config/test_deployment.yaml + config.kubernetes.io/index: '0' + config.kubernetes.io/path: 'config/test_deployment.yaml' spec: replicas: 11 selector: @@ -161,8 +161,8 @@ items: name: test app: nginx annotations: - config.kubernetes.io/index: "0" - config.kubernetes.io/path: config/test_service.yaml + config.kubernetes.io/index: '0' + config.kubernetes.io/path: 'config/test_service.yaml' spec: selector: name: test @@ -185,8 +185,8 @@ items: name: test app: nginx annotations: - config.kubernetes.io/index: "0" - config.kubernetes.io/path: config/test_deployment.yaml + config.kubernetes.io/index: '0' + config.kubernetes.io/path: 'config/test_deployment.yaml' spec: replicas: 11 selector: @@ -216,8 +216,8 @@ items: name: test app: nginx annotations: - config.kubernetes.io/index: "0" - config.kubernetes.io/path: config/test_service.yaml + config.kubernetes.io/index: '0' + config.kubernetes.io/path: 'config/test_service.yaml' spec: selector: name: test diff --git a/kyaml/kio/filters/fmtr.go b/kyaml/kio/filters/fmtr.go index 201c74f0d..04796ed9b 100644 --- a/kyaml/kio/filters/fmtr.go +++ b/kyaml/kio/filters/fmtr.go @@ -90,7 +90,6 @@ type formatter struct { // fmtNode recursively formats the Document Contents. func (f *formatter) fmtNode(n *yaml.Node, path string) error { - n.Style = 0 // sort the order of mapping fields if n.Kind == yaml.MappingNode { sort.Sort(sortedMapContents(*n)) diff --git a/kyaml/kio/filters/fmtr_test.go b/kyaml/kio/filters/fmtr_test.go index a60aa23a4..e40c403ea 100644 --- a/kyaml/kio/filters/fmtr_test.go +++ b/kyaml/kio/filters/fmtr_test.go @@ -17,6 +17,35 @@ import ( "sigs.k8s.io/kustomize/kyaml/kio/filters/testyaml" ) +func TestFormatInput_Style(t *testing.T) { + y := ` +apiVersion: v1 +kind: Foo +metadata: + name: foo +spec: + notBoolean: "true" + notBoolean2: "on" + isBoolean: on + isBoolean2: true +` + + expected := `apiVersion: v1 +kind: Foo +metadata: + name: foo +spec: + isBoolean: on + isBoolean2: true + notBoolean: "true" + notBoolean2: "on" +` + + s, err := FormatInput(strings.NewReader(y)) + assert.NoError(t, err) + assert.Equal(t, expected, s.String()) +} + // TestFormatInput_configMap verifies a ConfigMap yaml is formatted correctly func TestFormatInput_configMap(t *testing.T) { y := ` @@ -229,7 +258,7 @@ webhooks: - CONNECT - CREATE - UPDATE # this list is indented by 2 - scope: Namespaced + scope: "Namespaced" timeoutSeconds: 1 ` s, err := FormatInput(strings.NewReader(y)) @@ -467,7 +496,7 @@ func TestFormatFileOrDirectory_YamlExtFileWithJson(t *testing.T) { // check the result is formatted as yaml b, err := ioutil.ReadFile(f.Name()) assert.NoError(t, err) - assert.Equal(t, string(testyaml.FormattedYaml1), string(b)) + assert.Equal(t, string(testyaml.FormattedJSON1), string(b)) } // TestFormatFileOrDirectory_partialKubernetesYamlFile verifies that if a yaml file contains both diff --git a/kyaml/kio/filters/testyaml/testyaml.go b/kyaml/kio/filters/testyaml/testyaml.go index 67593d026..555d95da0 100644 --- a/kyaml/kio/filters/testyaml/testyaml.go +++ b/kyaml/kio/filters/testyaml/testyaml.go @@ -55,3 +55,7 @@ status2: - 1 - 2 `) + +var FormattedJSON1 = []byte(`{"apiVersion": "example.com/v1beta1", "kind": "MyType", "spec": "a", "status": {"conditions": [ + 3, 1, 2]}} +`) diff --git a/kyaml/kio/pkgio_writer_test.go b/kyaml/kio/pkgio_writer_test.go index b6bf6ce30..61562ab8a 100644 --- a/kyaml/kio/pkgio_writer_test.go +++ b/kyaml/kio/pkgio_writer_test.go @@ -68,13 +68,13 @@ func TestLocalPackageWriter_Write_keepReaderAnnotations(t *testing.T) { metadata: annotations: config.kubernetes.io/index: 0 - config.kubernetes.io/path: a/b/a_test.yaml + config.kubernetes.io/path: "a/b/a_test.yaml" --- c: d # second metadata: annotations: config.kubernetes.io/index: 1 - config.kubernetes.io/path: a/b/a_test.yaml + config.kubernetes.io/path: "a/b/a_test.yaml" `, string(b)) b, err = ioutil.ReadFile(filepath.Join(d, "a", "b", "b_test.yaml")) @@ -89,7 +89,7 @@ g: metadata: annotations: config.kubernetes.io/index: 0 - config.kubernetes.io/path: a/b/b_test.yaml + config.kubernetes.io/path: "a/b/b_test.yaml" `, string(b)) } diff --git a/kyaml/yaml/fns.go b/kyaml/yaml/fns.go index 74b2c027d..9de3e26eb 100644 --- a/kyaml/yaml/fns.go +++ b/kyaml/yaml/fns.go @@ -446,14 +446,18 @@ type FieldSetter struct { // value on the ScalarNode. Name string `yaml:"name,omitempty"` - // YNode is the value to set. + // Value is the value to set. // Optional if Kind is set. Value *RNode `yaml:"value,omitempty"` StringValue string `yaml:"stringValue,omitempty"` + + // OverrideStyle can be set to override the style of the existing node + // when setting it. Otherwise, if an existing node is found, the style is + // retained. + OverrideStyle bool `yaml:"overrideStyle,omitempty"` } -// FieldSetter returns an GrepFilter that sets the named field to the given value. func (s FieldSetter) Filter(rn *RNode) (*RNode, error) { if s.StringValue != "" && s.Value == nil { s.Value = NewScalarRNode(s.StringValue) @@ -463,6 +467,12 @@ func (s FieldSetter) Filter(rn *RNode) (*RNode, error) { if err := ErrorIfInvalid(rn, yaml.ScalarNode); err != nil { return rn, err } + // only apply the style if there is not an existing style + // or we want to override it + if !s.OverrideStyle || s.Value.YNode().Style == 0 { + // keep the original style if it exists + s.Value.YNode().Style = rn.YNode().Style + } rn.SetYNode(s.Value.YNode()) return rn, nil } @@ -477,6 +487,12 @@ func (s FieldSetter) Filter(rn *RNode) (*RNode, error) { return nil, err } if field != nil { + // only apply the style if there is not an existing style + // or we want to override it + if !s.OverrideStyle || field.YNode().Style == 0 { + // keep the original style if it exists + s.Value.YNode().Style = field.YNode().Style + } // need to def ref the Node since field is ephemeral field.SetYNode(s.Value.YNode()) return field, nil diff --git a/kyaml/yaml/fns_test.go b/kyaml/yaml/fns_test.go index 1bb05ce4d..e9590aa3f 100644 --- a/kyaml/yaml/fns_test.go +++ b/kyaml/yaml/fns_test.go @@ -646,6 +646,25 @@ metadata: `, assertNoErrorString(t)(r0.String())) } +func TestUpdateAnnotation_Fn(t *testing.T) { + // create metadata.annotations field + r0 := assertNoError(t)(Parse(`apiVersion: apps/v1 +kind: Deployment +metadata: + annotations: + a.b.c: "h.i.j" +`)) + + rn := assertNoError(t)(r0.Pipe(SetAnnotation("a.b.c", "d.e.f"))) + assert.Equal(t, "\"d.e.f\"\n", assertNoErrorString(t)(rn.String())) + assert.Equal(t, `apiVersion: apps/v1 +kind: Deployment +metadata: + annotations: + a.b.c: "d.e.f" +`, assertNoErrorString(t)(r0.String())) +} + func TestRNode_GetMeta(t *testing.T) { s := `apiVersion: v1/apps kind: Deployment diff --git a/kyaml/yaml/merge2/element_test.go b/kyaml/yaml/merge2/element_test.go index bdb1100d2..2a9cd5d6e 100644 --- a/kyaml/yaml/merge2/element_test.go +++ b/kyaml/yaml/merge2/element_test.go @@ -23,8 +23,7 @@ kind: Deployment items: - name: foo image: foo:v1 - command: - - run.sh + command: ['run.sh'] `, }, @@ -47,8 +46,7 @@ kind: Deployment items: - name: foo image: foo:v1 - command: - - run.sh + command: ['run.sh'] `, }, @@ -62,15 +60,14 @@ items: `, ` kind: Deployment -items: {} +items: [] `, ` kind: Deployment items: - name: foo image: foo:v1 - command: - - run.sh + command: ['run.sh'] `, }, @@ -90,8 +87,7 @@ kind: Deployment items: - name: foo image: foo:v1 - command: - - run.sh + command: ['run.sh'] `, }, @@ -118,8 +114,7 @@ items: image: foo:v1 - name: bar image: bar:v1 - command: - - run2.sh + command: ['run2.sh'] `, }, @@ -146,8 +141,7 @@ items: image: foo:v1 - name: bar image: bar:v1 - command: - - run2.sh + command: ['run2.sh'] `, }, @@ -174,8 +168,7 @@ items: image: foo:v1 - name: bar image: bar:v1 - command: - - run2.sh + command: ['run2.sh'] `, }, @@ -205,8 +198,7 @@ items: image: foo:v1 - name: bar image: bar:v1 - command: - - run2.sh + command: ['run2.sh'] `, }, @@ -225,8 +217,7 @@ items: image: foo:v1 - name: bar image: bar:v1 - command: - - run2.sh + command: ['run2.sh'] `, ` kind: Deployment @@ -235,8 +226,7 @@ items: image: foo:v1 - name: bar image: bar:v1 - command: - - run2.sh + command: ['run2.sh'] `, }, @@ -255,8 +245,7 @@ items: image: foo:v1 - name: bar image: bar:v1 - command: - - run2.sh + command: ['run2.sh'] `, ` kind: Deployment diff --git a/kyaml/yaml/merge2/merge2.go b/kyaml/yaml/merge2/merge2.go index f4620f748..86f3b5ece 100644 --- a/kyaml/yaml/merge2/merge2.go +++ b/kyaml/yaml/merge2/merge2.go @@ -43,6 +43,9 @@ func (m Merger) VisitMap(nodes walk.Sources) (*yaml.RNode, error) { if err := m.SetComments(nodes); err != nil { return nil, err } + if err := m.SetStyle(nodes); err != nil { + return nil, err + } if yaml.IsEmpty(nodes.Dest()) { // Add return nodes.Origin(), nil @@ -59,6 +62,9 @@ func (m Merger) VisitScalar(nodes walk.Sources) (*yaml.RNode, error) { if err := m.SetComments(nodes); err != nil { return nil, err } + if err := m.SetStyle(nodes); err != nil { + return nil, err + } // Override value if nodes.Origin() != nil { return nodes.Origin(), nil @@ -71,6 +77,9 @@ func (m Merger) VisitList(nodes walk.Sources, kind walk.ListKind) (*yaml.RNode, if err := m.SetComments(nodes); err != nil { return nil, err } + if err := m.SetStyle(nodes); err != nil { + return nil, err + } if kind == walk.NonAssociateList { // Override value if nodes.Origin() != nil { @@ -92,6 +101,25 @@ func (m Merger) VisitList(nodes walk.Sources, kind walk.ListKind) (*yaml.RNode, return nodes.Dest(), nil } +func (m Merger) SetStyle(sources walk.Sources) error { + source := sources.Origin() + dest := sources.Dest() + if dest == nil || dest.YNode() == nil || source == nil || source.YNode() == nil { + // avoid panic + return nil + } + + // copy the style from the source. + // special case: if the dest was an empty map or seq, then it probably had + // folded style applied, but we actually want to keep the style of the origin + // in this case (even if it was the default). otherwise the merged elements + // will get folded even though this probably isn't what is desired. + if dest.YNode().Kind != yaml.ScalarNode && len(dest.YNode().Content) == 0 { + dest.YNode().Style = source.YNode().Style + } + return nil +} + // SetComments copies the dest comments to the source comments if they are present // on the source. func (m Merger) SetComments(sources walk.Sources) error { diff --git a/kyaml/yaml/merge2/merge2_old_test.go b/kyaml/yaml/merge2/merge2_old_test.go index 95fce2b41..e4855b72c 100644 --- a/kyaml/yaml/merge2/merge2_old_test.go +++ b/kyaml/yaml/merge2/merge2_old_test.go @@ -83,10 +83,7 @@ spec: containers: - name: nginx image: nginx:1.7.9 - args: - - c - - a - - b + args: ['c', 'a', 'b'] env: - name: DEMO_GREETING value: "Hello from the environment" @@ -139,10 +136,7 @@ spec: containers: - name: nginx image: nginx:1.7.9 - args: - - c - - a - - b + args: ['c', 'a', 'b'] env: - name: DEMO_GREETING value: "Hello from the environment" @@ -208,10 +202,7 @@ spec: containers: - name: nginx image: nginx:1.7.9 - args: - - c - - a - - b + args: ['c', 'a', 'b'] env: - name: DEMO_GREETING value: "Hello from the environment" @@ -276,10 +267,7 @@ spec: containers: - name: nginx image: nginx:1.7.9 - args: - - c - - a - - b + args: ['c', 'a', 'b'] env: - name: DEMO_GREETING value: "New Demo Greeting" @@ -343,10 +331,7 @@ spec: containers: - name: nginx image: nginx:1.7.9 - args: - - e - - d - - f + args: ['e', 'd', 'f'] env: - name: DEMO_GREETING value: "Hello from the environment" diff --git a/kyaml/yaml/merge3/element_test.go b/kyaml/yaml/merge3/element_test.go index d3a28a549..7438d5ce8 100644 --- a/kyaml/yaml/merge3/element_test.go +++ b/kyaml/yaml/merge3/element_test.go @@ -174,8 +174,7 @@ kind: Deployment containers: - name: foo image: foo:bar - command: - - run2.sh + command: ['run2.sh'] `, nil}, // @@ -206,8 +205,7 @@ kind: Deployment containers: - name: foo image: foo:bar - command: - - run2.sh + command: ['run2.sh'] `, nil}, // @@ -239,8 +237,7 @@ kind: Deployment containers: - name: foo image: foo:bar - command: - - run2.sh + command: ['run2.sh'] `, nil}, // @@ -367,8 +364,7 @@ kind: Deployment containers: - name: foo image: foo:bar - command: - - run.sh + command: ['run.sh'] `, nil}, // diff --git a/kyaml/yaml/types.go b/kyaml/yaml/types.go index 46fa745fe..99d3eb668 100644 --- a/kyaml/yaml/types.go +++ b/kyaml/yaml/types.go @@ -146,9 +146,6 @@ func NewListRNode(values ...string) *RNode { // NewRNode returns a new RNode pointer containing the provided Node. func NewRNode(value *yaml.Node) *RNode { - if value != nil { - value.Style = 0 - } return &RNode{value: value} } @@ -389,7 +386,7 @@ const ( Flow = "Flow" ) -// String returns a string valuer for a Node, applying the supplied formatting options +// String returns a string value for a Node, applying the supplied formatting options func String(node *yaml.Node, opts ...string) (string, error) { if node == nil { return "", nil