diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml new file mode 100644 index 000000000..e9f35f5b6 --- /dev/null +++ b/.github/workflows/go.yml @@ -0,0 +1,28 @@ +name: Go + +on: + push: + branches: [ master ] + pull_request: + branches: [ master ] + +jobs: + + build: + name: Build + runs-on: ubuntu-latest + steps: + + - name: Set up Go 1.x + uses: actions/setup-go@v2 + with: + go-version: ^1.13 + id: go + + - name: Check out code into the Go module directory + uses: actions/checkout@v2 + + - name: Build + run: ./travis/kyaml-pre-commit.sh + env: + KUSTOMIZE_DOCKER_E2E: true diff --git a/api/krusty/baseandoverlaymedium_test.go b/api/krusty/baseandoverlaymedium_test.go index 569dc6c27..19bd3d8da 100644 --- a/api/krusty/baseandoverlaymedium_test.go +++ b/api/krusty/baseandoverlaymedium_test.go @@ -154,9 +154,8 @@ LEGUME=chickpea `) th.WriteF("/app/overlay/configmap/dummy.txt", `Lorem ipsum dolor sit amet, consectetur - -adipiscing elit, sed do eiusmod tempor -incididunt ut labore et dolore magna aliqua. +adipiscing elit, sed do eiusmod tempor +incididunt ut labore et dolore magna aliqua. `) th.WriteF("/app/overlay/deployment/deployment.yaml", ` apiVersion: apps/v1 @@ -293,11 +292,8 @@ metadata: --- apiVersion: v1 data: - nonsense: | - Lorem ipsum dolor sit amet, consectetur - - adipiscing elit, sed do eiusmod tempor - incididunt ut labore et dolore magna aliqua. + nonsense: "Lorem ipsum dolor sit amet, consectetur\nadipiscing elit, sed do eiusmod + tempor\nincididunt ut labore et dolore magna aliqua. \n" kind: ConfigMap metadata: annotations: @@ -306,6 +302,6 @@ metadata: app: mungebot org: kubernetes repo: test-infra - name: test-infra-app-config-hh272bg5d4 + name: test-infra-app-config-f462h769f9 `) } diff --git a/api/kv/kv.go b/api/kv/kv.go index 7bda10fd7..b887e583c 100644 --- a/api/kv/kv.go +++ b/api/kv/kv.go @@ -9,7 +9,6 @@ import ( "fmt" "os" "path" - "regexp" "strings" "unicode" "unicode/utf8" @@ -86,17 +85,11 @@ func (kvl *loader) keyValuesFromFileSources(sources []string) ([]types.Pair, err if err != nil { return nil, err } - kvs = append(kvs, types.Pair{Key: k, Value: trimTrailingSpacesInLines(string(content))}) + kvs = append(kvs, types.Pair{Key: k, Value: string(content)}) } return kvs, nil } -// trimTrailingSpacesInLines takes string with multiple lines and trims the trailing white spaces and tabs from each line. -func trimTrailingSpacesInLines(str string) string { - re := regexp.MustCompile(`[ \t]*\n`) - return re.ReplaceAllString(str, "\n") -} - func (kvl *loader) keyValuesFromEnvFiles(paths []string) ([]types.Pair, error) { var kvs []types.Pair for _, p := range paths { diff --git a/api/kv/kv_test.go b/api/kv/kv_test.go index 7241fcd98..bc362adce 100644 --- a/api/kv/kv_test.go +++ b/api/kv/kv_test.go @@ -95,12 +95,3 @@ func TestKeyValuesFromFileSources(t *testing.T) { } } } - -func TestTrimTrailingSpacesInLines(t *testing.T) { - input := "\"fooKey\": \"fooValue\" \t\n \t\t \n\t\"barKey\": \"barValue\"" - expected := "\"fooKey\": \"fooValue\"\n\n\t\"barKey\": \"barValue\"" - res := trimTrailingSpacesInLines(input) - if !reflect.DeepEqual(res, expected) { - t.Errorf("Trim trailing spaces in lines should succeed, got: %s exptected: %s", res, expected) - } -} diff --git a/cmd/config/.golangci.yml b/cmd/config/.golangci.yml index 26827c174..fc4b84297 100644 --- a/cmd/config/.golangci.yml +++ b/cmd/config/.golangci.yml @@ -19,7 +19,7 @@ linters: # - gochecknoinits # - goconst # - gocritic - - gocyclo +# - gocyclo - gofmt - goimports # - golint diff --git a/cmd/config/Makefile b/cmd/config/Makefile index be9f2f345..d3b450817 100644 --- a/cmd/config/Makefile +++ b/cmd/config/Makefile @@ -32,7 +32,7 @@ lint: $(GOBIN)/golangci-lint run ./... test: - go test -cover ./... + go test -v -timeout 45m -cover ./... vet: go vet ./... diff --git a/cmd/config/go.mod b/cmd/config/go.mod index 8b5dcb00f..19a5dbd6b 100644 --- a/cmd/config/go.mod +++ b/cmd/config/go.mod @@ -3,15 +3,12 @@ module sigs.k8s.io/kustomize/cmd/config go 1.13 require ( - github.com/coreos/go-etcd v2.0.0+incompatible // indirect - github.com/cpuguy83/go-md2man v1.0.10 // indirect github.com/go-errors/errors v1.0.1 github.com/olekukonko/tablewriter v0.0.4 github.com/posener/complete/v2 v2.0.1-alpha.12 github.com/spf13/cobra v1.0.0 github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.4.0 - github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8 // indirect k8s.io/apimachinery v0.17.0 sigs.k8s.io/kustomize/kyaml v0.1.7 ) diff --git a/cmd/config/go.sum b/cmd/config/go.sum index 23657cce4..3fe32149d 100644 --- a/cmd/config/go.sum +++ b/cmd/config/go.sum @@ -23,11 +23,9 @@ github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMn github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= -github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= -github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/davecgh/go-spew v0.0.0-20151105211317-5215b55f46b2/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -161,7 +159,6 @@ github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40T github.com/qri-io/starlib v0.4.2-0.20200213133954-ff2e8cd5ef8d h1:K6eOUihrFLdZjZnA4XlRp864fmWXv9YTIk7VPLhRacA= github.com/qri-io/starlib v0.4.2-0.20200213133954-ff2e8cd5ef8d/go.mod h1:7DPO4domFU579Ga6E61sB9VFNaniPVwJP5C4bBCu3wA= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= -github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= @@ -180,7 +177,6 @@ github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzu github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -192,12 +188,10 @@ github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJy github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= -github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca h1:1CFlNzQhALwjS9mBAUkycX616GzgsuYUOCHA5+HSlXI= github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= -github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.starlark.net v0.0.0-20190528202925-30ae18b8564f/go.mod h1:c1/X6cHgvdXj6pUlmWKMkuqRnW4K8x2vwt6JAaaircg= go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5 h1:+FNtrFTmVw0YZGpBGX56XDee331t6JAXeK2bcyhLOOc= @@ -206,12 +200,9 @@ go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -219,10 +210,8 @@ golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191004110552-13f9640d40b9 h1:rjwSpXsdiK0dV8/Naq3kAw9ymfAeJIyd0upUIElB+lI= golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -232,17 +221,13 @@ golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAG golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191002063906-3421d5a6bb1c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -256,11 +241,6 @@ golang.org/x/tools v0.0.0-20181011042414-1f849cf54d09/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20200430040329-4b814e061378/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= diff --git a/cmd/config/internal/commands/e2e/e2e_test.go b/cmd/config/internal/commands/e2e/e2e_test.go new file mode 100644 index 000000000..a324eea87 --- /dev/null +++ b/cmd/config/internal/commands/e2e/e2e_test.go @@ -0,0 +1,661 @@ +// Copyright 2019 The Kubernetes Authors. +// SPDX-License-Identifier: Apache-2.0 + +package e2e + +import ( + "bytes" + "fmt" + "io/ioutil" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestRunE2e(t *testing.T) { + binDir, err := ioutil.TempDir("", "kustomize-test-") + if !assert.NoError(t, err) { + t.FailNow() + } + //defer os.RemoveAll(binDir) + build(t, binDir) + + tests := []struct { + name string + args func(string) []string + files func(string) map[string]string + expectedFiles func(string) map[string]string + expectedErr string + skipIfFalseEnv string + }{ + { + name: "exec_function_no_args", + args: func(d string) []string { + return []string{ + "--enable-exec", "--exec-path", filepath.Join(d, "e2econtainerconfig"), + } + }, + files: func(d string) map[string]string { + return map[string]string{ + "deployment.yaml": ` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: foo +`, + } + }, + expectedFiles: func(d string) map[string]string { + return map[string]string{ + "deployment.yaml": ` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: foo + annotations: + a-string-value: '' + a-int-value: '0' + a-bool-value: 'false' +`, + } + }, + }, + + { + name: "exec_function_args", + args: func(d string) []string { + return []string{ + "--enable-exec", "--exec-path", filepath.Join(d, "e2econtainerconfig"), + "--", "stringValue=a", "intValue=1", "boolValue=true", + } + }, + files: func(d string) map[string]string { + return map[string]string{ + "deployment.yaml": ` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: foo +`, + } + }, + expectedFiles: func(d string) map[string]string { + return map[string]string{ + "deployment.yaml": ` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: foo + annotations: + a-string-value: 'a' + a-int-value: '1' + a-bool-value: 'true' +`, + } + }, + }, + + { + name: "exec_function_config", + args: func(d string) []string { + return []string{"--enable-exec"} + }, + files: func(d string) map[string]string { + return map[string]string{ + "deployment.yaml": fmt.Sprintf(` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: foo + annotations: + config.kubernetes.io/function: | + exec: + path: "%s" +`, filepath.Join(d, "e2econtainerconfig")), + } + }, + expectedFiles: func(d string) map[string]string { + return map[string]string{ + "deployment.yaml": fmt.Sprintf(` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: foo + annotations: + config.kubernetes.io/function: | + exec: + path: "%s" + a-string-value: '' + a-int-value: '0' + a-bool-value: 'false' +`, filepath.Join(d, "e2econtainerconfig"))} + }, + }, + + // + // Starklark function tests + // + { + name: "exec_function_config", + args: func(d string) []string { + return []string{"--enable-exec"} + }, + files: func(d string) map[string]string { + return map[string]string{ + "config.yaml": fmt.Sprintf(` +apiVersion: example.com/v1alpha1 +kind: Input +metadata: + name: foo + annotations: + config.kubernetes.io/function: | + exec: + path: "%s" +data: + stringValue: a + intValue: 2 + boolValue: true +`, filepath.Join(d, "e2econtainerconfig")), + "deployment.yaml": ` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: foo +`, + } + }, + expectedFiles: func(d string) map[string]string { + return map[string]string{ + "config.yaml": fmt.Sprintf(` +apiVersion: example.com/v1alpha1 +kind: Input +metadata: + name: foo + annotations: + config.kubernetes.io/function: | + exec: + path: "%s" + a-string-value: 'a' + a-int-value: '2' + a-bool-value: 'true' +data: + stringValue: a + intValue: 2 + boolValue: true +`, filepath.Join(d, "e2econtainerconfig")), + "deployment.yaml": ` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: foo + annotations: + a-string-value: 'a' + a-int-value: '2' + a-bool-value: 'true' +`, + } + }, + }, + + { + // + // NOTE: Do not change the expected value of this test. It is to ensure that + // exec functions are off by default when run from the CLI. + // exec functions execute arbitrary code outside of a sandbox environment. + // + name: "exec_function_config_disabled", + args: func(d string) []string { return []string{} }, + files: func(d string) map[string]string { + return map[string]string{ + "config.yaml": fmt.Sprintf(` +apiVersion: example.com/v1alpha1 +kind: Input +metadata: + name: foo + annotations: + config.kubernetes.io/function: | + exec: + path: "%s" +data: + stringValue: a + intValue: 2 + boolValue: true +`, filepath.Join(d, "e2econtainerconfig")), + "deployment.yaml": ` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: foo +`, + } + }, + expectedFiles: func(d string) map[string]string { + return map[string]string{ + "config.yaml": fmt.Sprintf(` +apiVersion: example.com/v1alpha1 +kind: Input +metadata: + name: foo + annotations: + config.kubernetes.io/function: | + exec: + path: "%s" +data: + stringValue: a + intValue: 2 + boolValue: true +`, filepath.Join(d, "e2econtainerconfig")), + "deployment.yaml": ` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: foo +`, + } + }, + }, + + { + name: "exec_function_no_enable", + expectedErr: "must specify --enable-exec with --exec-path", + args: func(d string) []string { + return []string{"--exec-path", filepath.Join(d, "e2econtainerconfig")} + }, + files: func(d string) map[string]string { + return map[string]string{ + "deployment.yaml": ` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: foo +`, + } + }, + expectedFiles: func(d string) map[string]string { + return map[string]string{ + "deployment.yaml": ` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: foo + annotations: + a-string-value: '' + a-int-value: '0' + a-bool-value: 'false' +`, + } + }, + }, + + // + // Container + // + { + name: "container_function_no_args", + skipIfFalseEnv: "KUSTOMIZE_DOCKER_E2E", + args: func(d string) []string { + return []string{"--image", "gcr.io/kustomize-functions/e2econtainerconfig"} + }, + files: func(d string) map[string]string { + return map[string]string{ + "deployment.yaml": ` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: foo +`, + } + }, + expectedFiles: func(d string) map[string]string { + return map[string]string{ + "deployment.yaml": ` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: foo + annotations: + a-string-value: '' + a-int-value: '0' + a-bool-value: 'false' +`, + } + }, + }, + + { + name: "container_function_args", + skipIfFalseEnv: "KUSTOMIZE_DOCKER_E2E", + args: func(d string) []string { + return []string{ + "--image", "gcr.io/kustomize-functions/e2econtainerconfig", + "--", "stringValue=a", "intValue=1", "boolValue=true", + } + }, + files: func(d string) map[string]string { + return map[string]string{ + "deployment.yaml": ` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: foo +`, + } + }, + expectedFiles: func(d string) map[string]string { + return map[string]string{ + "deployment.yaml": ` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: foo + annotations: + a-string-value: 'a' + a-int-value: '1' + a-bool-value: 'true' +`, + } + }, + }, + + { + name: "container_function_config", + skipIfFalseEnv: "KUSTOMIZE_DOCKER_E2E", + args: func(d string) []string { return []string{} }, + files: func(d string) map[string]string { + return map[string]string{ + "deployment.yaml": ` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: foo + annotations: + config.kubernetes.io/function: | + container: + image: "gcr.io/kustomize-functions/e2econtainerconfig" +`, + } + }, + expectedFiles: func(d string) map[string]string { + return map[string]string{ + "deployment.yaml": ` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: foo + annotations: + config.kubernetes.io/function: | + container: + image: "gcr.io/kustomize-functions/e2econtainerconfig" + a-string-value: '' + a-int-value: '0' + a-bool-value: 'false' +`} + }, + }, + + { + name: "container_function_config", + skipIfFalseEnv: "KUSTOMIZE_DOCKER_E2E", + args: func(d string) []string { return []string{} }, + files: func(d string) map[string]string { + return map[string]string{ + "config.yaml": ` +apiVersion: example.com/v1alpha1 +kind: Input +metadata: + name: foo + annotations: + config.kubernetes.io/function: | + container: + image: "gcr.io/kustomize-functions/e2econtainerconfig" +data: + stringValue: a + intValue: 2 + boolValue: true +`, + "deployment.yaml": ` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: foo +`, + } + }, + expectedFiles: func(d string) map[string]string { + return map[string]string{ + "config.yaml": ` +apiVersion: example.com/v1alpha1 +kind: Input +metadata: + name: foo + annotations: + config.kubernetes.io/function: | + container: + image: "gcr.io/kustomize-functions/e2econtainerconfig" + a-string-value: 'a' + a-int-value: '2' + a-bool-value: 'true' +data: + stringValue: a + intValue: 2 + boolValue: true +`, + "deployment.yaml": ` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: foo + annotations: + a-string-value: 'a' + a-int-value: '2' + a-bool-value: 'true' +`, + } + }, + }, + + { + name: "starlark_function_config", + args: func(d string) []string { return []string{"--enable-star"} }, + files: func(d string) map[string]string { + return map[string]string{ + "script.star": ` +# set the foo annotation on each resource +def run(r, fc): + for resource in r: + resource["metadata"]["annotations"]["a-string-value"] = fc["data"]["stringValue"] + resource["metadata"]["annotations"]["a-int-value"] = fc["data"]["intValue"] + resource["metadata"]["annotations"]["a-bool-value"] = fc["data"]["boolValue"] + +run(ctx.resource_list["items"], ctx.resource_list["functionConfig"]) +`, + "config.yaml": ` +apiVersion: example.com/v1alpha1 +kind: Input +metadata: + name: foo + annotations: + config.kubernetes.io/function: | + starlark: + path: script.star + name: fn +data: + boolValue: true + intValue: 2 + stringValue: a +`, + "deployment.yaml": ` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: foo +`, + } + }, + expectedFiles: func(d string) map[string]string { + return map[string]string{ + "config.yaml": ` +apiVersion: example.com/v1alpha1 +kind: Input +metadata: + name: foo + annotations: + a-bool-value: true + a-int-value: 2 + a-string-value: a + config.kubernetes.io/function: | + starlark: + path: script.star + name: fn +data: + boolValue: true + intValue: 2 + stringValue: a +`, + "deployment.yaml": ` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: foo + annotations: + a-bool-value: true + a-int-value: 2 + a-string-value: a +`, + } + }, + }, + + { + name: "starlark_function_path", + args: func(d string) []string { + return []string{ + "--enable-star", "--star-path", "script.star", + "--", "stringValue=a", "intValue=2", "boolValue=true", + } + }, + files: func(d string) map[string]string { + return map[string]string{ + "script.star": ` +# set the foo annotation on each resource +def run(r, fc): + for resource in r: + resource["metadata"]["annotations"]["a-string-value"] = fc["data"]["stringValue"] + resource["metadata"]["annotations"]["a-int-value"] = fc["data"]["intValue"] + resource["metadata"]["annotations"]["a-bool-value"] = fc["data"]["boolValue"] + +run(ctx.resource_list["items"], ctx.resource_list["functionConfig"]) +`, + "deployment.yaml": ` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: foo +`, + } + }, + expectedFiles: func(d string) map[string]string { + return map[string]string{ + "deployment.yaml": ` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: foo + annotations: + a-bool-value: true + a-int-value: 2 + a-string-value: a +`, + } + }, + }, + } + + for i := range tests { + tt := tests[i] + t.Run(tt.name, func(t *testing.T) { + if tt.skipIfFalseEnv != "" && os.Getenv(tt.skipIfFalseEnv) == "false" { + t.Skip() + } + + dir, err := ioutil.TempDir("", "kustomize-test-data-") + if !assert.NoError(t, err) { + t.FailNow() + } + defer os.RemoveAll(dir) + os.Chdir(dir) + + // write the input + for path, data := range tt.files(binDir) { + err := ioutil.WriteFile(path, []byte(data), 0600) + if !assert.NoError(t, err) { + t.FailNow() + } + } + + args := append([]string{"run", "."}, tt.args(binDir)...) + cmd := exec.Command(filepath.Join(binDir, "kyaml"), args...) + cmd.Dir = dir + var stdErr, stdOut bytes.Buffer + cmd.Stdout = &stdOut + cmd.Stderr = &stdErr + cmd.Env = os.Environ() + + err = cmd.Run() + if tt.expectedErr != "" { + if !assert.Contains(t, stdErr.String(), tt.expectedErr) { + t.FailNow() + } + return + } + if !assert.NoError(t, err, stdErr.String()) { + t.FailNow() + } + + for path, data := range tt.expectedFiles(binDir) { + b, err := ioutil.ReadFile(path) + if !assert.NoError(t, err) { + t.FailNow() + } + if !assert.Equal(t, strings.TrimSpace(data), strings.TrimSpace(string(b))) { + t.FailNow() + } + } + }) + } +} + +func build(t *testing.T, binDir string) { + build := exec.Command("go", "build", "-o", + filepath.Join(binDir, "e2econtainerconfig")) + build.Dir = "e2econtainerconfig" + build.Stdout = os.Stdout + build.Stderr = os.Stderr + if !assert.NoError(t, build.Run()) { + t.FailNow() + } + + build = exec.Command("go", "build", "-o", filepath.Join(binDir, "kyaml")) + build.Dir = filepath.Join("..", "..", "..") + build.Stdout = os.Stdout + build.Stderr = os.Stderr + if !assert.NoError(t, build.Run()) { + t.FailNow() + } + + if os.Getenv("KUSTOMIZE_DOCKER_E2E") == "false" { + return + } + build = exec.Command( + "docker", "build", ".", "-t", "gcr.io/kustomize-functions/e2econtainerconfig") + build.Dir = "e2econtainerconfig" + build.Stdout = os.Stdout + build.Stderr = os.Stderr + if !assert.NoError(t, build.Run()) { + t.FailNow() + } +} diff --git a/cmd/config/internal/commands/e2e/e2econtainerconfig/Dockerfile b/cmd/config/internal/commands/e2e/e2econtainerconfig/Dockerfile new file mode 100644 index 000000000..d177d7df1 --- /dev/null +++ b/cmd/config/internal/commands/e2e/e2econtainerconfig/Dockerfile @@ -0,0 +1,12 @@ +# Copyright 2019 The Kubernetes Authors. +# SPDX-License-Identifier: Apache-2.0 + +FROM golang:1.13-stretch +ENV CGO_ENABLED=0 +WORKDIR /go/src/ +COPY . . +RUN go build -v -o /usr/local/bin/function ./ + +FROM alpine:latest +COPY --from=0 /usr/local/bin/function /usr/local/bin/function +CMD ["function"] diff --git a/cmd/config/internal/commands/e2e/e2econtainerconfig/Makefile b/cmd/config/internal/commands/e2e/e2econtainerconfig/Makefile new file mode 100644 index 000000000..c02fa8594 --- /dev/null +++ b/cmd/config/internal/commands/e2e/e2econtainerconfig/Makefile @@ -0,0 +1,13 @@ +# Copyright 2019 The Kubernetes Authors. +# SPDX-License-Identifier: Apache-2.0 + +.PHONY: generate license fix vet fmt test build tidy + +GOBIN := $(shell go env GOPATH)/bin + +test: + go test + +image: + docker build . -t gcr.io/kustomize-functions/e2econtainerconfig + docker push gcr.io/kustomize-functions/e2econtainerconfig diff --git a/cmd/config/internal/commands/e2e/e2econtainerconfig/doc.go b/cmd/config/internal/commands/e2e/e2econtainerconfig/doc.go new file mode 100644 index 000000000..610a6b9b6 --- /dev/null +++ b/cmd/config/internal/commands/e2e/e2econtainerconfig/doc.go @@ -0,0 +1,9 @@ +// Package main contains a function to be used for e2e testing. +// +// The function is written using the framework, and parses the ResourceList.functionConfig +// into a go struct. +// +// The function will set 3 annotations on each resource. +// +// See https://github.com/kubernetes-sigs/kustomize/blob/master/cmd/config/docs/api-conventions/functions-spec.md +package main diff --git a/cmd/config/internal/commands/e2e/e2econtainerconfig/go.mod b/cmd/config/internal/commands/e2e/e2econtainerconfig/go.mod new file mode 100644 index 000000000..5f2de0270 --- /dev/null +++ b/cmd/config/internal/commands/e2e/e2econtainerconfig/go.mod @@ -0,0 +1,6 @@ +module sigs.k8s.io/kustomize/cmd/config/internal/commands/e2e/e2econtainerconfig + +go 1.14 + +require sigs.k8s.io/kustomize/kyaml v0.1.9-0.20200501190629-f7909fad7167 + diff --git a/cmd/config/internal/commands/e2e/e2econtainerconfig/go.sum b/cmd/config/internal/commands/e2e/e2econtainerconfig/go.sum new file mode 100644 index 000000000..0768b14bf --- /dev/null +++ b/cmd/config/internal/commands/e2e/e2econtainerconfig/go.sum @@ -0,0 +1,183 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +github.com/360EntSecGroup-Skylar/excelize v1.4.1/go.mod h1:vnax29X2usfl7HHkBrX5EvSCJcmH3dT9luvxzu8iGAE= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/PuerkitoBio/goquery v1.5.0/go.mod h1:qD2PgZ9lccMbQlc7eEOjaeRlFQON7xY8kdmcsrnKqMg= +github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI= +github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= +github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M= +github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/andybalholm/cascadia v1.0.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= +github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= +github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= +github.com/dustmop/soup v1.1.2-0.20190516214245-38228baa104e/go.mod h1:CgNC6SGbT+Xb8wGGvzilttZL1mc5sQ/5KkcxsZttMIk= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-errors/errors v1.0.1 h1:LUHzmkK3GUKUrL/1gfBUxAHzcev3apQlezX/+O7ma6w= +github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= +github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= +github.com/go-openapi/jsonpointer v0.19.3 h1:gihV7YNZK1iK6Tgwwsxo2rJbD1GTbdm72325Bq8FI3w= +github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/jsonreference v0.19.2 h1:o20suLFB4Ri0tuzpWtyHlh7E7HnkqTNLq6aR6WVNS1w= +github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc= +github.com/go-openapi/spec v0.19.5 h1:Xm0Ao53uqnk9QE/LlYV5DEU09UAgpliA85QoT9LzqPw= +github.com/go-openapi/spec v0.19.5/go.mod h1:Hm2Jr4jv8G1ciIAo+frC/Ft+rR2kQDh8JHKHb3gWUSk= +github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-openapi/swag v0.19.5 h1:lTz6Ys4CmqqCQmZPBlbQENR1/GucA2bzYTE12Pw4tFY= +github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= +github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= +github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e h1:hB2xlXdHp/pmPZq0y3QnmWAArdw9PqbmotexnWx/FU8= +github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8= +github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= +github.com/paulmach/orb v0.1.3/go.mod h1:VFlX/8C+IQ1p6FTRRKzKoOPJnvEtA5G0Veuqwbu//Vk= +github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= +github.com/qri-io/starlib v0.4.2-0.20200213133954-ff2e8cd5ef8d/go.mod h1:7DPO4domFU579Ga6E61sB9VFNaniPVwJP5C4bBCu3wA= +github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= +github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= +github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v1.0.0 h1:6m/oheQuQ13N9ks4hubMG6BnvwOeaJrqSPLahSnczz8= +github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= +github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.2.3-0.20181224173747-660f15d67dbb/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= +github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca h1:1CFlNzQhALwjS9mBAUkycX616GzgsuYUOCHA5+HSlXI= +github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg= +github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= +go.starlark.net v0.0.0-20190528202925-30ae18b8564f/go.mod h1:c1/X6cHgvdXj6pUlmWKMkuqRnW4K8x2vwt6JAaaircg= +go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5/go.mod h1:nmDLcffg48OtT/PSW0Hg7FvpRQsQh5OSqIylirxKC7o= +go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b h1:0mm1VjtFUOIlE1SbDlwjYaDxZVDP2S5ou6y0gSgXHu8= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20191002063906-3421d5a6bb1c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= +gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.7 h1:VUgggvou5XRW9mHwD/yXxIYSMtY0zoKQf/v226p2nyo= +gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20191120175047-4206685974f2 h1:XZx7nhd5GMaZpmDaEHFVafUZC7ya0fuo7cSJ3UCKYmM= +gopkg.in/yaml.v3 v3.0.0-20191120175047-4206685974f2/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +sigs.k8s.io/kustomize/kyaml v0.1.9-0.20200501190629-f7909fad7167 h1:138Q3rZVU5mLBF6dFekNuP6D4ZeF4mndI54RBJ8kK8c= +sigs.k8s.io/kustomize/kyaml v0.1.9-0.20200501190629-f7909fad7167/go.mod h1:I4OFZ1vTPdteiqqCBwW3DI0swPzxBpd99y9CHN5IMUU= diff --git a/cmd/config/internal/commands/e2e/e2econtainerconfig/main.go b/cmd/config/internal/commands/e2e/e2econtainerconfig/main.go new file mode 100644 index 000000000..0e7fe0994 --- /dev/null +++ b/cmd/config/internal/commands/e2e/e2econtainerconfig/main.go @@ -0,0 +1,57 @@ +// Copyright 2019 The Kubernetes Authors. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "fmt" + "os" + + "sigs.k8s.io/kustomize/kyaml/fn/framework" + "sigs.k8s.io/kustomize/kyaml/yaml" +) + +// Data contains the items +type Data struct { + StringValue string `yaml:"stringValue,omitempty"` + + IntValue int `yaml:"intValue,omitempty"` + + BoolValue bool `yaml:"boolValue,omitempty"` +} + +// Example defines the ResourceList.functionConfig schema. +type Example struct { + // Data contains configuration data for the Example + // Nest values under Data so that the function can accept a ConfigMap as its + // functionConfig (`run` generates a ConfigMap for the functionConfig when run with --) + Data Data `yaml:"data,omitempty"` +} + +func main() { + functionConfig := &Example{} + + cmd := framework.Command(functionConfig, func(items []*yaml.RNode) ([]*yaml.RNode, error) { + for i := range items { + if err := items[i].PipeE(yaml.SetAnnotation("a-string-value", + functionConfig.Data.StringValue)); err != nil { + return nil, err + } + + if err := items[i].PipeE(yaml.SetAnnotation("a-int-value", + fmt.Sprintf("%v", functionConfig.Data.IntValue))); err != nil { + return nil, err + } + + if err := items[i].PipeE(yaml.SetAnnotation("a-bool-value", + fmt.Sprintf("%v", functionConfig.Data.BoolValue))); err != nil { + return nil, err + } + } + return items, nil + }) + + if err := cmd.Execute(); err != nil { + os.Exit(1) + } +} diff --git a/cmd/config/internal/commands/run-fns.go b/cmd/config/internal/commands/run-fns.go index 38ba38190..0a2734e55 100644 --- a/cmd/config/internal/commands/run-fns.go +++ b/cmd/config/internal/commands/run-fns.go @@ -11,7 +11,7 @@ import ( "github.com/spf13/cobra" "sigs.k8s.io/kustomize/cmd/config/internal/generateddocs/commands" "sigs.k8s.io/kustomize/kyaml/errors" - "sigs.k8s.io/kustomize/kyaml/kio/filters" + "sigs.k8s.io/kustomize/kyaml/fn/runtime/runtimeutil" "sigs.k8s.io/kustomize/kyaml/runfn" "sigs.k8s.io/kustomize/kyaml/yaml" ) @@ -41,15 +41,17 @@ func GetRunFnRunner(name string) *RunFnRunner { r.Command.Flags().StringVar( &r.Image, "image", "", "run this image as a function instead of discovering them.") + // NOTE: exec plugins execute arbitrary code -- never change the default value of this flag!!! r.Command.Flags().BoolVar( - &r.EnableStar, "enable-star", false, "enable support for starlark functions.") - r.Command.Flags().MarkHidden("enable-star") + &r.EnableExec, "enable-exec", false /*do not change!*/, "enable support for exec functions -- note: exec functions run arbitrary code -- do not use for untrusted configs!!! (Alpha)") r.Command.Flags().StringVar( - &r.StarPath, "star-path", "", "run a starlark script as a function.") - r.Command.Flags().MarkHidden("star-path") + &r.ExecPath, "exec-path", "", "run an executable as a function. (Alpha)") + r.Command.Flags().BoolVar( + &r.EnableStar, "enable-star", false, "enable support for starlark functions. (Alpha)") r.Command.Flags().StringVar( - &r.StarName, "star-name", "", "name of starlark program.") - r.Command.Flags().MarkHidden("star-name") + &r.StarPath, "star-path", "", "run a starlark script as a function. (Alpha)") + r.Command.Flags().StringVar( + &r.StarName, "star-name", "", "name of starlark program. (Alpha)") r.Command.Flags().StringVar( &r.ResultsDir, "results-dir", "", "write function results to this dir") @@ -79,6 +81,8 @@ type RunFnRunner struct { EnableStar bool StarPath string StarName string + EnableExec bool + ExecPath string RunFns runfn.RunFns ResultsDir string Network bool @@ -94,14 +98,13 @@ func (r *RunFnRunner) runE(c *cobra.Command, args []string) error { // Functions to run. func (r *RunFnRunner) getContainerFunctions(c *cobra.Command, args, dataItems []string) ( []*yaml.RNode, error) { - if r.Image == "" && r.StarPath == "" { + if r.Image == "" && r.StarPath == "" && r.ExecPath == "" { return nil, nil } var fn *yaml.RNode var err error - // if image isn't specified, then Functions is empty if r.Image != "" { // create the function spec to set as an annotation fn, err = yaml.Parse(`container: {}`) @@ -143,8 +146,19 @@ func (r *RunFnRunner) getContainerFunctions(c *cobra.Command, args, dataItems [] if err != nil { return nil, err } - } else { - return nil, nil + } else if r.EnableExec && r.ExecPath != "" { + // create the function spec to set as an annotation + fn, err = yaml.Parse(`exec: {}`) + if err != nil { + return nil, err + } + + err = fn.PipeE( + yaml.Lookup("exec"), + yaml.SetField("path", yaml.NewScalarRNode(r.ExecPath))) + if err != nil { + return nil, err + } } // create the function config @@ -208,10 +222,10 @@ data: {} return []*yaml.RNode{rc}, nil } -func toStorageMounts(mounts []string) []filters.StorageMount { - var sms []filters.StorageMount +func toStorageMounts(mounts []string) []runtimeutil.StorageMount { + var sms []runtimeutil.StorageMount for _, mount := range mounts { - sms = append(sms, filters.StringToStorageMount(mount)) + sms = append(sms, runtimeutil.StringToStorageMount(mount)) } return sms } @@ -221,8 +235,12 @@ func (r *RunFnRunner) preRunE(c *cobra.Command, args []string) error { return errors.Errorf("must specify --enable-star with --star-path") } + if !r.EnableExec && r.ExecPath != "" { + return errors.Errorf("must specify --enable-exec with --exec-path") + } + if c.ArgsLenAtDash() >= 0 && r.Image == "" && - !(r.EnableStar && r.StarPath != "") { + !(r.EnableStar && r.StarPath != "") && !(r.EnableExec && r.ExecPath != "") { return errors.Errorf("must specify --image") } @@ -270,6 +288,7 @@ func (r *RunFnRunner) preRunE(c *cobra.Command, args []string) error { Network: r.Network, NetworkName: r.NetworkName, EnableStarlark: r.EnableStar, + EnableExec: r.EnableExec, StorageMounts: storageMounts, ResultsDir: r.ResultsDir, } diff --git a/docs/zh/README.md b/docs/zh/README.md index d012297c3..0e87c17b0 100644 --- a/docs/zh/README.md +++ b/docs/zh/README.md @@ -19,6 +19,16 @@ ## 发行说明 +* [kustomize/3.2.2](/../../releases/tag/kustomize%2Fv3.2.2) - 基于 kustomize Go API [3.3.0](../v3.3.0.md) 的 `kustomize` CLI。 + + * [API 3.3.0](../v3.3.0.md) - kustomize Go API 的首个 release，该 release 不包含 `kustomize` CLI，且之后 CLI 和 API 将独立发布。 + + * [kustomize/3.2.1](/../../releases/tag/kustomize%2Fv3.2.1) - `kustomize` CLI 基于 kustomize Go API release [3.2.0](../v3.2.0.md) 发布的补丁。 + + * [3.2.0](../v3.2.0.md) + + * [3.1.1](../v3.1.0.md) + * [3.1](../v3.1.0.md) - 2019年7月下旬，扩展 patches 和改进的资源匹配。 * [3.0](../v3.0.0.md) - 2019年6月下旬，插件开发者发布。 diff --git a/docs/zh/fields.md b/docs/zh/fields.md index bbb8fd470..ab81179bf 100644 --- a/docs/zh/fields.md +++ b/docs/zh/fields.md @@ -1,6 +1,18 @@ # Kustomization 文件字段 -介绍 [kustomization](../glossary.md#kustomization) 文件中各字段的含义。 +[field-name-namespace]: ../plugins/builtins.md#field-name-namespace +[field-name-images]: ../plugins/builtins.md#field-name-images +[field-names-namePrefix-nameSuffix]: ../plugins/builtins.md#field-names-namePrefix-nameSuffix +[field-name-patches]: ../plugins/builtins.md#field-name-patches +[field-name-patchesStrategicMerge]: ../plugins/builtins.md#field-name-patchesStrategicMerge +[field-name-patchesJson6902]: ../plugins/builtins.md#field-name-patchesJson6902 +[field-name-replicas]: ../plugins/builtins.md#field-name-replicas +[field-name-secretGenerator]: ../plugins/builtins.md#field-name-secretGenerator +[field-name-commonLabels]: ../plugins/builtins.md#field-name-commonLabels +[field-name-commonAnnotations]: ../plugins/builtins.md#field-name-commonAnnotations +[field-name-configMapGenerator]: ../plugins/builtins.md#field-name-configMapGenerator + +介绍 [kustomization.yaml](../glossary.md#kustomization) 配置文件中各字段的含义。 ## Resources @@ -8,36 +20,36 @@ | 字段 | 类型 | 说明 | | --- | --- | --- | -|[resources](#resources) | list | 包含 k8s API 对象的文件，或其他包含 kustomizations 文件的目录。 | -|[CRDs](#crds)| list | CDR 文件，以允许在资源列表中指定自定义资源。 | +|[resources](#resources) | list | 包含 k8s API 对象的文件，或其他包含 `kustomization.yaml` 文件的目录。 | +|[CRDs](#crds)| list | CDR 文件，允许在资源列表中指定自定义资源。 | ## Generators -生成可定制的对象。 +资源生成器。 | 字段 | 类型 | 说明 | | --- | --- | --- | -|[configMapGenerator](#configmapgenerator)| list | 列表中的每个条目都将创建一个 ConfigMap （它是n个 ConfigMap 的生成器）。 | -|[secretGenerator](#secretgenerator)| list | 此列表中的每个条目都将创建一个 Secret 资源（它是n个 secrets 的生成器）。 | +|[configMapGenerator](#configmapgenerator)| list | 列表中的每个条目都将生成一个 ConfigMap （合计可以生成 n 个 ConfigMap）。 | +|[secretGenerator](#secretgenerator)| list | 列表中的每个条目都将生成一个 Secret（合计可以生成 n 个 Secrets）。 | |[generatorOptions](#generatoroptions)| string | generatorOptions 可以修改所有 ConfigMapGenerator 和 SecretGenerator 的行为。 | |[generators](#generators)| list | [插件](../plugins)配置文件。 | ## Transformers -可用的转换。 +资源字段转换项。 | 字段 | 类型 | 说明 | | --- | --- | --- | | [commonLabels](#commonlabels) | string | 为所有资源和 selectors 增加 Labels 。 | | [commonAnnotations](#commonannotations) | string | 为所有资源增加 Annotations 。 | -| [images](#images) | list | 修改镜像的名称、tag 或 image digest ，而无需使用 patches 。 | +| [images](#images) | list | 无需使用 patches，即可修改镜像的名称、tag 或 image digest。 | | [inventory](#inventory) | struct | 用于生成一个包含清单信息的对象。 | | [namespace](#namespace) | string | 为所有 resources 添加 namespace 。 | -| [namePrefix](#nameprefix) | string | 该字段的值将添加在所有资源的名称之前。 | -| [nameSuffix](#namesuffix) | string | 该字段的值将添加在所有资源的名称后面。 | +| [namePrefix](#nameprefix) | string | 为所有资源的名称添加前缀。 | +| [nameSuffix](#namesuffix) | string | 为所有资源的名称添加后缀。 | | [replicas](#replicas) | list | 修改资源的副本数。 | | [patchesStrategicMerge](#patchesstrategicmerge) | list | 此列表中的每个条目都应可以解析为部分或完整的资源定义文件。 | -| [patchesJson6902](#patchesjson6902) | list | 列表中的每个条目都应可以解析为 kubernetes 对象和将应用于该对象的 JSON patch 。 | +| [patchesJson6902](#patchesjson6902) | list | 列表中的每个条目都应可以解析为 Kubernetes 对象和将应用于该对象的 JSON patch 。 | | [transformers](#transformers) | list | [插件](../plugins)配置文件。 | ## Meta @@ -46,7 +58,7 @@ | 字段 | 类型 | 说明 | | --- | --- | --- | -| [vars](#vars) | string | 获取一个对象中的字段并插入到另外的对象中。 | +| [vars](#vars) | string | 将一个对象中的字段值插入另一个对象中。 | | [apiVersion](#apiversion) | string | [k8s metadata] 字段。 | | [kind](#kind) | string | [k8s metadata] 字段。 | @@ -67,59 +79,26 @@ apiVersion: kustomize.config.k8s.io/v1beta1 ### commonLabels -为所有资源和 selectors 增加 Labels - -``` -commonLabels: - someName: someValue - owner: alice - app: bingo -``` +详见 [field-name-commonLabels]。 ### commonAnnotations -为所有资源增加 Annotations ，和 labels 一样是 key:value 的键值对。 - -``` -commonAnnotations: - oncallPager: 800-555-1212 -``` +详见 [field-name-commonAnnotations]. ### configMapGenerator -列表中的每个条目都将创建一个 ConfigMap （它是n个 ConfigMap 的生成器）。 - -下面的示例创建了两个 ConfigMaps： - -- 一个具有给定文件的名称和内容 -- 另一个包含 key/value 键值对数据 - -每个 configMapGenerator 项都可以使用 `behavior: [create|replace|merge]` 参数。 - -允许 overlay 从父级修改或替换现有的 configMap。 - -``` -configMapGenerator: -- name: myJavaServerProps - files: - - application.properties - - more.properties -- name: myJavaServerEnvVars - literals: - - JAVA_HOME=/opt/java/jdk - - JAVA_TOOL_OPTIONS=-agentlib:hprof -``` +详见 [field-name-configMapGenerator]. ### crds 此列表中的每个条目都应该是自定义资源定义（CRD）文件的相对路径。 -该字段的存在是为了让 kustomize 知道用户自定义的 CRD ，并对这些类型中的对象应用适当的转换。 +该字段的存在是为了让 kustomize 识别用户自定义的 CRD ，并对这些类型中的对象应用适当的转换。 典型用例：CRD 引用 ConfigMap 对象 -在 kustomization 中，ConfigMap 对象名称可能会通过 namePrefix 、nameSuffix 或 hashing 来更改 CRD 对象中此 ConfigMap 对象的名称， -引用时需要以相同的方式使用 namePrefix 、 nameSuffix 或 hashing 来进行更新。 +在 kustomization 中，ConfigMap 对象名称可能会通过 `namePrefix` 、`nameSuffix` 或 `hashing` 来更改 CRD 对象中该 ConfigMap 对象的名称， +引用时需要以相同的方式使用 `namePrefix` 、 `nameSuffix` 或 `hashing` 来进行更新。 Annotations 可以放入 openAPI 的定义中： @@ -140,7 +119,7 @@ crds: ### generatorOptions -generatorOptions 修改所有 [ConfigMapGenerator](#configmapgenerator) 和 [SecretGenerator](#secretgenerator) 的行为。 +generatorOptions 可以修改所有 [ConfigMapGenerator](#configmapgenerator) 和 [SecretGenerator](#secretgenerator) 的行为。 ``` generatorOptions: @@ -166,41 +145,7 @@ generators: ### images -修改镜像的名称、tag 或 image digest ，而无需使用 patches 。例如，对于这种 kubernetes Deployment 片段： - -``` -containers: - - name: mypostgresdb - image: postgres:8 - - name: nginxapp - image: nginx:1.7.9 - - name: myapp - image: my-demo-app:latest - - name: alpine-app - image: alpine:3.7 -``` - -可以通过以下方式更改 `image` ： - - - `postgres:8` to `my-registry/my-postgres:v1`, - - nginx tag `1.7.9` to `1.8.0`, - - image name `my-demo-app` to `my-app`, - - alpine's tag `3.7` to a digest value - -可以在 *kustomization* 中添加以下内容： - -``` -images: -- name: postgres - newName: my-registry/my-postgres - newTag: v1 -- name: nginx - newTag: 1.8.0 -- name: my-demo-app - newName: my-app -- name: alpine - digest: sha256:24a0c4b4a4c0eb97a1aabb8e29f18e917d05abfe1b7a7c07857230879ce7d3d3 -``` +详见 [field-name-images]。 ### inventory @@ -217,120 +162,31 @@ kind: Kustomization ### namespace -为所有 resources 添加 namespace 。 - -``` -namespace: my-namespace -``` +详见 [field-name-namespace]。 ### namePrefix -该字段的值将添加在所有资源的名称之前，例如 将资源名称 `wordpress` 变为 `alices-wordpress` 。 - -``` -namePrefix: alices- -``` +详见 [field-names-namePrefix-nameSuffix]。 ### nameSuffix -该字段的值将添加在所有资源的名称后面，例如 将资源名称 `wordpress` 变为 `wordpress-v2` 。 +详见 [field-names-namePrefix-nameSuffix]。 -如果资源类型为 ConfigMap 或 Secret ，则在哈希值之前附加后缀。 +### patches -``` -nameSuffix: -v2 -``` +详见 [field-name-patches]。 ### patchesStrategicMerge -此列表中的每个条目都应可以解析为部分或完整的资源定义文件。 - -这些（也可能是部分的）资源文件中的 name 必须与已经通过 `resources` 加载的 name 字段匹配，或者通过 `bases` 中的 name 字段匹配。这些条目将用于 _patch_（修改）已知资源。 - -推荐使用小的 patches，例如：修改内存的 request/limit，更改 ConfigMap 中的 env 变量等小的 patches 易于维护和查看，并且易于在 overlays 中混合使用。 - -``` -patchesStrategicMerge: -- service_port_8888.yaml -- deployment_increase_replicas.yaml -- deployment_increase_memory.yaml -``` +详见 [field-name-patchesStrategicMerge]。 ### patchesJson6902 -patchesJson6902 列表中的每个条目都应可以解析为 kubernetes 对象和将应用于该对象的 JSON patch - -JSON patch 的文档地址：https://tools.ietf.org/html/rfc6902 - -目标字段指向的 kubernetes 对象的 group、 version、 kind、 name 和 namespace 在同一 kustomization 内 path 字段内容是 JSON patch 文件的相对路径。 - -patch 文件中的内容可以如下这种 JSON 格式： - -``` - [ - {"op": "add", "path": "/some/new/path", "value": "value"}, - {"op": "replace", "path": "/some/existing/path", "value": "new value"} - ] - ``` - -也可以使用 YAML 格式表示： - -``` -- op: add - path: /some/new/path - value: value -- op: replace - path: /some/existing/path - value: new value -``` - -``` -patchesJson6902: -- target: - version: v1 - kind: Deployment - name: my-deployment - path: add_init_container.yaml -- target: - version: v1 - kind: Service - name: my-service - path: add_service_annotation.yaml -``` +详见 [field-name-patchesJson6902]。 ### replicas -修改资源的副本数。 - -例如：对于如下 kubernetes Deployment 片段： - -``` -kind: Deployment -metadata: - name: deployment-name -spec: - replicas: 3 -``` - -在 kustomization 中添加以下内容，将副本数更改为5： - -``` -replicas: -- name: deployment-name - count: 5 -``` - -该字段内容为列表，所以可以同时修改许多资源。 - -#### Limitation - -由于这个声明无法设置 `kind:` 或 `group:` 它将匹配任何可以匹配名称的 `group` 和 `kind` ，并且它是以下之一： -- `Deployment` -- `ReplicationController` -- `ReplicaSet` -- `StatefulSet` - -对于更复杂的用例，请使用 patch 。 +详见 [field-name-replicas]。 ### resources @@ -348,8 +204,7 @@ resource: 将以深度优先的顺序读取和处理资源。 - -文件应包含 YAML 格式的 k8s 资源。一个资源描述文件可以含有多个由（“---”）分隔的资源。 +文件应包含 YAML 格式的 k8s 资源。一个资源描述文件可以含有多个由（`---`）分隔的资源。 应该包含 `resources` 字段的 kustomization 文件的指定文件目录的相对路径。 [hashicorp URL]: https://github.com/hashicorp/go-getter#url-format @@ -358,31 +213,11 @@ resource: ### secretGenerator -此列表中的每个条目都将创建一个 Secret 资源（它是n个 secrets 的生成器）。 - -``` -secretGenerator: -- name: app-tls - files: - - secret/tls.cert - - secret/tls.key - type: "kubernetes.io/tls" -- name: app-tls-namespaced - # you can define a namespace to generate secret in, defaults to: "default" - namespace: apps - files: - - tls.crt=catsecret/tls.cert - - tls.key=secret/tls.key - type: "kubernetes.io/tls" -- name: env_file_secret - envs: - - env.txt - type: Opaque -``` +详见 [field-name-secretGenerator]。 ### vars -Vars 用于从一个 resource 字段中获取文本，并将该文本插入指定位置 - 反射功能。 +Vars 用于从一个 resource 字段中获取值，并将该值插入指定位置 - 反射功能。 例如，假设需要在容器的 command 中指定了 Service 对象的名称，并在容器的 env 中指定了 Secret 对象的名称来确保以下内容可以正常工作： diff --git a/kyaml/fn/framework/doc.go b/kyaml/fn/framework/doc.go index 9691e10d3..f5b82fd18 100644 --- a/kyaml/fn/framework/doc.go +++ b/kyaml/fn/framework/doc.go @@ -1,54 +1,120 @@ // Copyright 2019 The Kubernetes Authors. // SPDX-License-Identifier: Apache-2.0 -// Package framework contains a framework for writing functions in go. +// Package framework contains a framework for writing functions in go. The function spec +// is defined at: https://github.com/kubernetes-sigs/kustomize/blob/master/cmd/config/docs/api-conventions/functions-spec.md // -// Example +// Examples // -// Example function implementation to set an annotation on each resource. +// Example function implementation using framework.ResourceList with functionConfig // +// type Spec struct { +// Value string `yaml:"value,omitempty"` +// } +// type Example struct { +// Spec Spec `yaml:"spec,omitempty"` +// } +// functionConfig := &Example{} +// +// rl := framework.ResourceList{FunctionConfig: functionConfig} +// if err := rl.Read(); err != nil { return err } +// +// for i := range rl.Items { +// // modify the items... +// } +// if err := rl.Write(); err != nil { return err } +// +// Example function implementation using framework.Command with flags +// +// var value string // cmd := framework.Command(nil, func(items []*yaml.RNode) ([]*yaml.RNode, error) { // for i := range items { -// if err := items[i].PipeE(yaml.SetAnnotation("value", value)); err != nil { -// return nil, err -// } +// // modify the items... // } // return items, nil // }) // cmd.Flags().StringVar(&value, "value", "", "annotation value") -// if err := cmd.Execute(); err != nil { -// panic(err) -// } +// if err := cmd.Execute(); err != nil { return err } // // Architecture // -// Functions are implemented as a go function which accept a slice of resources (items) -// and returns a modified slice of resources (items). +// Functions modify a slice of resources (ResourceList.items) which are read as input and written +// as output. The function itself may be configured through a functionConfig +// (ResourceList.functionConfig). +// +// Example Function Input: +// +// kind: ResourceList +// items: +// - kind: Deployment +// ... +// - kind: Service +// .... +// functionConfig: +// kind: Example +// spec: +// value: foo +// +// The functionConfig may be specified declaratively and run with +// +// config run DIR/ +// +// Declarative function declaration: +// +// kind: Example +// metadata: +// annotations: +// # run the function by creating this container and providing this +// # Example as the functionConfig +// config.kubernetes.io/function: | +// image: image/containing/fuction:impl +// spec: +// value: foo +// +// The framework takes care of serializing and deserializing the ResourceList. +// +// Generated ResourceList.functionConfig -- ConfigMaps +// +// Functions may also be specified imperatively and run using: +// +// config run DIR/ --image image/containing/fuction:impl -- value=foo +// +// When run imperatively, a ConfigMap is generated for the functionConfig, and the command +// arguments are set as ConfigMap data entries. +// +// kind: ConfigMap +// data: +// value: foo +// +// To write a function that can be run imperatively on the commandline, have it take a +// ConfigMap as its functionConfig. // // Mutator and Generator Functions // -// Functions may add, delete or modify resources for the returned slice. +// Functions may add, delete or modify resources by modifying the items slice. +// When using framework.Command this is done through returning the new items slice. +// When using framework.ResourceList this is done through modifying ResourceList.Items in place. // // Validator Functions // -// Functions may validate resources, returning Results as go errors. Results may contain -// different items for different validation failures. +// A function may validate resources by providing a Result. +// When using framework.Command this is done through returning a framework.Result as an error. +// WHen using framework.ResourceList this is done through setting ResourceList.Result. // // Configuring Functions // // Functions may be configured through a functionConfig (i.e. a client side custom resource), // or through flags (which the framework parses from a ConfigMap provided as input). -// Any flags registered on the cobra.Command will be parsed from the functionConfig input -// if they are defined as functionConfig.data entries. +// +// When using framework.Command, any flags registered on the cobra.Command will be parsed +// from the functionConfig input if they are defined as functionConfig.data entries. +// +// When using framework.ResourceList, any flags set on the ResourceList.Flags will be +// parsed from the functionConfig input if they are defined as functionConfig.data entries. // // Functions may also access environment variables set by the caller. // -// Function Input -// -// The framework parses the function ResourceList.items into a slice of yaml.RNodes, and -// parses the ResourceList.functionConfig into a passed in struct (optional). -// -// Building the Container +// Building a container image for the function // // The go program must be built into a container to be run as a function. The framework // can be used to generate a Dockerfile to build the function container. diff --git a/kyaml/fn/framework/example_test.go b/kyaml/fn/framework/example_test.go index e54bf3538..d791eae67 100644 --- a/kyaml/fn/framework/example_test.go +++ b/kyaml/fn/framework/example_test.go @@ -7,10 +7,82 @@ import ( "bytes" "fmt" + "github.com/spf13/pflag" "sigs.k8s.io/kustomize/kyaml/fn/framework" "sigs.k8s.io/kustomize/kyaml/yaml" ) +const service = "Service" + +// ExampleResourceList_modify implements a function that sets an annotation on each resource. +// The annotation value is configured via a flag value parsed from ResourceList.functionConfig.data +func ExampleResourceList_modify() { + // for testing purposes only -- normally read from stdin when Executing + input := bytes.NewBufferString(` +apiVersion: config.kubernetes.io/v1alpha1 +kind: ResourceList +# items are provided as nodes +items: +- apiVersion: apps/v1 + kind: Deployment + metadata: + name: foo +- apiVersion: v1 + kind: Service + metadata: + name: foo +# functionConfig is parsed into flags by framework.Command +functionConfig: + apiVersion: v1 + kind: ConfigMap + data: + value: baz +`) + + // configure the annotation value using a flag parsed from + // ResourceList.functionConfig.data.value + fs := pflag.NewFlagSet("tests", pflag.ContinueOnError) + value := fs.String("value", "", "annotation value") + rl := framework.ResourceList{ + Flags: fs, + Reader: input, // for testing only + } + if err := rl.Read(); err != nil { + panic(err) + } + for i := range rl.Items { + // set the annotation on each resource item + if err := rl.Items[i].PipeE(yaml.SetAnnotation("value", *value)); err != nil { + panic(err) + } + } + if err := rl.Write(); err != nil { + panic(err) + } + + // Output: + // apiVersion: config.kubernetes.io/v1alpha1 + // kind: ResourceList + // items: + // - apiVersion: apps/v1 + // kind: Deployment + // metadata: + // name: foo + // annotations: + // value: 'baz' + // - apiVersion: v1 + // kind: Service + // metadata: + // name: foo + // annotations: + // value: 'baz' + // functionConfig: + // apiVersion: v1 + // kind: ConfigMap + // data: + // value: baz +} + // ExampleCommand_modify implements a function that sets an annotation on each resource. // The annotation value is configured via a flag value parsed from // ResourceList.functionConfig.data @@ -102,7 +174,7 @@ func ExampleCommand_generateReplace() { // something we already generated, remove it from the list so we regenerate it if meta.Name == functionConfig.Spec.Name && - meta.Kind == "Service" && + meta.Kind == service && meta.APIVersion == "v1" { continue } @@ -163,6 +235,95 @@ functionConfig: // name: bar } +// ExampleResourceList_generateReplace generates a resource from a functionConfig. +// If the resource already exist s, it replaces the resource with a new copy. +func ExampleResourceList_generateReplace() { + input := bytes.NewBufferString(` +apiVersion: config.kubernetes.io/v1alpha1 +kind: ResourceList +# items are provided as nodes +items: +- apiVersion: apps/v1 + kind: Deployment + metadata: + name: foo +# functionConfig is parsed into flags by framework.Command +functionConfig: + apiVersion: example.com/v1alpha1 + kind: ExampleServiceGenerator + spec: + name: bar +`) + + // function API definition which will be parsed from the ResourceList.functionConfig + // read from stdin + type Spec struct { + Name string `yaml:"name,omitempty"` + } + type ExampleServiceGenerator struct { + Spec Spec `yaml:"spec,omitempty"` + } + functionConfig := &ExampleServiceGenerator{} + + rl := framework.ResourceList{ + FunctionConfig: functionConfig, + Reader: input, // for testing only + } + if err := rl.Read(); err != nil { + panic(err) + } + + // remove the last generated resource + var newNodes []*yaml.RNode + for i := range rl.Items { + meta, err := rl.Items[i].GetMeta() + if err != nil { + panic(err) + } + // something we already generated, remove it from the list so we regenerate it + if meta.Name == functionConfig.Spec.Name && + meta.Kind == service && + meta.APIVersion == "v1" { + continue + } + newNodes = append(newNodes, rl.Items[i]) + } + rl.Items = newNodes + + // generate the resource again + n, err := yaml.Parse(fmt.Sprintf(`apiVersion: v1 +kind: Service +metadata: + name: %s +`, functionConfig.Spec.Name)) + if err != nil { + panic(err) + } + rl.Items = append(rl.Items, n) + + if err := rl.Write(); err != nil { + panic(err) + } + + // Output: + // apiVersion: config.kubernetes.io/v1alpha1 + // kind: ResourceList + // items: + // - apiVersion: apps/v1 + // kind: Deployment + // metadata: + // name: foo + // - apiVersion: v1 + // kind: Service + // metadata: + // name: bar + // functionConfig: + // apiVersion: example.com/v1alpha1 + // kind: ExampleServiceGenerator + // spec: + // name: bar +} + // ExampleCommand_generateUpdate generates a resource, updating the previously generated // copy rather than replacing it. // @@ -191,7 +352,7 @@ func ExampleCommand_generateUpdate() { // something we already generated, reconcile it to make sure it matches what // is specified by the functionConfig if meta.Name == functionConfig.Spec.Name && - meta.Kind == "Service" && + meta.Kind == service && meta.APIVersion == "v1" { // set some values for k, v := range functionConfig.Spec.Annotations { @@ -281,7 +442,7 @@ functionConfig: } // ExampleCommand_validate validates that all Deployment resources have the replicas field set. -// If any Deployments do not contain spec.replicas, then the function will return Results +// If any Deployments do not contain spec.replicas, then the function will return results // which will be set on ResourceList.results func ExampleCommand_validate() { cmd := framework.Command(nil, func(nodes []*yaml.RNode) ([]*yaml.RNode, error) { @@ -320,7 +481,7 @@ func ExampleCommand_validate() { }) } - // framework will only consider Results an error if it has at least 1 item + // framework will only consider results an error if it has at least 1 item return nodes, framework.Result{ Name: "replicas-validator", Items: validationResults, diff --git a/kyaml/fn/framework/framework.go b/kyaml/fn/framework/framework.go index 6d58401f1..c89246cec 100644 --- a/kyaml/fn/framework/framework.go +++ b/kyaml/fn/framework/framework.go @@ -5,22 +5,176 @@ package framework import ( "fmt" + "io" "io/ioutil" + "os" "path/filepath" "github.com/spf13/cobra" + "github.com/spf13/pflag" "sigs.k8s.io/kustomize/kyaml/errors" "sigs.k8s.io/kustomize/kyaml/kio" "sigs.k8s.io/kustomize/kyaml/yaml" ) -// Command provides a cobra.Command for running the function. +// ResourceList reads the function input and writes the function output. // -// If functionConfig is nil, the function may be configured with flags parsed from -// the ResourceList.functionConfig by creating flags on the returned command. +// Adheres to the spec: https://github.com/kubernetes-sigs/kustomize/blob/master/cmd/config/docs/api-conventions/functions-spec.md +type ResourceList struct { + // FunctionConfig is the ResourceList.functionConfig input value. If FunctionConfig + // is set to a value such as a struct or map[string]interface{} before ResourceList.Read() + // is called, then the functionConfig will be parsed into that value. + // If it is nil, the functionConfig will be set to a map[string]interface{} + // before it is parsed. + // + // e.g. given the function input: + // + // kind: ResourceList + // functionConfig: + // kind: Example + // spec: + // foo: var + // + // FunctionConfig will contain the Example unmarshalled into its value. + FunctionConfig interface{} + + // Items is the ResourceList.items input and output value. Items will be set by + // ResourceList.Read() and written by ResourceList.Write(). + // + // e.g. given the function input: + // + // kind: ResourceList + // items: + // - kind: Deployment + // ... + // - kind: Service + // ... + // + // Items will be a slice containing the Deployment and Service resources + Items []*yaml.RNode + + // Result is ResourceList.result output value. Result will be written by + // ResourceList.Write() + Result *Result + + // Flags are an optional set of flags to parse the ResourceList.functionConfig.data. + // If non-nil, ResourceList.Read() will set the flag value for each flag name matching + // a ResourceList.functionConfig.data map entry. + // + // e.g. given the function input: + // + // kind: ResourceList + // functionConfig: + // data: + // foo: bar + // a: b + // + // The flags --a=b and --foo=bar will be set in Flags. + Flags *pflag.FlagSet + + // Reader is used to read the function input (ResourceList). + // Defaults to os.Stdin. + Reader io.Reader + + // Writer is used to write the function output (ResourceList) + // Defaults to os.Stdout. + Writer io.Writer + + // rw reads function input and writes function output + rw *kio.ByteReadWriter +} + +// Read reads the ResourceList +func (r *ResourceList) Read() error { + if r.Reader == nil { + r.Reader = os.Stdin + } + if r.Writer == nil { + r.Writer = os.Stdout + } + r.rw = &kio.ByteReadWriter{ + Reader: r.Reader, + Writer: r.Writer, + KeepReaderAnnotations: true, + } + + var err error + r.Items, err = r.rw.Read() + if err != nil { + return errors.Wrap(err) + } + + // parse the functionConfig + return func() error { + if r.rw.FunctionConfig == nil { + // no function config exists + return nil + } + if r.FunctionConfig == nil { + // set directly from r.rw + r.FunctionConfig = r.rw.FunctionConfig + } else { + // unmarshal the functionConfig into the provided value + err := yaml.Unmarshal([]byte(r.rw.FunctionConfig.MustString()), r.FunctionConfig) + if err != nil { + return errors.Wrap(err) + } + } + + // set the functionConfig values as flags so they are easy to access + if r.Flags == nil || !r.Flags.HasFlags() { + return nil + } + // flags are always set from the "data" field + data, err := r.rw.FunctionConfig.Pipe(yaml.Lookup("data")) + if err != nil || data == nil { + return err + } + return data.VisitFields(func(node *yaml.MapNode) error { + f := r.Flags.Lookup(node.Key.YNode().Value) + if f == nil { + return nil + } + return f.Value.Set(node.Value.YNode().Value) + }) + }() +} + +// Write writes the ResourceList +func (r *ResourceList) Write() error { + // set the ResourceList.results for validating functions + if r.Result != nil { + if len(r.Result.Items) > 0 { + b, err := yaml.Marshal(r.Result) + if err != nil { + return errors.Wrap(err) + } + y, err := yaml.Parse(string(b)) + if err != nil { + return errors.Wrap(err) + } + r.rw.Results = y + } + } + + // write the results + return r.rw.Write(r.Items) +} + +// Command returns a cobra.Command to run a function. +// +// The cobra.Command will use a ResourceList to Read() the input, run the provided function, +// and Write() the output. +// +// If functionConfig is non-nil, the ResourceList.functionConfig will be unmarshalled into it. +// +// The returned cobra.Command will have a "gen" subcommand which can be used to generate +// a Dockerfile to build the function into a container image +// +// go run main.go gen DIR/ func Command(functionConfig interface{}, function Function) cobra.Command { cmd := cobra.Command{} - addGenerate(&cmd) + AddGenerateDockerfile(&cmd) cmd.RunE = func(cmd *cobra.Command, args []string) error { err := execute(function, functionConfig, cmd) if err != nil { @@ -33,7 +187,9 @@ func Command(functionConfig interface{}, function Function) cobra.Command { return cmd } -func addGenerate(cmd *cobra.Command) { +// AddGenerateDockerfile adds a "gen" subcommand to create a Dockerfile for building +// the function as a container. +func AddGenerateDockerfile(cmd *cobra.Command) { gen := &cobra.Command{ Use: "gen", Args: cobra.ExactArgs(1), @@ -54,89 +210,33 @@ CMD ["function"] } func execute(function Function, functionConfig interface{}, cmd *cobra.Command) error { - rw := &kio.ByteReadWriter{ - Reader: cmd.InOrStdin(), - Writer: cmd.OutOrStdout(), - KeepReaderAnnotations: true, - } - nodes, err := rw.Read() - if err != nil { - return errors.Wrap(err) + rl := ResourceList{ + FunctionConfig: functionConfig, + Flags: cmd.Flags(), + Writer: cmd.OutOrStdout(), + Reader: cmd.InOrStdin(), } - // parse the functionConfig - if rw.FunctionConfig != nil { - if functionConfig == nil { - functionConfig = map[string]interface{}{} - } - - // unmarshal into the provided structure - err := yaml.Unmarshal([]byte(rw.FunctionConfig.MustString()), functionConfig) - if err != nil { - return errors.Wrap(err) - } - - // set the functionConfig values as flags so they are easy to access - err = func() error { - if !cmd.HasFlags() { - return nil - } - // kpt serializes function arguments as a ConfigMap, read them from - // the data field. - fc, ok := functionConfig.(map[string]interface{}) - if !ok { - // serialized as something else - return nil - } - if fc["data"] == nil { - return nil - } - data := fc["data"].(map[string]interface{}) - // set the value of each flag from the ResourceList.function config input - // values - for k, v := range data { - s, ok := v.(string) - if !ok { - continue - } - if err = cmd.Flag(k).Value.Set(s); err != nil { - return errors.Wrap(err) - } - } - return nil - }() - if err != nil { - return err - } + if err := rl.Read(); err != nil { + return err } // run the function implementation - nodes, err = function(nodes) + var err error + rl.Items, err = function(rl.Items) // set the ResourceList.results for validating functions var result *Result if err != nil { if val, ok := err.(Result); ok { - if len(val.Items) > 0 { - result = &val - b, err := yaml.Marshal(val) - if err != nil { - return errors.Wrap(err) - } - y, err := yaml.Parse(string(b)) - if err != nil { - return errors.Wrap(err) - } - rw.Results = y - } + rl.Result = &val } else { return errors.Wrap(err) } } - // write the results - if err := rw.Write(nodes); err != nil { - return errors.Wrap(err) + if err := rl.Write(); err != nil { + return err } if result != nil && result.ExitCode() != 0 { diff --git a/kyaml/fn/runtime/container/container.go b/kyaml/fn/runtime/container/container.go new file mode 100644 index 000000000..5e205f368 --- /dev/null +++ b/kyaml/fn/runtime/container/container.go @@ -0,0 +1,202 @@ +// Copyright 2019 The Kubernetes Authors. +// SPDX-License-Identifier: Apache-2.0 + +package container + +import ( + "fmt" + "os" + "strings" + + runtimeexec "sigs.k8s.io/kustomize/kyaml/fn/runtime/exec" + "sigs.k8s.io/kustomize/kyaml/fn/runtime/runtimeutil" + + "sigs.k8s.io/kustomize/kyaml/yaml" +) + +// Filter filters Resources using a container image. +// The container must start a process that reads the list of +// input Resources from stdin, reads the Configuration from the env +// API_CONFIG, and writes the filtered Resources to stdout. +// If there is a error or validation failure, the process must exit +// non-zero. +// The full set of environment variables from the parent process +// are passed to the container. +// +// Function Scoping: +// Filter applies the function only to Resources to which it is scoped. +// +// Resources are scoped to a function if any of the following are true: +// - the Resource were read from the same directory as the function config +// - the Resource were read from a subdirectory of the function config directory +// - the function config is in a directory named "functions" and +// they were read from a subdirectory of "functions" parent +// - the function config doesn't have a path annotation (considered globally scoped) +// - the Filter has GlobalScope == true +// +// In Scope Examples: +// +// Example 1: deployment.yaml and service.yaml in function.yaml scope +// same directory as the function config directory +// . +// ├── function.yaml +// ├── deployment.yaml +// └── service.yaml +// +// Example 2: apps/deployment.yaml and apps/service.yaml in function.yaml scope +// subdirectory of the function config directory +// . +// ├── function.yaml +// └── apps +//    ├── deployment.yaml +//    └── service.yaml +// +// Example 3: apps/deployment.yaml and apps/service.yaml in functions/function.yaml scope +// function config is in a directory named "functions" +// . +// ├── functions +// │   └── function.yaml +// └── apps +//    ├── deployment.yaml +//    └── service.yaml +// +// Out of Scope Examples: +// +// Example 1: apps/deployment.yaml and apps/service.yaml NOT in stuff/function.yaml scope +// . +// ├── stuff +// │   └── function.yaml +// └── apps +//    ├── deployment.yaml +//    └── service.yaml +// +// Example 2: apps/deployment.yaml and apps/service.yaml NOT in stuff/functions/function.yaml scope +// . +// ├── stuff +// │   └── functions +// │    └── function.yaml +// └── apps +//    ├── deployment.yaml +//    └── service.yaml +// +// Default Paths: +// Resources emitted by functions will have default path applied as annotations +// if none is present. +// The default path will be the function-dir/ (or parent directory in the case of "functions") +// + function-file-name/ + namespace/ + kind_name.yaml +// +// Example 1: Given a function in fn.yaml that produces a Deployment name foo and a Service named bar +// dir +// └── fn.yaml +// +// Would default newly generated Resources to: +// +// dir +// ├── fn.yaml +// └── fn +//    ├── deployment_foo.yaml +//    └── service_bar.yaml +// +// Example 2: Given a function in functions/fn.yaml that produces a Deployment name foo and a Service named bar +// dir +// └── fn.yaml +// +// Would default newly generated Resources to: +// +// dir +// ├── functions +// │   └── fn.yaml +// └── fn +//    ├── deployment_foo.yaml +//    └── service_bar.yaml +// +// Example 3: Given a function in fn.yaml that produces a Deployment name foo, namespace baz and a Service named bar namespace baz +// dir +// └── fn.yaml +// +// Would default newly generated Resources to: +// +// dir +// ├── fn.yaml +// └── fn +// └── baz +//    ├── deployment_foo.yaml +//    └── service_bar.yaml +type Filter struct { + + // Image is the container image to use to create a container. + Image string `yaml:"image,omitempty"` + + // Network is the container network to use. + Network string `yaml:"network,omitempty"` + + // StorageMounts is a list of storage options that the container will have mounted. + StorageMounts []runtimeutil.StorageMount `yaml:"mounts,omitempty"` + + Exec runtimeexec.Filter +} + +func (c Filter) String() string { + if c.Exec.DeferFailure { + return fmt.Sprintf("%s deferFailure: %v", c.Image, c.Exec.DeferFailure) + } + return c.Image +} +func (c Filter) GetExit() error { + return c.Exec.GetExit() +} + +func (c *Filter) Filter(nodes []*yaml.RNode) ([]*yaml.RNode, error) { + c.setupExec() + return c.Exec.Filter(nodes) +} + +func (c *Filter) setupExec() { + // don't init 2x + if c.Exec.Path != "" { + return + } + + path, args := c.getCommand() + c.Exec.Path = path + c.Exec.Args = args +} + +// getArgs returns the command + args to run to spawn the container +func (c *Filter) getCommand() (string, []string) { + // run the container using docker. this is simpler than using the docker + // libraries, and ensures things like auth work the same as if the container + // was run from the cli. + + network := "none" + if c.Network != "" { + network = c.Network + } + + args := []string{"run", + "--rm", // delete the container afterward + "-i", "-a", "STDIN", "-a", "STDOUT", "-a", "STDERR", // attach stdin, stdout, stderr + "--network", network, + + // added security options + "--user", "nobody", // run as nobody + "--security-opt=no-new-privileges", // don't allow the user to escalate privileges + // note: don't make fs readonly because things like heredoc rely on writing tmp files + } + + // TODO(joncwong): Allow StorageMount fields to have default values. + for _, storageMount := range c.StorageMounts { + args = append(args, "--mount", storageMount.String()) + } + + os.Setenv("LOG_TO_STDERR", "true") + os.Setenv("STRUCTURED_RESULTS", "true") + + // export the local environment vars to the container + for _, pair := range os.Environ() { + args = append(args, "-e", strings.Split(pair, "=")[0]) + } + a := append(args, c.Image) + + return "docker", a +} diff --git a/kyaml/fn/runtime/container/container_test.go b/kyaml/fn/runtime/container/container_test.go new file mode 100644 index 000000000..1c84eda6c --- /dev/null +++ b/kyaml/fn/runtime/container/container_test.go @@ -0,0 +1,203 @@ +// Copyright 2019 The Kubernetes Authors. +// SPDX-License-Identifier: Apache-2.0 + +package container + +import ( + "bytes" + "fmt" + "os" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "sigs.k8s.io/kustomize/kyaml/fn/runtime/runtimeutil" + "sigs.k8s.io/kustomize/kyaml/kio" + "sigs.k8s.io/kustomize/kyaml/yaml" +) + +func TestFilter_setupExec(t *testing.T) { + var tests = []struct { + name string + functionConfig string + expectedArgs []string + instance Filter + }{ + { + name: "command", + functionConfig: `apiVersion: apps/v1 +kind: Deployment +metadata: + name: foo +`, + expectedArgs: []string{ + "run", + "--rm", + "-i", "-a", "STDIN", "-a", "STDOUT", "-a", "STDERR", + "--network", "none", + "--user", "nobody", + "--security-opt=no-new-privileges", + }, + instance: Filter{Image: "example.com:version"}, + }, + + { + name: "network", + functionConfig: `apiVersion: apps/v1 +kind: Deployment +metadata: + name: foo +`, + expectedArgs: []string{ + "run", + "--rm", + "-i", "-a", "STDIN", "-a", "STDOUT", "-a", "STDERR", + "--network", "test-1", + "--user", "nobody", + "--security-opt=no-new-privileges", + }, + instance: Filter{Image: "example.com:version", Network: "test-1"}, + }, + + { + name: "storage_mounts", + functionConfig: `apiVersion: apps/v1 +kind: Deployment +metadata: + name: foo +`, + expectedArgs: []string{ + "run", + "--rm", + "-i", "-a", "STDIN", "-a", "STDOUT", "-a", "STDERR", + "--network", "none", + "--user", "nobody", + "--security-opt=no-new-privileges", + "--mount", fmt.Sprintf("type=%s,src=%s,dst=%s:ro", "bind", "/mount/path", "/local/"), + "--mount", fmt.Sprintf("type=%s,src=%s,dst=%s:ro", "volume", "myvol", "/local/"), + "--mount", fmt.Sprintf("type=%s,src=%s,dst=%s:ro", "tmpfs", "", "/local/"), + }, + instance: Filter{ + Image: "example.com:version", + StorageMounts: []runtimeutil.StorageMount{ + {MountType: "bind", Src: "/mount/path", DstPath: "/local/"}, + {MountType: "volume", Src: "myvol", DstPath: "/local/"}, + {MountType: "tmpfs", Src: "", DstPath: "/local/"}, + }, + }, + }, + } + + for i := range tests { + tt := tests[i] + t.Run(tt.name, func(t *testing.T) { + cfg, err := yaml.Parse(tt.functionConfig) + if !assert.NoError(t, err) { + t.FailNow() + } + tt.instance.Exec.FunctionConfig = cfg + + os.Setenv("KYAML_TEST", "FOO") + tt.instance.setupExec() + + // configure expected env + for _, e := range os.Environ() { + // the process env + tt.expectedArgs = append(tt.expectedArgs, "-e", strings.Split(e, "=")[0]) + } + tt.expectedArgs = append(tt.expectedArgs, tt.instance.Image) + + if !assert.Equal(t, "docker", tt.instance.Exec.Path) { + t.FailNow() + } + if !assert.Equal(t, tt.expectedArgs, tt.instance.Exec.Args) { + t.FailNow() + } + }) + } +} + +func TestFilter_Filter(t *testing.T) { + cfg, err := yaml.Parse(`apiVersion: apps/v1 +kind: Deployment +metadata: + name: foo +`) + if !assert.NoError(t, err) { + return + } + + input, err := (&kio.ByteReader{Reader: bytes.NewBufferString(` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: deployment-foo +--- +apiVersion: v1 +kind: Service +metadata: + name: service-foo +`)}).Read() + if !assert.NoError(t, err) { + return + } + + instance := Filter{} + instance.Exec.FunctionConfig = cfg + instance.Exec.Path = "sed" + instance.Exec.Args = []string{"s/Deployment/StatefulSet/g"} + output, err := instance.Filter(input) + if !assert.NoError(t, err) { + t.FailNow() + } + + b := &bytes.Buffer{} + err = kio.ByteWriter{Writer: b, KeepReaderAnnotations: true}.Write(output) + if !assert.NoError(t, err) { + t.FailNow() + } + + if !assert.Equal(t, `apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: deployment-foo + annotations: + config.kubernetes.io/index: '0' + config.kubernetes.io/path: 'statefulset_deployment-foo.yaml' +--- +apiVersion: v1 +kind: Service +metadata: + name: service-foo + annotations: + config.kubernetes.io/index: '1' + config.kubernetes.io/path: 'service_service-foo.yaml' +`, b.String()) { + t.FailNow() + } +} +func TestFilter_String(t *testing.T) { + instance := Filter{Image: "foo"} + if !assert.Equal(t, "foo", instance.String()) { + t.FailNow() + } + + instance.Exec.DeferFailure = true + if !assert.Equal(t, "foo deferFailure: true", instance.String()) { + t.FailNow() + } +} + +func TestFilter_ExitCode(t *testing.T) { + instance := Filter{} + instance.Exec.Path = "/not/real/command" + instance.Exec.DeferFailure = true + _, err := instance.Filter(nil) + if !assert.NoError(t, err) { + t.FailNow() + } + + if !assert.EqualError(t, instance.GetExit(), "fork/exec /not/real/command: no such file or directory") { + t.FailNow() + } +} diff --git a/kyaml/fn/runtime/exec/doc.go b/kyaml/fn/runtime/exec/doc.go new file mode 100644 index 000000000..2747a96fb --- /dev/null +++ b/kyaml/fn/runtime/exec/doc.go @@ -0,0 +1,5 @@ +// Copyright 2019 The Kubernetes Authors. +// SPDX-License-Identifier: Apache-2.0 + +// Package exec contains the exec function implementation. +package exec diff --git a/kyaml/fn/runtime/exec/exec.go b/kyaml/fn/runtime/exec/exec.go new file mode 100644 index 000000000..cf8edeaa5 --- /dev/null +++ b/kyaml/fn/runtime/exec/exec.go @@ -0,0 +1,34 @@ +// Copyright 2019 The Kubernetes Authors. +// SPDX-License-Identifier: Apache-2.0 + +package exec + +import ( + "io" + "os/exec" + + "sigs.k8s.io/kustomize/kyaml/fn/runtime/runtimeutil" + "sigs.k8s.io/kustomize/kyaml/yaml" +) + +type Filter struct { + // Path is the path to the executable to run + Path string `yaml:"path,omitempty"` + + // Args are the arguments to the executable + Args []string `yaml:"args,omitempty"` + + runtimeutil.FunctionFilter +} + +func (c *Filter) Filter(nodes []*yaml.RNode) ([]*yaml.RNode, error) { + c.FunctionFilter.Run = c.Run + return c.FunctionFilter.Filter(nodes) +} + +func (c *Filter) Run(reader io.Reader, writer io.Writer) error { + cmd := exec.Command(c.Path, c.Args...) + cmd.Stdin = reader + cmd.Stdout = writer + return cmd.Run() +} diff --git a/kyaml/fn/runtime/exec/exec_test.go b/kyaml/fn/runtime/exec/exec_test.go new file mode 100644 index 000000000..c878ed993 --- /dev/null +++ b/kyaml/fn/runtime/exec/exec_test.go @@ -0,0 +1,112 @@ +// Copyright 2019 The Kubernetes Authors. +// SPDX-License-Identifier: Apache-2.0 + +package exec_test + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "sigs.k8s.io/kustomize/kyaml/fn/runtime/exec" + "sigs.k8s.io/kustomize/kyaml/yaml" +) + +func TestFunctionFilter_Filter(t *testing.T) { + var tests = []struct { + name string + input []string + functionConfig string + expectedOutput []string + expectedError string + instance exec.Filter + }{ + { + name: "exec_sed", + input: []string{ + `apiVersion: apps/v1 +kind: Deployment +metadata: + name: deployment-foo`, + `apiVersion: v1 +kind: Service +metadata: + name: service-foo`, + }, + expectedOutput: []string{ + `apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: deployment-foo + annotations: + config.kubernetes.io/path: 'statefulset_deployment-foo.yaml' +`, + `apiVersion: v1 +kind: Service +metadata: + name: service-foo + annotations: + config.kubernetes.io/path: 'service_service-foo.yaml' +`, + }, + expectedError: "", + instance: exec.Filter{ + Path: "sed", + Args: []string{"s/Deployment/StatefulSet/g"}, + }, + }, + } + + for i := range tests { + tt := tests[i] + t.Run(tt.name, func(t *testing.T) { + // initialize the inputs for the FunctionFilter + var inputs []*yaml.RNode + for i := range tt.input { + node, err := yaml.Parse(tt.input[i]) + if !assert.NoError(t, err) { + t.FailNow() + } + inputs = append(inputs, node) + } + if tt.functionConfig != "" { + fc, err := yaml.Parse(tt.functionConfig) + if !assert.NoError(t, err) { + t.FailNow() + } + tt.instance.FunctionConfig = fc + } + + // run the function + output, err := tt.instance.Filter(inputs) + + // check for errors + if tt.expectedError != "" { + if !assert.EqualError(t, err, tt.expectedError) { + t.FailNow() + } + return + } + if !assert.NoError(t, err) { + t.FailNow() + } + + // verify the output + var actual []string + for i := range output { + s, err := output[i].String() + if !assert.NoError(t, err) { + t.FailNow() + } + actual = append(actual, strings.TrimSpace(s)) + } + var expected []string + for i := range tt.expectedOutput { + expected = append(expected, strings.TrimSpace(tt.expectedOutput[i])) + } + if !assert.Equal(t, expected, actual) { + t.FailNow() + } + }) + } +} diff --git a/kyaml/fn/runtime/runtimeutil/doc.go b/kyaml/fn/runtime/runtimeutil/doc.go new file mode 100644 index 000000000..89f9036a4 --- /dev/null +++ b/kyaml/fn/runtime/runtimeutil/doc.go @@ -0,0 +1,5 @@ +// Copyright 2019 The Kubernetes Authors. +// SPDX-License-Identifier: Apache-2.0 + +// Package runtimeutil contains libraries for implementing function runtimes. +package runtimeutil diff --git a/kyaml/kio/filters/functiontypes.go b/kyaml/fn/runtime/runtimeutil/functiontypes.go similarity index 67% rename from kyaml/kio/filters/functiontypes.go rename to kyaml/fn/runtime/runtimeutil/functiontypes.go index 212cb589c..a5a0887ee 100644 --- a/kyaml/kio/filters/functiontypes.go +++ b/kyaml/fn/runtime/runtimeutil/functiontypes.go @@ -1,9 +1,12 @@ // Copyright 2019 The Kubernetes Authors. // SPDX-License-Identifier: Apache-2.0 -package filters +package runtimeutil import ( + "fmt" + "strings" + "sigs.k8s.io/kustomize/kyaml/yaml" ) @@ -27,10 +30,17 @@ type FunctionSpec struct { // Starlark is the spec for running a function as a starlark script Starlark StarlarkSpec `json:"starlark,omitempty" yaml:"starlark,omitempty"` + // ExecSpec is the spec for running a function as an executable + Exec ExecSpec `json:"exec,omitempty" yaml:"exec,omitempty"` + // Mounts are the storage or directories to mount into the container StorageMounts []StorageMount `json:"mounts,omitempty" yaml:"mounts,omitempty"` } +type ExecSpec struct { + Path string `json:"path,omitempty" yaml:"path,omitempty"` +} + // ContainerSpec defines a spec for running a function as a container type ContainerSpec struct { // Image is the container image to run @@ -72,6 +82,10 @@ type StorageMount struct { DstPath string `json:"dst,omitempty" yaml:"dst,omitempty"` } +func (s *StorageMount) String() string { + return fmt.Sprintf("type=%s,src=%s,dst=%s:ro", s.MountType, s.Src, s.DstPath) +} + // GetFunctionSpec returns the FunctionSpec for a resource. Returns // nil if the resource does not have a FunctionSpec. // @@ -119,3 +133,52 @@ func getFunctionSpecFromAnnotation(n *yaml.RNode, meta yaml.ResourceMeta) *Funct _ = yaml.Unmarshal([]byte(s), &fs) return &fs } + +func StringToStorageMount(s string) StorageMount { + m := make(map[string]string) + options := strings.Split(s, ",") + for _, option := range options { + keyVal := strings.SplitN(option, "=", 2) + m[keyVal[0]] = keyVal[1] + } + var sm StorageMount + for key, value := range m { + switch { + case key == "type": + sm.MountType = value + case key == "src": + sm.Src = value + case key == "dst": + sm.DstPath = value + } + } + return sm +} + +// IsReconcilerFilter filters Resources based on whether or not they are Reconciler Resource. +// Resources with an apiVersion starting with '*.gcr.io', 'gcr.io' or 'docker.io' are considered +// Reconciler Resources. +type IsReconcilerFilter struct { + // ExcludeReconcilers if set to true, then Reconcilers will be excluded -- e.g. + // Resources with a reconcile container through the apiVersion (gcr.io prefix) or + // through the annotations + ExcludeReconcilers bool `yaml:"excludeReconcilers,omitempty"` + + // IncludeNonReconcilers if set to true, the NonReconciler will be included. + IncludeNonReconcilers bool `yaml:"includeNonReconcilers,omitempty"` +} + +// Filter implements kio.Filter +func (c *IsReconcilerFilter) Filter(inputs []*yaml.RNode) ([]*yaml.RNode, error) { + var out []*yaml.RNode + for i := range inputs { + isFnResource := GetFunctionSpec(inputs[i]) != nil + if isFnResource && !c.ExcludeReconcilers { + out = append(out, inputs[i]) + } + if !isFnResource && c.IncludeNonReconcilers { + out = append(out, inputs[i]) + } + } + return out, nil +} diff --git a/kyaml/fn/runtime/runtimeutil/runtimeutil.go b/kyaml/fn/runtime/runtimeutil/runtimeutil.go new file mode 100644 index 000000000..d2321bf01 --- /dev/null +++ b/kyaml/fn/runtime/runtimeutil/runtimeutil.go @@ -0,0 +1,198 @@ +// Copyright 2019 The Kubernetes Authors. +// SPDX-License-Identifier: Apache-2.0 + +package runtimeutil + +import ( + "bytes" + "io" + "io/ioutil" + "path" + "strings" + + "sigs.k8s.io/kustomize/kyaml/errors" + "sigs.k8s.io/kustomize/kyaml/kio" + "sigs.k8s.io/kustomize/kyaml/kio/kioutil" + + "sigs.k8s.io/kustomize/kyaml/yaml" +) + +// FunctionFilter wraps another filter to be invoked in the context of a function. +// FunctionFilter manages scoping the function, deferring failures, and saving results +// to files. +type FunctionFilter struct { + // Run implements the function. + Run func(reader io.Reader, writer io.Writer) error + + // FunctionConfig is passed to the function through ResourceList.functionConfig. + FunctionConfig *yaml.RNode `yaml:"functionConfig,omitempty"` + + // GlobalScope explicitly scopes the function to all input resources rather than only those + // resources scoped to it by path. + GlobalScope bool + + // ResultsFile is the file to write function ResourceList.results to. + // If unset, results will not be written. + ResultsFile string + + // DeferFailure will cause the Filter to return a nil error even if Run returns an error. + // The Run error will be available through GetExit(). + DeferFailure bool + + // results saves the results emitted from Run + results *yaml.RNode + + // exit saves the error returned from Run + exit error +} + +// GetExit returns the error from Run +func (c FunctionFilter) GetExit() error { + return c.exit +} + +// functionsDirectoryName is keyword directory name for functions scoped 1 directory higher +const functionsDirectoryName = "functions" + +// getFunctionScope returns the path of the directory containing the function config, +// or its parent directory if the base directory is named "functions" +func (c *FunctionFilter) getFunctionScope() (string, error) { + m, err := c.FunctionConfig.GetMeta() + if err != nil { + return "", errors.Wrap(err) + } + p, found := m.Annotations[kioutil.PathAnnotation] + if !found { + return "", nil + } + + functionDir := path.Clean(path.Dir(p)) + + if path.Base(functionDir) == functionsDirectoryName { + // the scope of functions in a directory called "functions" is 1 level higher + // this is similar to how the golang "internal" directory scoping works + functionDir = path.Dir(functionDir) + } + return functionDir, nil +} + +// scope partitions the input nodes into 2 slices. The first slice contains only Resources +// which are scoped under dir, and the second slice contains the Resources which are not. +func (c *FunctionFilter) scope(dir string, nodes []*yaml.RNode) ([]*yaml.RNode, []*yaml.RNode, error) { + // scope container filtered Resources to Resources under that directory + var input, saved []*yaml.RNode + if c.GlobalScope { + return nodes, nil, nil + } + + // global function + if dir == "" || dir == "." { + return nodes, nil, nil + } + + // identify Resources read from directories under the function configuration + for i := range nodes { + m, err := nodes[i].GetMeta() + if err != nil { + return nil, nil, err + } + p, found := m.Annotations[kioutil.PathAnnotation] + if !found { + // this Resource isn't scoped under the function -- don't know where it came from + // consider it out of scope + saved = append(saved, nodes[i]) + continue + } + + resourceDir := path.Clean(path.Dir(p)) + if path.Base(resourceDir) == functionsDirectoryName { + // Functions in the `functions` directory are scoped to + // themselves, and should see themselves as input + resourceDir = path.Dir(resourceDir) + } + if !strings.HasPrefix(resourceDir, dir) { + // this Resource doesn't fall under the function scope if it + // isn't in a subdirectory of where the function lives + saved = append(saved, nodes[i]) + continue + } + + // this input is scoped under the function + input = append(input, nodes[i]) + } + + return input, saved, nil +} + +func (c *FunctionFilter) Filter(nodes []*yaml.RNode) ([]*yaml.RNode, error) { + in := &bytes.Buffer{} + out := &bytes.Buffer{} + + // only process Resources scoped to this function, save the others + functionDir, err := c.getFunctionScope() + if err != nil { + return nil, err + } + input, saved, err := c.scope(functionDir, nodes) + if err != nil { + return nil, err + } + + // write the input + err = kio.ByteWriter{ + WrappingAPIVersion: kio.ResourceListAPIVersion, + WrappingKind: kio.ResourceListKind, + Writer: in, + KeepReaderAnnotations: true, + FunctionConfig: c.FunctionConfig}.Write(input) + if err != nil { + return nil, err + } + + // capture the command stdout for the return value + r := &kio.ByteReader{Reader: out} + + // don't exit immediately if the function fails -- write out the validation + c.exit = c.Run(in, out) + + output, err := r.Read() + if err != nil { + return nil, err + } + + if err := c.doResults(r); err != nil { + return nil, err + } + + if c.exit != nil && !c.DeferFailure { + return append(output, saved...), c.exit + } + + // annotate any generated Resources with a path and index if they don't already have one + if err := kioutil.DefaultPathAnnotation(functionDir, output); err != nil { + return nil, err + } + + // emit both the Resources output from the function, and the out-of-scope Resources + // which were not provided to the function + return append(output, saved...), nil +} + +func (c *FunctionFilter) doResults(r *kio.ByteReader) error { + // Write the results to a file if configured to do so + if c.ResultsFile != "" && r.Results != nil { + results, err := r.Results.String() + if err != nil { + return err + } + err = ioutil.WriteFile(c.ResultsFile, []byte(results), 0600) + if err != nil { + return err + } + } + + if r.Results != nil { + c.results = r.Results + } + return nil +} diff --git a/kyaml/fn/runtime/runtimeutil/runtimeutil_test.go b/kyaml/fn/runtime/runtimeutil/runtimeutil_test.go new file mode 100644 index 000000000..883264d89 --- /dev/null +++ b/kyaml/fn/runtime/runtimeutil/runtimeutil_test.go @@ -0,0 +1,1267 @@ +// Copyright 2019 The Kubernetes Authors. +// SPDX-License-Identifier: Apache-2.0 + +package runtimeutil + +import ( + "fmt" + "io" + "io/ioutil" + "os" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "sigs.k8s.io/kustomize/kyaml/yaml" +) + +type testRun struct { + err error + expectedInput string + output string + t *testing.T +} + +func (r testRun) run(reader io.Reader, writer io.Writer) error { + if r.expectedInput != "" { + input, err := ioutil.ReadAll(reader) + if !assert.NoError(r.t, err) { + r.t.FailNow() + } + + // verify input matches expected + if !assert.Equal(r.t, r.expectedInput, string(input)) { + r.t.FailNow() + } + } + + _, err := writer.Write([]byte(r.output)) + if !assert.NoError(r.t, err) { + r.t.FailNow() + } + + return r.err +} + +func TestFunctionFilter_Filter(t *testing.T) { + var tests = []struct { + run testRun + name string + input []string + functionConfig string + expectedOutput []string + expectedError string + expectedSavedError string + expectedResults string + noMakeResultsFile bool + instance FunctionFilter + }{ + // verify that resources emitted from the function have a file path defaulted + // if none already exists + { + name: "default_file_path_annotation", + run: testRun{ + output: ` +apiVersion: config.kubernetes.io/v1alpha1 +kind: ResourceList +items: +- apiVersion: apps/v1 + kind: Deployment + metadata: + name: deployment-foo +- apiVersion: v1 + kind: Service + metadata: + name: service-foo +`, + }, + expectedOutput: []string{ + ` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: deployment-foo + annotations: + config.kubernetes.io/path: 'deployment_deployment-foo.yaml' +`, + ` +apiVersion: v1 +kind: Service +metadata: + name: service-foo + annotations: + config.kubernetes.io/path: 'service_service-foo.yaml' +`, + }, + }, + + // verify that resources emitted from the function do not have a file path defaulted + // if one already exists + { + name: "no_default_file_path_annotation", + run: testRun{ + output: ` +apiVersion: config.kubernetes.io/v1alpha1 +kind: ResourceList +items: +- apiVersion: apps/v1 + kind: Deployment + metadata: + name: deployment-foo +- apiVersion: v1 + kind: Service + metadata: + name: service-foo + annotations: + config.kubernetes.io/path: 'foo.yaml' +`, + }, + expectedOutput: []string{ + ` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: deployment-foo + annotations: + config.kubernetes.io/path: 'deployment_deployment-foo.yaml' +`, + ` +apiVersion: v1 +kind: Service +metadata: + name: service-foo + annotations: + config.kubernetes.io/path: 'foo.yaml' +`, + }, + }, + + // verify the FunctionFilter correctly writes the inputs and reads the outputs + // of Run + { + name: "write_read", + run: testRun{ + output: ` +apiVersion: config.kubernetes.io/v1alpha1 +kind: ResourceList +items: +- apiVersion: v1 + kind: Service + metadata: + name: service-foo + annotations: + config.kubernetes.io/path: 'foo.yaml' +- apiVersion: v1 + kind: ConfigMap + metadata: + name: configmap-foo + annotations: + config.kubernetes.io/path: 'foo.yaml' +`, + }, + input: []string{ + ` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: deployment-foo +`, + ` +apiVersion: v1 +kind: Service +metadata: + name: service-foo +`, + }, + expectedOutput: []string{` +apiVersion: v1 +kind: Service +metadata: + name: service-foo + annotations: + config.kubernetes.io/path: 'foo.yaml' +`, + ` +apiVersion: v1 +kind: ConfigMap +metadata: + name: configmap-foo + annotations: + config.kubernetes.io/path: 'foo.yaml' +`, + }, + }, + + // verify that the results file is written + // + { + name: "write_results_file", + run: testRun{ + output: `apiVersion: config.kubernetes.io/v1alpha1 +kind: ResourceList +items: +- apiVersion: apps/v1 + kind: Deployment + metadata: + name: deployment-foo +- apiVersion: v1 + kind: Service + metadata: + name: service-foo +results: +- apiVersion: config.k8s.io/v1alpha1 + kind: ObjectError + name: "some-validator" + items: + - type: error + message: "some message" + resourceRef: + apiVersion: apps/v1 + kind: Deployment + name: foo + namespace: bar + file: + path: deploy.yaml + index: 0 + field: + path: "spec.template.spec.containers[3].resources.limits.cpu" + currentValue: "200" + suggestedValue: "2" +`, + }, + expectedOutput: []string{` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: deployment-foo + annotations: + config.kubernetes.io/path: 'deployment_deployment-foo.yaml' +`, ` +apiVersion: v1 +kind: Service +metadata: + name: service-foo + annotations: + config.kubernetes.io/path: 'service_service-foo.yaml' +`, + }, + expectedResults: ` +- apiVersion: config.k8s.io/v1alpha1 + kind: ObjectError + name: "some-validator" + items: + - type: error + message: "some message" + resourceRef: + apiVersion: apps/v1 + kind: Deployment + name: foo + namespace: bar + file: + path: deploy.yaml + index: 0 + field: + path: "spec.template.spec.containers[3].resources.limits.cpu" + currentValue: "200" + suggestedValue: "2" +`, + }, + + // verify that the results file is written for functions that exist non-0 + // and the FunctionFilter returns the error + { + name: "write_results_file_function_exit_non_0", + expectedError: "failed", + run: testRun{ + err: fmt.Errorf("failed"), + output: ` +apiVersion: config.kubernetes.io/v1alpha1 +kind: ResourceList +items: +- apiVersion: apps/v1 + kind: Deployment + metadata: + name: deployment-foo +- apiVersion: v1 + kind: Service + metadata: + name: service-foo +results: +- apiVersion: config.k8s.io/v1alpha1 + kind: ObjectError + name: "some-validator" + items: + - type: error + message: "some message" + resourceRef: + apiVersion: apps/v1 + kind: Deployment + name: foo + namespace: bar + file: + path: deploy.yaml + index: 0 + field: + path: "spec.template.spec.containers[3].resources.limits.cpu" + currentValue: "200" + suggestedValue: "2" +`, + }, + expectedOutput: []string{ + ` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: deployment-foo + annotations: + config.kubernetes.io/path: 'deployment_deployment-foo.yaml' + `, ` +apiVersion: v1 +kind: Service +metadata: + name: service-foo + annotations: + config.kubernetes.io/path: 'service_service-foo.yaml' + `, + }, + expectedResults: ` +- apiVersion: config.k8s.io/v1alpha1 + kind: ObjectError + name: "some-validator" + items: + - type: error + message: "some message" + resourceRef: + apiVersion: apps/v1 + kind: Deployment + name: foo + namespace: bar + file: + path: deploy.yaml + index: 0 + field: + path: "spec.template.spec.containers[3].resources.limits.cpu" + currentValue: "200" + suggestedValue: "2" + `, + }, + + // verify that if deferFailure is set, the results file is written and the + // exit error is saved, but the FunctionFilter does not return an error. + { + name: "write_results_defer_failure", + instance: FunctionFilter{DeferFailure: true}, + expectedSavedError: "failed", + run: testRun{ + err: fmt.Errorf("failed"), + output: ` +apiVersion: config.kubernetes.io/v1alpha1 +kind: ResourceList +items: +- apiVersion: apps/v1 + kind: Deployment + metadata: + name: deployment-foo +- apiVersion: v1 + kind: Service + metadata: + name: service-foo +results: +- apiVersion: config.k8s.io/v1alpha1 + kind: ObjectError + name: "some-validator" + items: + - type: error + message: "some message" + resourceRef: + apiVersion: apps/v1 + kind: Deployment + name: foo + namespace: bar + file: + path: deploy.yaml + index: 0 + field: + path: "spec.template.spec.containers[3].resources.limits.cpu" + currentValue: "200" + suggestedValue: "2"`, + }, + expectedOutput: []string{ + ` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: deployment-foo + annotations: + config.kubernetes.io/path: 'deployment_deployment-foo.yaml' + `, ` +apiVersion: v1 +kind: Service +metadata: + name: service-foo + annotations: + config.kubernetes.io/path: 'service_service-foo.yaml' + `, + }, + expectedResults: ` +- apiVersion: config.k8s.io/v1alpha1 + kind: ObjectError + name: "some-validator" + items: + - type: error + message: "some message" + resourceRef: + apiVersion: apps/v1 + kind: Deployment + name: foo + namespace: bar + file: + path: deploy.yaml + index: 0 + field: + path: "spec.template.spec.containers[3].resources.limits.cpu" + currentValue: "200" + suggestedValue: "2"`, + }, + + { + name: "write_results_bad_results_file", + expectedError: "open /not/real/file: no such file or directory", + noMakeResultsFile: true, + run: testRun{ + output: ` +apiVersion: config.kubernetes.io/v1alpha1 +kind: ResourceList +items: +- apiVersion: apps/v1 + kind: Deployment + metadata: + name: deployment-foo +- apiVersion: v1 + kind: Service + metadata: + name: service-foo +results: +- apiVersion: config.k8s.io/v1alpha1 + name: "some-validator" +`, + }, + expectedOutput: []string{ + ` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: deployment-foo + annotations: + config.kubernetes.io/path: 'deployment_deployment-foo.yaml' + `, ` +apiVersion: v1 +kind: Service +metadata: + name: service-foo + annotations: + config.kubernetes.io/path: 'service_service-foo.yaml' + `, + }, + // these aren't written, expect an error + expectedResults: ` +- apiVersion: config.k8s.io/v1alpha1 + name: "some-validator" +`, + }, + + // verify the function only sees resources scoped to it based on the directory + // containing the functionConfig and the directory containing each resource. + // resources not provided to the function should still appear in the FunctionFilter + // output + { + name: "scope_resources_by_directory", + run: testRun{ + expectedInput: `apiVersion: config.kubernetes.io/v1alpha1 +kind: ResourceList +items: +- apiVersion: v1 + kind: Service + metadata: + name: service-foo + annotations: + config.kubernetes.io/path: 'foo/bar/s.yaml' +functionConfig: + apiVersion: example.com/v1 + kind: Example + metadata: + name: foo + annotations: + config.kubernetes.io/path: 'foo/bar.yaml' +`, + output: `apiVersion: config.kubernetes.io/v1alpha1 +kind: ResourceList +items: +- apiVersion: v1 + kind: Service + metadata: + name: service-foo + annotations: + config.kubernetes.io/path: 'foo/bar/s.yaml' + new: annotation +functionConfig: + apiVersion: example.com/v1 + kind: Example + metadata: + name: foo + annotations: + config.kubernetes.io/path: 'foo/bar.yaml' +`, + }, + functionConfig: ` +apiVersion: example.com/v1 +kind: Example +metadata: + name: foo + annotations: + config.kubernetes.io/path: 'foo/bar.yaml' +`, + input: []string{ + // this should not be in scope + ` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: deployment-foo + annotations: + config.kubernetes.io/path: 'baz/bar/d.yaml' +`, + // this should be in scope + ` +apiVersion: v1 +kind: Service +metadata: + name: service-foo + annotations: + config.kubernetes.io/path: 'foo/bar/s.yaml' +`}, + expectedOutput: []string{ + ` +apiVersion: v1 +kind: Service +metadata: + name: service-foo + annotations: + config.kubernetes.io/path: 'foo/bar/s.yaml' + new: annotation +`, ` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: deployment-foo + annotations: + config.kubernetes.io/path: 'baz/bar/d.yaml' +`, + }, + }, + + // verify functions without file path annotation are not scoped to functions + { + name: "scope_resources_by_directory_resources_missing_path", + run: testRun{ + expectedInput: `apiVersion: config.kubernetes.io/v1alpha1 +kind: ResourceList +items: +- apiVersion: v1 + kind: Service + metadata: + name: service-foo + annotations: + config.kubernetes.io/path: 'foo/bar/s.yaml' +functionConfig: + apiVersion: example.com/v1 + kind: Example + metadata: + name: foo + annotations: + config.kubernetes.io/path: 'foo/bar.yaml' +`, + output: `apiVersion: config.kubernetes.io/v1alpha1 +kind: ResourceList +items: +- apiVersion: v1 + kind: Service + metadata: + name: service-foo + annotations: + config.kubernetes.io/path: 'foo/bar/s.yaml' + new: annotation +functionConfig: + apiVersion: example.com/v1 + kind: Example + metadata: + name: foo + annotations: + config.kubernetes.io/path: 'foo/bar.yaml' +`, + }, + functionConfig: ` +apiVersion: example.com/v1 +kind: Example +metadata: + name: foo + annotations: + config.kubernetes.io/path: 'foo/bar.yaml' +`, + input: []string{ + // this should not be in scope + ` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: deployment-foo +`, + // this should be in scope + ` +apiVersion: v1 +kind: Service +metadata: + name: service-foo + annotations: + config.kubernetes.io/path: 'foo/bar/s.yaml' +`}, + expectedOutput: []string{ + ` +apiVersion: v1 +kind: Service +metadata: + name: service-foo + annotations: + config.kubernetes.io/path: 'foo/bar/s.yaml' + new: annotation +`, ` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: deployment-foo +`, + }, + }, + + // verify the functions can see all resources if global scope is set + { + name: "scope_resources_global", + instance: FunctionFilter{GlobalScope: true}, + run: testRun{ + expectedInput: `apiVersion: config.kubernetes.io/v1alpha1 +kind: ResourceList +items: +- apiVersion: apps/v1 + kind: Deployment + metadata: + name: deployment-foo + annotations: + config.kubernetes.io/path: 'baz/bar/d.yaml' +- apiVersion: v1 + kind: Service + metadata: + name: service-foo + annotations: + config.kubernetes.io/path: 'foo/bar/s.yaml' +functionConfig: + apiVersion: example.com/v1 + kind: Example + metadata: + name: foo + annotations: + config.kubernetes.io/path: 'foo/bar.yaml' +`, + output: `apiVersion: config.kubernetes.io/v1alpha1 +kind: ResourceList +items: +- apiVersion: apps/v1 + kind: Deployment + metadata: + name: deployment-foo + annotations: + config.kubernetes.io/path: 'baz/bar/d.yaml' +- apiVersion: v1 + kind: Service + metadata: + name: service-foo + annotations: + config.kubernetes.io/path: 'foo/bar/s.yaml' + new: annotation +functionConfig: + apiVersion: example.com/v1 + kind: Example + metadata: + name: foo + annotations: + config.kubernetes.io/path: 'foo/bar.yaml' +`, + }, + functionConfig: ` +apiVersion: example.com/v1 +kind: Example +metadata: + name: foo + annotations: + config.kubernetes.io/path: 'foo/bar.yaml' +`, + input: []string{ + ` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: deployment-foo + annotations: + config.kubernetes.io/path: 'baz/bar/d.yaml' +`, + ` +apiVersion: v1 +kind: Service +metadata: + name: service-foo + annotations: + config.kubernetes.io/path: 'foo/bar/s.yaml' +`}, + expectedOutput: []string{ + ` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: deployment-foo + annotations: + config.kubernetes.io/path: 'baz/bar/d.yaml' +`, ` +apiVersion: v1 +kind: Service +metadata: + name: service-foo + annotations: + config.kubernetes.io/path: 'foo/bar/s.yaml' + new: annotation +`, + }, + }, + + { + name: "scope_no_resources", + run: testRun{ + expectedInput: `apiVersion: config.kubernetes.io/v1alpha1 +kind: ResourceList +items: [] +functionConfig: + apiVersion: example.com/v1 + kind: Example + metadata: + name: foo + annotations: + config.kubernetes.io/path: 'foo/bar.yaml' +`, + output: `apiVersion: config.kubernetes.io/v1alpha1 +kind: ResourceList +items: [] +functionConfig: + apiVersion: example.com/v1 + kind: Example + metadata: + name: foo + annotations: + config.kubernetes.io/path: 'foo/bar.yaml' +`, + }, + functionConfig: ` +apiVersion: example.com/v1 +kind: Example +metadata: + name: foo + annotations: + config.kubernetes.io/path: 'foo/bar.yaml' +`, + input: []string{ + // these should not be in scope + ` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: deployment-foo + annotations: + config.kubernetes.io/path: 'baz/bar/d.yaml' +`, ` +apiVersion: v1 +kind: Service +metadata: + name: service-foo + annotations: + config.kubernetes.io/path: 'biz/bar/s.yaml' +`}, + expectedOutput: []string{ + ` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: deployment-foo + annotations: + config.kubernetes.io/path: 'baz/bar/d.yaml' +`, ` +apiVersion: v1 +kind: Service +metadata: + name: service-foo + annotations: + config.kubernetes.io/path: 'biz/bar/s.yaml' +`, + }, + }, + + { + name: "scope_functions_dir", + run: testRun{ + expectedInput: `apiVersion: config.kubernetes.io/v1alpha1 +kind: ResourceList +items: +- apiVersion: v1 + kind: Service + metadata: + name: service-foo + annotations: + config.kubernetes.io/path: 'foo/bar/s.yaml' +functionConfig: + apiVersion: example.com/v1 + kind: Example + metadata: + name: foo + annotations: + config.kubernetes.io/path: 'foo/functions/bar.yaml' +`, + output: `apiVersion: config.kubernetes.io/v1alpha1 +kind: ResourceList +items: +- apiVersion: v1 + kind: Service + metadata: + name: service-foo + annotations: + config.kubernetes.io/path: 'foo/bar/s.yaml' + new: annotation +functionConfig: + apiVersion: example.com/v1 + kind: Example + metadata: + name: foo + annotations: + config.kubernetes.io/path: 'foo/functions/bar.yaml' +`, + }, + functionConfig: ` +apiVersion: example.com/v1 +kind: Example +metadata: + name: foo + annotations: + config.kubernetes.io/path: 'foo/functions/bar.yaml' +`, + input: []string{ + // this should not be in scope + ` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: deployment-foo + annotations: + config.kubernetes.io/path: 'baz/bar/d.yaml' +`, + // this should be in scope + ` +apiVersion: v1 +kind: Service +metadata: + name: service-foo + annotations: + config.kubernetes.io/path: 'foo/bar/s.yaml' +`}, + expectedOutput: []string{ + ` +apiVersion: v1 +kind: Service +metadata: + name: service-foo + annotations: + config.kubernetes.io/path: 'foo/bar/s.yaml' + new: annotation +`, ` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: deployment-foo + annotations: + config.kubernetes.io/path: 'baz/bar/d.yaml' +`, + }, + }, + } + + for i := range tests { + tt := tests[i] + t.Run(tt.name, func(t *testing.T) { + // results file setup + if len(tt.expectedResults) > 0 && !tt.noMakeResultsFile { + // expect result files to be written -- create a directory for them + f, err := ioutil.TempFile("", "test-kyaml-*.yaml") + if !assert.NoError(t, err) { + t.FailNow() + } + defer os.RemoveAll(f.Name()) + tt.instance.ResultsFile = f.Name() + } else if len(tt.expectedResults) > 0 { + // failure case for writing to bad results location + tt.instance.ResultsFile = "/not/real/file" + } + + // initialize the inputs for the FunctionFilter + var inputs []*yaml.RNode + for i := range tt.input { + node, err := yaml.Parse(tt.input[i]) + if !assert.NoError(t, err) { + t.FailNow() + } + inputs = append(inputs, node) + } + + // run the FunctionFilter + tt.run.t = t + tt.instance.Run = tt.run.run + if tt.functionConfig != "" { + fc, err := yaml.Parse(tt.functionConfig) + if !assert.NoError(t, err) { + t.FailNow() + } + tt.instance.FunctionConfig = fc + } + output, err := tt.instance.Filter(inputs) + if tt.expectedError != "" { + if !assert.EqualError(t, err, tt.expectedError) { + t.FailNow() + } + return + } + + // check for saved error + if tt.expectedSavedError != "" { + if !assert.EqualError(t, tt.instance.exit, tt.expectedSavedError) { + t.FailNow() + } + } + + if !assert.NoError(t, err) { + t.FailNow() + } + + // verify function output + var actual []string + for i := range output { + s, err := output[i].String() + if !assert.NoError(t, err) { + t.FailNow() + } + actual = append(actual, strings.TrimSpace(s)) + } + var expected []string + for i := range tt.expectedOutput { + expected = append(expected, strings.TrimSpace(tt.expectedOutput[i])) + } + + if !assert.Equal(t, expected, actual) { + t.FailNow() + } + + // verify results files + if len(tt.instance.ResultsFile) > 0 { + tt.expectedResults = strings.TrimSpace(tt.expectedResults) + + results, err := tt.instance.results.String() + if !assert.NoError(t, err) { + t.FailNow() + } + if !assert.Equal(t, tt.expectedResults, strings.TrimSpace(results)) { + t.FailNow() + } + + b, err := ioutil.ReadFile(tt.instance.ResultsFile) + writtenResults := strings.TrimSpace(string(b)) + if !assert.NoError(t, err) { + t.FailNow() + } + if !assert.Equal(t, tt.expectedResults, writtenResults) { + t.FailNow() + } + } + }) + } +} + +func Test_GetFunction(t *testing.T) { + var tests = []struct { + name string + resource string + expectedFn string + missingFn bool + }{ + + // fn annotation + { + name: "fn annotation", + resource: ` +apiVersion: v1beta1 +kind: Example +metadata: + annotations: + config.kubernetes.io/function: |- + container: + image: foo:v1.0.0 +`, + expectedFn: ` +container: + image: foo:v1.0.0`, + }, + + { + name: "storage mounts json style", + resource: ` +apiVersion: v1beta1 +kind: Example +metadata: + annotations: + config.kubernetes.io/function: |- + container: + image: foo:v1.0.0 + mounts: [ {type: bind, src: /mount/path, dst: /local/}, {src: myvol, dst: /local/, type: volume}, {dst: /local/, type: tmpfs} ] +`, + expectedFn: ` +container: + image: foo:v1.0.0 + mounts: + - type: bind + src: /mount/path + dst: /local/ + - type: volume + src: myvol + dst: /local/ + - type: tmpfs + dst: /local/ +`, + }, + + { + name: "storage mounts yaml style", + resource: ` +apiVersion: v1beta1 +kind: Example +metadata: + annotations: + config.kubernetes.io/function: |- + container: + image: foo:v1.0.0 + mounts: + - src: /mount/path + type: bind + dst: /local/ + - dst: /local/ + src: myvol + type: volume + - type: tmpfs + dst: /local/ +`, + expectedFn: ` +container: + image: foo:v1.0.0 + mounts: + - type: bind + src: /mount/path + dst: /local/ + - type: volume + src: myvol + dst: /local/ + - type: tmpfs + dst: /local/ +`, + }, + + { + name: "network", + resource: ` +apiVersion: v1beta1 +kind: Example +metadata: + annotations: + config.kubernetes.io/function: |- + container: + image: foo:v1.0.0 + network: + required: true +`, + expectedFn: ` +container: + image: foo:v1.0.0 + network: + required: true +`, + }, + + { + name: "path", + resource: ` +apiVersion: v1beta1 +kind: Example +metadata: + annotations: + config.kubernetes.io/function: |- + path: foo + container: + image: foo:v1.0.0 +`, + // path should be erased + expectedFn: ` +container: + image: foo:v1.0.0 +`, + }, + + { + name: "network", + resource: ` +apiVersion: v1beta1 +kind: Example +metadata: + annotations: + config.kubernetes.io/function: |- + network: foo + container: + image: foo:v1.0.0 +`, + // network should be erased + expectedFn: ` +container: + image: foo:v1.0.0 +`, + }, + + // legacy fn style + {name: "legacy fn meta", + resource: ` +apiVersion: v1beta1 +kind: Example +metadata: + configFn: + container: + image: foo:v1.0.0 +`, + expectedFn: ` +container: + image: foo:v1.0.0 +`, + }, + + // no fn + {name: "no fn", + resource: ` +apiVersion: v1beta1 +kind: Example +metadata: + annotations: {} +`, + missingFn: true, + }, + + // test network, etc... + } + + for i := range tests { + tt := tests[i] + t.Run(tt.name, func(t *testing.T) { + resource := yaml.MustParse(tt.resource) + fn := GetFunctionSpec(resource) + if tt.missingFn { + if !assert.Nil(t, fn) { + t.FailNow() + } + } else { + b, err := yaml.Marshal(fn) + if !assert.NoError(t, err) { + t.FailNow() + } + if !assert.Equal(t, + strings.TrimSpace(tt.expectedFn), + strings.TrimSpace(string(b))) { + t.FailNow() + } + } + }) + } +} + +func Test_GetContainerNetworkRequired(t *testing.T) { + tests := []struct { + input string + required bool + }{ + { + input: `apiVersion: v1 +kind: Foo +metadata: + name: foo + configFn: + container: + image: gcr.io/kustomize-functions/example-tshirt:v0.1.0 + network: + required: true +`, + required: true, + }, + { + input: `apiVersion: v1 +kind: Foo +metadata: + name: foo + configFn: + container: + image: gcr.io/kustomize-functions/example-tshirt:v0.1.0 + network: + required: false +`, + required: false, + }, + { + + input: `apiVersion: v1 +kind: Foo +metadata: + name: foo + configFn: + container: + image: gcr.io/kustomize-functions/example-tshirt:v0.1.0 +`, + required: false, + }, + { + input: `apiVersion: v1 +kind: Foo +metadata: + name: foo + annotations: + config.kubernetes.io/function: | + container: + image: gcr.io/kustomize-functions/example-tshirt:v0.1.0 + network: + required: true +`, + required: true, + }, + } + + for _, tc := range tests { + cfg, err := yaml.Parse(tc.input) + if !assert.NoError(t, err) { + return + } + fn := GetFunctionSpec(cfg) + assert.Equal(t, tc.required, fn.Container.Network.Required) + } +} diff --git a/kyaml/fn/runtime/runtimeutil/types.go b/kyaml/fn/runtime/runtimeutil/types.go new file mode 100644 index 000000000..5edc4ebc3 --- /dev/null +++ b/kyaml/fn/runtime/runtimeutil/types.go @@ -0,0 +1,8 @@ +// Copyright 2019 The Kubernetes Authors. +// SPDX-License-Identifier: Apache-2.0 + +package runtimeutil + +type DeferFailureFunction interface { + GetExit() error +} diff --git a/kyaml/starlark/context.go b/kyaml/fn/runtime/starlark/context.go similarity index 80% rename from kyaml/starlark/context.go rename to kyaml/fn/runtime/starlark/context.go index c13eae8f5..d72f5940c 100644 --- a/kyaml/starlark/context.go +++ b/kyaml/fn/runtime/starlark/context.go @@ -60,22 +60,6 @@ func env() (starlark.Value, error) { return value, nil } -func nodeToValue(node *yaml.RNode) (starlark.Value, error) { - s, err := node.String() - if err != nil { - return nil, errors.Wrap(err) - } - var in map[string]interface{} - if err := yaml.Unmarshal([]byte(s), &in); err != nil { - return nil, errors.Wrap(err) - } - value, err := util.Marshal(in) - if err != nil { - return nil, errors.Wrap(err) - } - return value, nil -} - func interfaceToValue(i interface{}) (starlark.Value, error) { b, err := json.Marshal(i) if err != nil { diff --git a/kyaml/starlark/doc.go b/kyaml/fn/runtime/starlark/doc.go similarity index 100% rename from kyaml/starlark/doc.go rename to kyaml/fn/runtime/starlark/doc.go diff --git a/kyaml/starlark/example_test.go b/kyaml/fn/runtime/starlark/example_test.go similarity index 93% rename from kyaml/starlark/example_test.go rename to kyaml/fn/runtime/starlark/example_test.go index 3c67bd6b3..5c4b3c1f7 100644 --- a/kyaml/starlark/example_test.go +++ b/kyaml/fn/runtime/starlark/example_test.go @@ -11,8 +11,9 @@ import ( "os" "path/filepath" + "sigs.k8s.io/kustomize/kyaml/fn/runtime/runtimeutil" + "sigs.k8s.io/kustomize/kyaml/fn/runtime/starlark" "sigs.k8s.io/kustomize/kyaml/kio" - "sigs.k8s.io/kustomize/kyaml/starlark" "sigs.k8s.io/kustomize/kyaml/yaml" ) @@ -75,6 +76,7 @@ run(ctx.resource_list["items"]) // name: deployment-1 // annotations: // foo: bar + // config.kubernetes.io/path: 'deployment_deployment-1.yaml' // spec: // template: // spec: @@ -88,6 +90,7 @@ run(ctx.resource_list["items"]) // name: deployment-2 // annotations: // foo: bar + // config.kubernetes.io/path: 'deployment_deployment-2.yaml' // spec: // template: // spec: @@ -141,7 +144,7 @@ def run(items, value): run(ctx.resource_list["items"], ctx.resource_list["functionConfig"]["spec"]["value"]) `, - FunctionConfig: fc, + FunctionFilter: runtimeutil.FunctionFilter{FunctionConfig: fc}, } // output contains the transformed resources @@ -165,6 +168,7 @@ run(ctx.resource_list["items"], ctx.resource_list["functionConfig"]["spec"]["val // name: deployment-1 // annotations: // foo: hello world + // config.kubernetes.io/path: 'deployment_deployment-1.yaml' // spec: // template: // spec: @@ -178,6 +182,7 @@ run(ctx.resource_list["items"], ctx.resource_list["functionConfig"]["spec"]["val // name: deployment-2 // annotations: // foo: hello world + // config.kubernetes.io/path: 'deployment_deployment-2.yaml' // spec: // template: // spec: diff --git a/kyaml/fn/runtime/starlark/starlark.go b/kyaml/fn/runtime/starlark/starlark.go new file mode 100644 index 000000000..1d2bd37e2 --- /dev/null +++ b/kyaml/fn/runtime/starlark/starlark.go @@ -0,0 +1,224 @@ +// Copyright 2019 The Kubernetes Authors. +// SPDX-License-Identifier: Apache-2.0 + +package starlark + +import ( + "bytes" + "fmt" + "io" + "io/ioutil" + "net/http" + + "github.com/qri-io/starlib/util" + "go.starlark.net/starlark" + "sigs.k8s.io/kustomize/kyaml/comments" + "sigs.k8s.io/kustomize/kyaml/errors" + "sigs.k8s.io/kustomize/kyaml/fn/runtime/runtimeutil" + "sigs.k8s.io/kustomize/kyaml/kio/filters" + "sigs.k8s.io/kustomize/kyaml/yaml" +) + +// Filter transforms a set of resources through the provided program +type Filter struct { + Name string + + // Program is a starlark script which will be run against the resources + Program string + + // URL is the url of a starlark program to fetch and run + URL string + + // Path is the path to a starlark program to read and run + Path string + + runtimeutil.FunctionFilter + + ids map[string]*yaml.RNode +} + +func (sf *Filter) String() string { + return fmt.Sprintf( + "name: %v path: %v url: %v program: %v", sf.Name, sf.Path, sf.URL, sf.Program) +} + +func (sf *Filter) Filter(nodes []*yaml.RNode) ([]*yaml.RNode, error) { + err := sf.setup() + if err != nil { + return nil, err + } + sf.FunctionFilter.Run = sf.Run + + return sf.FunctionFilter.Filter(nodes) +} + +func (sf *Filter) setup() error { + if sf.URL != "" && sf.Path != "" || + sf.URL != "" && sf.Program != "" || + sf.Path != "" && sf.Program != "" { + return errors.Errorf("Filter Path, Program and URL are mutually exclusive") + } + + // read the program from a file + if sf.Path != "" { + b, err := ioutil.ReadFile(sf.Path) + if err != nil { + return err + } + sf.Program = string(b) + } + + // read the program from a URL + if sf.URL != "" { + err := func() error { + resp, err := http.Get(sf.URL) + if err != nil { + return err + } + defer resp.Body.Close() + b, err := ioutil.ReadAll(resp.Body) + if err != nil { + return err + } + sf.Program = string(b) + return nil + }() + if err != nil { + return err + } + } + + return nil +} + +func (sf *Filter) Run(reader io.Reader, writer io.Writer) error { + // retain map of inputs to outputs by id so if the name is changed by the + // starlark program, we are able to match the same resources + value, err := sf.readResourceList(reader) + if err != nil { + return errors.Wrap(err) + } + + // run the starlark as program as transformation function + thread := &starlark.Thread{Name: sf.Name} + + ctx := &Context{resourceList: value} + pd, err := ctx.predeclared() + if err != nil { + return errors.Wrap(err) + } + _, err = starlark.ExecFile(thread, sf.Name, sf.Program, pd) + if err != nil { + return errors.Wrap(err) + } + + return sf.writeResourceList(value, writer) +} + +// inputToResourceList transforms input into a starlark.Value +func (sf *Filter) readResourceList(reader io.Reader) (starlark.Value, error) { + // read and parse the inputs + rl := bytes.Buffer{} + _, err := rl.ReadFrom(reader) + if err != nil { + return nil, errors.Wrap(err) + } + rn, err := yaml.Parse(rl.String()) + if err != nil { + return nil, errors.Wrap(err) + } + + // set the id on each node to map inputs to outputs + var id int + sf.ids = map[string]*yaml.RNode{} + items, err := rn.Pipe(yaml.Lookup("items")) + if err != nil { + return nil, errors.Wrap(err) + } + err = items.VisitElements(func(node *yaml.RNode) error { + id++ + idStr := fmt.Sprintf("%v", id) + sf.ids[idStr] = node + return node.PipeE(yaml.SetAnnotation("config.k8s.io/id", idStr)) + }) + if err != nil { + return nil, errors.Wrap(err) + } + + // convert to a starlark value + b, err := yaml.Marshal(rn.Document()) // convert to bytes + if err != nil { + return nil, errors.Wrap(err) + } + var in map[string]interface{} + err = yaml.Unmarshal(b, &in) // convert to map[string]interface{} + if err != nil { + return nil, errors.Wrap(err) + } + return util.Marshal(in) // convert to starlark value +} + +// resourceListToOutput converts the output of the starlark program to the filter output +func (sf *Filter) writeResourceList(value starlark.Value, writer io.Writer) error { + // convert the modified resourceList back into a slice of RNodes + // by first converting to a map[string]interface{} + out, err := util.Unmarshal(value) + if err != nil { + return errors.Wrap(err) + } + b, err := yaml.Marshal(out) + if err != nil { + return errors.Wrap(err) + } + + rl, err := yaml.Parse(string(b)) + if err != nil { + return errors.Wrap(err) + } + + // preserve the comments from the input + items, err := rl.Pipe(yaml.Lookup("items")) + if err != nil { + return errors.Wrap(err) + } + err = items.VisitElements(func(node *yaml.RNode) error { + anID, err := node.Pipe(yaml.GetAnnotation("config.k8s.io/id")) + if err != nil { + return errors.Wrap(err) + } + if anID == nil { + return nil + } + + var in *yaml.RNode + var found bool + if in, found = sf.ids[anID.YNode().Value]; !found { + return nil + } + if err := node.PipeE(yaml.ClearAnnotation("config.k8s.io/id")); err != nil { + return errors.Wrap(err) + } + if err := comments.CopyComments(in, node); err != nil { + return errors.Wrap(err) + } + + // starlark will serialize the resources sorting the fields alphabetically, + // format them to have a better ordering + fmtFltr := filters.FormatFilter{} + if _, err := fmtFltr.Filter([]*yaml.RNode{node}); err != nil { + return errors.Wrap(err) + } + return nil + }) + if err != nil { + return errors.Wrap(err) + } + + s, err := rl.String() + if err != nil { + return errors.Wrap(err) + } + + _, err = writer.Write([]byte(s)) + return err +} diff --git a/kyaml/starlark/starlark_test.go b/kyaml/fn/runtime/starlark/starlark_test.go similarity index 91% rename from kyaml/starlark/starlark_test.go rename to kyaml/fn/runtime/starlark/starlark_test.go index 3b0f59805..8e2ec4eae 100644 --- a/kyaml/starlark/starlark_test.go +++ b/kyaml/fn/runtime/starlark/starlark_test.go @@ -56,6 +56,7 @@ metadata: name: nginx-deployment annotations: foo: bar + config.kubernetes.io/path: 'deployment_nginx-deployment.yaml' spec: template: spec: @@ -95,6 +96,7 @@ metadata: name: nginx-deployment annotations: foo: annotation-value + config.kubernetes.io/path: 'deployment_nginx-deployment.yaml' spec: template: spec: @@ -133,6 +135,7 @@ metadata: name: nginx-deployment annotations: foo: Deployment enables declarative updates for Pods and ReplicaSets. + config.kubernetes.io/path: 'deployment_nginx-deployment.yaml' spec: template: spec: @@ -174,6 +177,7 @@ metadata: name: nginx-deployment annotations: foo: bar + config.kubernetes.io/path: 'deployment_nginx-deployment.yaml' spec: template: spec: @@ -213,6 +217,8 @@ apiVersion: apps/v1 kind: Deployment metadata: name: nginx-deployment + annotations: + config.kubernetes.io/path: 'deployment_nginx-deployment.yaml' spec: template: spec: @@ -266,6 +272,7 @@ metadata: name: nginx-deployment-1 annotations: foo: bar + config.kubernetes.io/path: 'deployment_nginx-deployment-1.yaml' spec: template: spec: @@ -280,6 +287,7 @@ metadata: name: nginx-deployment-2 annotations: foo: bar + config.kubernetes.io/path: 'deployment_nginx-deployment-2.yaml' spec: template: spec: @@ -318,13 +326,10 @@ run(ctx.resource_list["items"]) expected: ` apiVersion: apps/v1 kind: Deployment -metadata: - name: nginx-deployment-2 ---- -apiVersion: apps/v1 -kind: Deployment metadata: name: nginx-deployment-1 + annotations: + config.kubernetes.io/path: 'deployment_nginx-deployment-1.yaml' spec: template: spec: @@ -332,6 +337,13 @@ spec: - name: nginx # head comment image: nginx:1.8.1 # {"$ref": "#/definitions/io.k8s.cli.substitutions.image"} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nginx-deployment-2 + annotations: + config.kubernetes.io/path: 'deployment_nginx-deployment-2.yaml' `, }, { @@ -357,6 +369,8 @@ apiVersion: apps/v1 kind: Deployment metadata: name: nginx-deployment-1 + annotations: + config.kubernetes.io/path: 'deployment_nginx-deployment-1.yaml' `, }, { @@ -395,6 +409,7 @@ metadata: name: nginx-deployment annotations: foo: hello world + config.kubernetes.io/path: 'deployment_nginx-deployment.yaml' spec: template: spec: @@ -406,7 +421,7 @@ spec: expectedFunctionConfig: ` kind: Script spec: - value: hello world + value: "hello world" `, }, @@ -447,6 +462,7 @@ metadata: name: nginx-deployment annotations: foo: hello world + config.kubernetes.io/path: 'deployment_nginx-deployment.yaml' spec: template: spec: @@ -458,7 +474,7 @@ spec: expectedFunctionConfig: ` kind: Script spec: - value: updated + value: "hello world" `, }, } diff --git a/kyaml/go.mod b/kyaml/go.mod index 1ff78e7fe..5f8bfc392 100644 --- a/kyaml/go.mod +++ b/kyaml/go.mod @@ -9,6 +9,7 @@ require ( github.com/qri-io/starlib v0.4.2-0.20200213133954-ff2e8cd5ef8d github.com/sergi/go-diff v1.1.0 github.com/spf13/cobra v1.0.0 + github.com/spf13/pflag v1.0.3 github.com/stretchr/testify v1.4.0 github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5 diff --git a/kyaml/kio/filters/container.go b/kyaml/kio/filters/container.go deleted file mode 100644 index 8c30d0225..000000000 --- a/kyaml/kio/filters/container.go +++ /dev/null @@ -1,450 +0,0 @@ -// Copyright 2019 The Kubernetes Authors. -// SPDX-License-Identifier: Apache-2.0 - -package filters - -import ( - "bytes" - "fmt" - "io/ioutil" - "os" - "os/exec" - "path" - "strings" - - "sigs.k8s.io/kustomize/kyaml/errors" - "sigs.k8s.io/kustomize/kyaml/kio" - "sigs.k8s.io/kustomize/kyaml/kio/kioutil" - - "sigs.k8s.io/kustomize/kyaml/yaml" -) - -// ContainerFilter filters Resources using a container image. -// The container must start a process that reads the list of -// input Resources from stdin, reads the Configuration from the env -// API_CONFIG, and writes the filtered Resources to stdout. -// If there is a error or validation failure, the process must exit -// non-zero. -// The full set of environment variables from the parent process -// are passed to the container. -// -// Function Scoping: -// ContainerFilter applies the function only to Resources to which it is scoped. -// -// Resources are scoped to a function if any of the following are true: -// - the Resource were read from the same directory as the function config -// - the Resource were read from a subdirectory of the function config directory -// - the function config is in a directory named "functions" and -// they were read from a subdirectory of "functions" parent -// - the function config doesn't have a path annotation (considered globally scoped) -// - the ContainerFilter has GlobalScope == true -// -// In Scope Examples: -// -// Example 1: deployment.yaml and service.yaml in function.yaml scope -// same directory as the function config directory -// . -// ├── function.yaml -// ├── deployment.yaml -// └── service.yaml -// -// Example 2: apps/deployment.yaml and apps/service.yaml in function.yaml scope -// subdirectory of the function config directory -// . -// ├── function.yaml -// └── apps -//    ├── deployment.yaml -//    └── service.yaml -// -// Example 3: apps/deployment.yaml and apps/service.yaml in functions/function.yaml scope -// function config is in a directory named "functions" -// . -// ├── functions -// │   └── function.yaml -// └── apps -//    ├── deployment.yaml -//    └── service.yaml -// -// Out of Scope Examples: -// -// Example 1: apps/deployment.yaml and apps/service.yaml NOT in stuff/function.yaml scope -// . -// ├── stuff -// │   └── function.yaml -// └── apps -//    ├── deployment.yaml -//    └── service.yaml -// -// Example 2: apps/deployment.yaml and apps/service.yaml NOT in stuff/functions/function.yaml scope -// . -// ├── stuff -// │   └── functions -// │    └── function.yaml -// └── apps -//    ├── deployment.yaml -//    └── service.yaml -// -// Default Paths: -// Resources emitted by functions will have default path applied as annotations -// if none is present. -// The default path will be the function-dir/ (or parent directory in the case of "functions") -// + function-file-name/ + namespace/ + kind_name.yaml -// -// Example 1: Given a function in fn.yaml that produces a Deployment name foo and a Service named bar -// dir -// └── fn.yaml -// -// Would default newly generated Resources to: -// -// dir -// ├── fn.yaml -// └── fn -//    ├── deployment_foo.yaml -//    └── service_bar.yaml -// -// Example 2: Given a function in functions/fn.yaml that produces a Deployment name foo and a Service named bar -// dir -// └── fn.yaml -// -// Would default newly generated Resources to: -// -// dir -// ├── functions -// │   └── fn.yaml -// └── fn -//    ├── deployment_foo.yaml -//    └── service_bar.yaml -// -// Example 3: Given a function in fn.yaml that produces a Deployment name foo, namespace baz and a Service named bar namespace baz -// dir -// └── fn.yaml -// -// Would default newly generated Resources to: -// -// dir -// ├── fn.yaml -// └── fn -// └── baz -//    ├── deployment_foo.yaml -//    └── service_bar.yaml -type ContainerFilter struct { - - // Image is the container image to use to create a container. - Image string `yaml:"image,omitempty"` - - // Network is the container network to use. - Network string `yaml:"network,omitempty"` - - // StorageMounts is a list of storage options that the container will have mounted. - StorageMounts []StorageMount `yaml:"mounts,omitempty"` - - // Config is the API configuration for the container and passed through the - // API_CONFIG env var to the container. - // Typically a Kubernetes style Resource Config. - Config *yaml.RNode `yaml:"config,omitempty"` - - // GlobalScope will cause the function to be run against all input - // nodes instead of only nodes scoped under the function. - GlobalScope bool - - ResultsFile string - - Results *yaml.RNode - - DeferFailure bool - - Exit error - - // SetFlowStyleForConfig sets the style for config to Flow when serializing it - SetFlowStyleForConfig bool - - // args may be specified by tests to override how a container is spawned - args []string - - checkInput func(string) -} - -func (c ContainerFilter) GetExit() error { - return c.Exit -} - -type DeferFailureFunction interface { - GetExit() error -} - -func (c ContainerFilter) String() string { - if c.DeferFailure { - return fmt.Sprintf("%s deferFailure: %v", c.Image, c.DeferFailure) - } - return c.Image -} - -func (s *StorageMount) String() string { - return fmt.Sprintf("type=%s,src=%s,dst=%s:ro", s.MountType, s.Src, s.DstPath) -} - -func StringToStorageMount(s string) StorageMount { - m := make(map[string]string) - options := strings.Split(s, ",") - for _, option := range options { - keyVal := strings.SplitN(option, "=", 2) - m[keyVal[0]] = keyVal[1] - } - var sm StorageMount - for key, value := range m { - switch { - case key == "type": - sm.MountType = value - case key == "src": - sm.Src = value - case key == "dst": - sm.DstPath = value - } - } - return sm -} - -// functionsDirectoryName is keyword directory name for functions scoped 1 directory higher -const functionsDirectoryName = "functions" - -// getFunctionScope returns the path of the directory containing the function config, -// or its parent directory if the base directory is named "functions" -func (c *ContainerFilter) getFunctionScope() (string, error) { - m, err := c.Config.GetMeta() - if err != nil { - return "", errors.Wrap(err) - } - p, found := m.Annotations[kioutil.PathAnnotation] - if !found { - return "", nil - } - - functionDir := path.Clean(path.Dir(p)) - - if path.Base(functionDir) == functionsDirectoryName { - // the scope of functions in a directory called "functions" is 1 level higher - // this is similar to how the golang "internal" directory scoping works - functionDir = path.Dir(functionDir) - } - return functionDir, nil -} - -// scope partitions the input nodes into 2 slices. The first slice contains only Resources -// which are scoped under dir, and the second slice contains the Resources which are not. -func (c *ContainerFilter) scope(dir string, nodes []*yaml.RNode) ([]*yaml.RNode, []*yaml.RNode, error) { - // scope container filtered Resources to Resources under that directory - var input, saved []*yaml.RNode - if c.GlobalScope { - return nodes, nil, nil - } - - // global function - if dir == "" || dir == "." { - return nodes, nil, nil - } - - // identify Resources read from directories under the function configuration - for i := range nodes { - m, err := nodes[i].GetMeta() - if err != nil { - return nil, nil, err - } - p, found := m.Annotations[kioutil.PathAnnotation] - if !found { - // this Resource isn't scoped under the function -- don't know where it came from - // consider it out of scope - saved = append(saved, nodes[i]) - continue - } - - resourceDir := path.Clean(path.Dir(p)) - if path.Base(resourceDir) == functionsDirectoryName { - // Functions in the `functions` directory are scoped to - // themselves, and should see themselves as input - resourceDir = path.Dir(resourceDir) - } - if !strings.HasPrefix(resourceDir, dir) { - // this Resource doesn't fall under the function scope if it - // isn't in a subdirectory of where the function lives - saved = append(saved, nodes[i]) - continue - } - - // this input is scoped under the function - input = append(input, nodes[i]) - } - - return input, saved, nil -} - -// GrepFilter implements kio.GrepFilter -func (c *ContainerFilter) Filter(nodes []*yaml.RNode) ([]*yaml.RNode, error) { - // get the command to filter the Resources - cmd := c.getCommand() - - in := &bytes.Buffer{} - out := &bytes.Buffer{} - - // only process Resources scoped to this function, save the others - functionDir, err := c.getFunctionScope() - if err != nil { - return nil, err - } - input, saved, err := c.scope(functionDir, nodes) - if err != nil { - return nil, err - } - - // write the input - err = kio.ByteWriter{ - WrappingAPIVersion: kio.ResourceListAPIVersion, - WrappingKind: kio.ResourceListKind, - Writer: in, - KeepReaderAnnotations: true, - FunctionConfig: c.Config}.Write(input) - if err != nil { - return nil, err - } - - // capture the command stdout for the return value - r := &kio.ByteReader{Reader: out} - - // do the filtering - if c.checkInput != nil { - c.checkInput(in.String()) - } - cmd.Stdin = in - cmd.Stdout = out - - // don't exit immediately if the function fails -- write out the validation - c.Exit = cmd.Run() - - output, err := r.Read() - if err != nil { - return nil, err - } - - if err := c.doResults(r); err != nil { - return nil, err - } - - if c.Exit != nil && !c.DeferFailure { - return append(output, saved...), c.Exit - } - - // annotate any generated Resources with a path and index if they don't already have one - if err := kioutil.DefaultPathAnnotation(functionDir, output); err != nil { - return nil, err - } - - // emit both the Resources output from the function, and the out-of-scope Resources - // which were not provided to the function - return append(output, saved...), nil -} - -func (c *ContainerFilter) doResults(r *kio.ByteReader) error { - // Write the results to a file if configured to do so - if c.ResultsFile != "" && r.Results != nil { - results, err := r.Results.String() - if err != nil { - return err - } - err = ioutil.WriteFile(c.ResultsFile, []byte(results), 0600) - if err != nil { - return err - } - } - - if r.Results != nil { - c.Results = r.Results - } - return nil -} - -// getArgs returns the command + args to run to spawn the container -func (c *ContainerFilter) getArgs() []string { - // run the container using docker. this is simpler than using the docker - // libraries, and ensures things like auth work the same as if the container - // was run from the cli. - - network := "none" - if c.Network != "" { - network = c.Network - } - - args := []string{"docker", "run", - "--rm", // delete the container afterward - "-i", "-a", "STDIN", "-a", "STDOUT", "-a", "STDERR", // attach stdin, stdout, stderr - - // added security options - "--network", network, - "--user", "nobody", // run as nobody - // don't make fs readonly because things like heredoc rely on writing tmp files - "--security-opt=no-new-privileges", // don't allow the user to escalate privileges - } - - // TODO(joncwong): Allow StorageMount fields to have default values. - for _, storageMount := range c.StorageMounts { - args = append(args, "--mount", storageMount.String()) - } - - // tell functions to write error messages to stderr as well as results - os.Setenv("LOG_TO_STDERR", "true") - os.Setenv("STRUCTURED_RESULTS", "true") - - // export the local environment vars to the container - for _, pair := range os.Environ() { - tokens := strings.Split(pair, "=") - if tokens[0] == "" { - continue - } - args = append(args, "-e", tokens[0]) - } - return append(args, c.Image) -} - -// getCommand returns a command which will apply the Filter using the container image -func (c *ContainerFilter) getCommand() *exec.Cmd { - if c.SetFlowStyleForConfig { - c.Config.YNode().Style = yaml.FlowStyle - } - - if len(c.args) == 0 { - c.args = c.getArgs() - } - - cmd := exec.Command(c.args[0], c.args[1:]...) - cmd.Stderr = os.Stderr - cmd.Env = os.Environ() - - // set stderr for err messaging - return cmd -} - -// IsReconcilerFilter filters Resources based on whether or not they are Reconciler Resource. -// Resources with an apiVersion starting with '*.gcr.io', 'gcr.io' or 'docker.io' are considered -// Reconciler Resources. -type IsReconcilerFilter struct { - // ExcludeReconcilers if set to true, then Reconcilers will be excluded -- e.g. - // Resources with a reconcile container through the apiVersion (gcr.io prefix) or - // through the annotations - ExcludeReconcilers bool `yaml:"excludeReconcilers,omitempty"` - - // IncludeNonReconcilers if set to true, the NonReconciler will be included. - IncludeNonReconcilers bool `yaml:"includeNonReconcilers,omitempty"` -} - -// Filter implements kio.Filter -func (c *IsReconcilerFilter) Filter(inputs []*yaml.RNode) ([]*yaml.RNode, error) { - var out []*yaml.RNode - for i := range inputs { - isFnResource := GetFunctionSpec(inputs[i]) != nil - if isFnResource && !c.ExcludeReconcilers { - out = append(out, inputs[i]) - } - if !isFnResource && c.IncludeNonReconcilers { - out = append(out, inputs[i]) - } - } - return out, nil -} diff --git a/kyaml/kio/filters/container_test.go b/kyaml/kio/filters/container_test.go deleted file mode 100644 index c960b8b66..000000000 --- a/kyaml/kio/filters/container_test.go +++ /dev/null @@ -1,1654 +0,0 @@ -// Copyright 2019 The Kubernetes Authors. -// SPDX-License-Identifier: Apache-2.0 - -package filters - -import ( - "bytes" - "fmt" - "io/ioutil" - "os" - "strings" - "testing" - - "github.com/stretchr/testify/assert" - "sigs.k8s.io/kustomize/kyaml/kio" - "sigs.k8s.io/kustomize/kyaml/yaml" -) - -func TestContainerFilter_Filter(t *testing.T) { - var tests = []struct { - name string - input []string - expectedOutput []string - expectedError string - expectedSavedError string - expectedResults string - noMakeResultsFile bool - instance ContainerFilter - }{ - { - name: "add_path_annotation", - instance: ContainerFilter{args: []string{ - "echo", ` -apiVersion: config.kubernetes.io/v1alpha1 -kind: ResourceList -items: -- apiVersion: apps/v1 - kind: Deployment - metadata: - name: deployment-foo -- apiVersion: v1 - kind: Service - metadata: - name: service-foo -`, - }, - }, - expectedOutput: []string{ - ` -apiVersion: apps/v1 -kind: Deployment -metadata: - name: deployment-foo - annotations: - config.kubernetes.io/path: 'deployment_deployment-foo.yaml' -`, - ` -apiVersion: v1 -kind: Service -metadata: - name: service-foo - annotations: - config.kubernetes.io/path: 'service_service-foo.yaml' -`, - }, - }, - - { - name: "write_results", - instance: ContainerFilter{args: []string{ - "echo", ` -apiVersion: config.kubernetes.io/v1alpha1 -kind: ResourceList -items: -- apiVersion: apps/v1 - kind: Deployment - metadata: - name: deployment-foo -- apiVersion: v1 - kind: Service - metadata: - name: service-foo -results: -- apiVersion: config.k8s.io/v1alpha1 - kind: ObjectError - name: "some-validator" - items: - - type: error - message: "some message" - resourceRef: - apiVersion: apps/v1 - kind: Deployment - name: foo - namespace: bar - file: - path: deploy.yaml - index: 0 - field: - path: "spec.template.spec.containers[3].resources.limits.cpu" - currentValue: "200" - suggestedValue: "2" -`, - }, - }, - expectedOutput: []string{ - ` -apiVersion: apps/v1 -kind: Deployment -metadata: - name: deployment-foo - annotations: - config.kubernetes.io/path: 'deployment_deployment-foo.yaml' -`, - ` -apiVersion: v1 -kind: Service -metadata: - name: service-foo - annotations: - config.kubernetes.io/path: 'service_service-foo.yaml' -`, - }, - expectedResults: ` -- apiVersion: config.k8s.io/v1alpha1 - kind: ObjectError - name: "some-validator" - items: - - type: error - message: "some message" - resourceRef: - apiVersion: apps/v1 - kind: Deployment - name: foo - namespace: bar - file: - path: deploy.yaml - index: 0 - field: - path: "spec.template.spec.containers[3].resources.limits.cpu" - currentValue: "200" - suggestedValue: "2" -`, - }, - - { - name: "write_results_non_0_exit", - expectedError: "exit status 1", - instance: ContainerFilter{args: []string{"sh", "-c", - `echo ' -apiVersion: config.kubernetes.io/v1alpha1 -kind: ResourceList -items: -- apiVersion: apps/v1 - kind: Deployment - metadata: - name: deployment-foo -- apiVersion: v1 - kind: Service - metadata: - name: service-foo -results: -- apiVersion: config.k8s.io/v1alpha1 - kind: ObjectError - name: "some-validator" - items: - - type: error - message: "some message" - resourceRef: - apiVersion: apps/v1 - kind: Deployment - name: foo - namespace: bar - file: - path: deploy.yaml - index: 0 - field: - path: "spec.template.spec.containers[3].resources.limits.cpu" - currentValue: "200" - suggestedValue: "2" -' && cat not-real-dir -`, - }, - }, - expectedOutput: []string{ - ` -apiVersion: apps/v1 -kind: Deployment -metadata: - name: deployment-foo - annotations: - config.kubernetes.io/path: 'deployment_deployment-foo.yaml' -`, - ` -apiVersion: v1 -kind: Service -metadata: - name: service-foo - annotations: - config.kubernetes.io/path: 'service_service-foo.yaml' -`, - }, - expectedResults: ` -- apiVersion: config.k8s.io/v1alpha1 - kind: ObjectError - name: "some-validator" - items: - - type: error - message: "some message" - resourceRef: - apiVersion: apps/v1 - kind: Deployment - name: foo - namespace: bar - file: - path: deploy.yaml - index: 0 - field: - path: "spec.template.spec.containers[3].resources.limits.cpu" - currentValue: "200" - suggestedValue: "2" -`, - }, - - { - name: "write_results_defer_failure", - expectedSavedError: "exit status 1", - instance: ContainerFilter{ - DeferFailure: true, - args: []string{"sh", "-c", - `echo ' -apiVersion: config.kubernetes.io/v1alpha1 -kind: ResourceList -items: -- apiVersion: apps/v1 - kind: Deployment - metadata: - name: deployment-foo -- apiVersion: v1 - kind: Service - metadata: - name: service-foo -results: -- apiVersion: config.k8s.io/v1alpha1 - kind: ObjectError - name: "some-validator" - items: - - type: error - message: "some message" - resourceRef: - apiVersion: apps/v1 - kind: Deployment - name: foo - namespace: bar - file: - path: deploy.yaml - index: 0 - field: - path: "spec.template.spec.containers[3].resources.limits.cpu" - currentValue: "200" - suggestedValue: "2" -' && cat not-real-dir -`, - }, - }, - expectedOutput: []string{ - ` -apiVersion: apps/v1 -kind: Deployment -metadata: - name: deployment-foo - annotations: - config.kubernetes.io/path: 'deployment_deployment-foo.yaml' -`, - ` -apiVersion: v1 -kind: Service -metadata: - name: service-foo - annotations: - config.kubernetes.io/path: 'service_service-foo.yaml' -`, - }, - expectedResults: ` -- apiVersion: config.k8s.io/v1alpha1 - kind: ObjectError - name: "some-validator" - items: - - type: error - message: "some message" - resourceRef: - apiVersion: apps/v1 - kind: Deployment - name: foo - namespace: bar - file: - path: deploy.yaml - index: 0 - field: - path: "spec.template.spec.containers[3].resources.limits.cpu" - currentValue: "200" - suggestedValue: "2" -`, - }, - - { - name: "write_results_non_0_exit_missing_file", - expectedError: "open /not/real/file: no such file or directory", - noMakeResultsFile: true, - instance: ContainerFilter{args: []string{"sh", "-c", - `echo ' -apiVersion: config.kubernetes.io/v1alpha1 -kind: ResourceList -items: -- apiVersion: apps/v1 - kind: Deployment - metadata: - name: deployment-foo -- apiVersion: v1 - kind: Service - metadata: - name: service-foo -results: -- apiVersion: config.k8s.io/v1alpha1 - kind: ObjectError - name: "some-validator" - items: - - type: error - message: "some message" - resourceRef: - apiVersion: apps/v1 - kind: Deployment - name: foo - namespace: bar - file: - path: deploy.yaml - index: 0 - field: - path: "spec.template.spec.containers[3].resources.limits.cpu" - currentValue: "200" - suggestedValue: "2" -' && cat not-real-dir -`, - }, - }, - expectedOutput: []string{ - ` -apiVersion: apps/v1 -kind: Deployment -metadata: - name: deployment-foo - annotations: - config.kubernetes.io/path: 'deployment_deployment-foo.yaml' -`, - ` -apiVersion: v1 -kind: Service -metadata: - name: service-foo - annotations: - config.kubernetes.io/path: 'service_service-foo.yaml' -`, - }, - expectedResults: ` -- apiVersion: config.k8s.io/v1alpha1 - kind: ObjectError - name: "some-validator" - items: - - type: error - message: "some message" - resourceRef: - apiVersion: apps/v1 - kind: Deployment - name: foo - namespace: bar - file: - path: deploy.yaml - index: 0 - field: - path: "spec.template.spec.containers[3].resources.limits.cpu" - currentValue: "200" - suggestedValue: "2" -`, - }, - } - - for i := range tests { - tt := tests[i] - t.Run(tt.name, func(t *testing.T) { - if len(tt.expectedResults) > 0 && !tt.noMakeResultsFile { - f, err := ioutil.TempFile("", "test-kyaml-*.yaml") - if !assert.NoError(t, err) { - t.FailNow() - } - defer os.RemoveAll(f.Name()) - tt.instance.ResultsFile = f.Name() - } else if len(tt.expectedResults) > 0 { - tt.instance.ResultsFile = "/not/real/file" - } - - var inputs []*yaml.RNode - for i := range tt.input { - node, err := yaml.Parse(tt.input[i]) - if !assert.NoError(t, err) { - t.FailNow() - } - inputs = append(inputs, node) - } - - output, err := tt.instance.Filter(inputs) - if tt.expectedError != "" { - if !assert.EqualError(t, err, tt.expectedError) { - t.FailNow() - } - return - } - - if tt.expectedSavedError != "" { - if !assert.EqualError(t, tt.instance.Exit, tt.expectedSavedError) { - t.FailNow() - } - return - } - - if !assert.NoError(t, err) { - t.FailNow() - } - - var actual []string - for i := range output { - s, err := output[i].String() - if !assert.NoError(t, err) { - t.FailNow() - } - actual = append(actual, strings.TrimSpace(s)) - } - var expected []string - for i := range tt.expectedOutput { - expected = append(expected, strings.TrimSpace(tt.expectedOutput[i])) - } - - if !assert.Equal(t, expected, actual) { - t.FailNow() - } - - if len(tt.instance.ResultsFile) > 0 { - tt.expectedResults = strings.TrimSpace(tt.expectedResults) - - results, err := tt.instance.Results.String() - if !assert.NoError(t, err) { - t.FailNow() - } - if !assert.Equal(t, tt.expectedResults, strings.TrimSpace(results)) { - t.FailNow() - } - - b, err := ioutil.ReadFile(tt.instance.ResultsFile) - writtenResults := strings.TrimSpace(string(b)) - if !assert.NoError(t, err) { - t.FailNow() - } - if !assert.Equal(t, tt.expectedResults, writtenResults) { - t.FailNow() - } - } - }) - } -} - -func TestFilter_command(t *testing.T) { - cfg, err := yaml.Parse(`apiVersion: apps/v1 -kind: Deployment -metadata: - name: foo -`) - if !assert.NoError(t, err) { - return - } - instance := &ContainerFilter{ - Image: "example.com:version", - Config: cfg, - } - os.Setenv("KYAML_TEST", "FOO") - cmd := instance.getCommand() - - expected := []string{ - "docker", "run", - "--rm", - "-i", "-a", "STDIN", "-a", "STDOUT", "-a", "STDERR", - "--network", "none", - "--user", "nobody", - "--security-opt=no-new-privileges", - } - for _, e := range os.Environ() { - // the process env - expected = append(expected, "-e", strings.Split(e, "=")[0]) - } - expected = append(expected, "example.com:version") - assert.Equal(t, expected, cmd.Args) - - foundKyaml := false - for _, e := range cmd.Env { - // verify the command has the right environment variables to pass to the container - split := strings.Split(e, "=") - if split[0] == "KYAML_TEST" { - assert.Equal(t, "FOO", split[1]) - foundKyaml = true - } - } - assert.True(t, foundKyaml) -} - -func TestFilter_command_StorageMount(t *testing.T) { - cfg, err := yaml.Parse(`apiVersion: apps/v1 -kind: Deployment -metadata: - name: foo -`) - if !assert.NoError(t, err) { - return - } - bindMount := StorageMount{"bind", "/mount/path", "/local/"} - localVol := StorageMount{"volume", "myvol", "/local/"} - tmpfs := StorageMount{"tmpfs", "", "/local/"} - instance := &ContainerFilter{ - Image: "example.com:version", - Config: cfg, - StorageMounts: []StorageMount{bindMount, localVol, tmpfs}, - } - cmd := instance.getCommand() - - expected := []string{ - "docker", "run", - "--rm", - "-i", "-a", "STDIN", "-a", "STDOUT", "-a", "STDERR", - "--network", "none", - "--user", "nobody", - "--security-opt=no-new-privileges", - "--mount", fmt.Sprintf("type=%s,src=%s,dst=%s:ro", "bind", "/mount/path", "/local/"), - "--mount", fmt.Sprintf("type=%s,src=%s,dst=%s:ro", "volume", "myvol", "/local/"), - "--mount", fmt.Sprintf("type=%s,src=%s,dst=%s:ro", "tmpfs", "", "/local/"), - } - for _, e := range os.Environ() { - // the process env - expected = append(expected, "-e", strings.Split(e, "=")[0]) - } - expected = append(expected, "example.com:version") - assert.Equal(t, expected, cmd.Args) -} - -func TestFilter_command_network(t *testing.T) { - cfg, err := yaml.Parse(`apiVersion: apps/v1 -kind: Deployment -metadata: - name: foo -`) - if !assert.NoError(t, err) { - return - } - instance := &ContainerFilter{ - Image: "example.com:version", - Network: "test-net", - Config: cfg, - } - cmd := instance.getCommand() - - expected := []string{ - "docker", "run", - "--rm", - "-i", "-a", "STDIN", "-a", "STDOUT", "-a", "STDERR", - "--network", "test-net", - "--user", "nobody", - "--security-opt=no-new-privileges", - } - for _, e := range os.Environ() { - // the process env - tokens := strings.Split(e, "=") - if tokens[0] == "" { - continue - } - expected = append(expected, "-e", tokens[0]) - } - expected = append(expected, "example.com:version") - assert.Equal(t, expected, cmd.Args) -} - -func TestFilter_Filter(t *testing.T) { - cfg, err := yaml.Parse(`apiVersion: apps/v1 -kind: Deployment -metadata: - name: foo -`) - if !assert.NoError(t, err) { - return - } - - input, err := (&kio.ByteReader{Reader: bytes.NewBufferString(` -apiVersion: apps/v1 -kind: Deployment -metadata: - name: deployment-foo ---- -apiVersion: v1 -kind: Service -metadata: - name: service-foo -`)}).Read() - if !assert.NoError(t, err) { - return - } - - called := false - result, err := (&ContainerFilter{ - SetFlowStyleForConfig: true, - Image: "example.com:version", - Config: cfg, - args: []string{"sed", "s/Deployment/StatefulSet/g"}, - checkInput: func(s string) { - called = true - if !assert.Equal(t, `apiVersion: config.kubernetes.io/v1alpha1 -kind: ResourceList -items: -- apiVersion: apps/v1 - kind: Deployment - metadata: - name: deployment-foo - annotations: - config.kubernetes.io/index: '0' -- apiVersion: v1 - kind: Service - metadata: - name: service-foo - annotations: - config.kubernetes.io/index: '1' -functionConfig: {apiVersion: apps/v1, kind: Deployment, metadata: {name: foo}} -`, s) { - t.FailNow() - } - }, - }).Filter(input) - if !assert.NoError(t, err) { - return - } - if !assert.True(t, called) { - return - } - - b := &bytes.Buffer{} - err = kio.ByteWriter{Writer: b, KeepReaderAnnotations: true}.Write(result) - if !assert.NoError(t, err) { - return - } - - assert.Equal(t, `apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: deployment-foo - annotations: - config.kubernetes.io/index: '0' - config.kubernetes.io/path: 'statefulset_deployment-foo.yaml' ---- -apiVersion: v1 -kind: Service -metadata: - name: service-foo - annotations: - config.kubernetes.io/index: '1' - config.kubernetes.io/path: 'service_service-foo.yaml' -`, b.String()) -} - -func TestFilter_Filter_noChange(t *testing.T) { - cfg, err := yaml.Parse(`apiVersion: apps/v1 -kind: Deployment -metadata: - name: foo -`) - if !assert.NoError(t, err) { - return - } - - input, err := (&kio.ByteReader{Reader: bytes.NewBufferString(` -apiVersion: apps/v1 -kind: Deployment -metadata: - name: deployment-foo ---- -apiVersion: v1 -kind: Service -metadata: - name: service-foo -`)}).Read() - if !assert.NoError(t, err) { - return - } - - called := false - result, err := (&ContainerFilter{ - SetFlowStyleForConfig: true, - Image: "example.com:version", - Config: cfg, - args: []string{"sh", "-c", "cat <&0"}, - checkInput: func(s string) { - called = true - if !assert.Equal(t, `apiVersion: config.kubernetes.io/v1alpha1 -kind: ResourceList -items: -- apiVersion: apps/v1 - kind: Deployment - metadata: - name: deployment-foo - annotations: - config.kubernetes.io/index: '0' -- apiVersion: v1 - kind: Service - metadata: - name: service-foo - annotations: - config.kubernetes.io/index: '1' -functionConfig: {apiVersion: apps/v1, kind: Deployment, metadata: {name: foo}} -`, s) { - t.FailNow() - } - }, - }).Filter(input) - if !assert.NoError(t, err) { - return - } - if !assert.True(t, called) { - return - } - - b := &bytes.Buffer{} - err = kio.ByteWriter{Writer: b, KeepReaderAnnotations: true}.Write(result) - if !assert.NoError(t, err) { - return - } - - assert.Equal(t, `apiVersion: apps/v1 -kind: Deployment -metadata: - name: deployment-foo - annotations: - config.kubernetes.io/index: '0' - config.kubernetes.io/path: 'deployment_deployment-foo.yaml' ---- -apiVersion: v1 -kind: Service -metadata: - name: service-foo - annotations: - config.kubernetes.io/index: '1' - config.kubernetes.io/path: 'service_service-foo.yaml' -`, b.String()) -} - -func Test_GetFunction(t *testing.T) { - var tests = []struct { - name string - resource string - expectedFn string - missingFn bool - }{ - - // fn annotation - { - name: "fn annotation", - resource: ` -apiVersion: v1beta1 -kind: Example -metadata: - annotations: - config.kubernetes.io/function: |- - container: - image: foo:v1.0.0 -`, - expectedFn: ` -container: - image: foo:v1.0.0`, - }, - - { - name: "storage mounts json style", - resource: ` -apiVersion: v1beta1 -kind: Example -metadata: - annotations: - config.kubernetes.io/function: |- - container: - image: foo:v1.0.0 - mounts: [ {type: bind, src: /mount/path, dst: /local/}, {src: myvol, dst: /local/, type: volume}, {dst: /local/, type: tmpfs} ] -`, - expectedFn: ` -container: - image: foo:v1.0.0 - mounts: - - type: bind - src: /mount/path - dst: /local/ - - type: volume - src: myvol - dst: /local/ - - type: tmpfs - dst: /local/ -`, - }, - - { - name: "storage mounts yaml style", - resource: ` -apiVersion: v1beta1 -kind: Example -metadata: - annotations: - config.kubernetes.io/function: |- - container: - image: foo:v1.0.0 - mounts: - - src: /mount/path - type: bind - dst: /local/ - - dst: /local/ - src: myvol - type: volume - - type: tmpfs - dst: /local/ -`, - expectedFn: ` -container: - image: foo:v1.0.0 - mounts: - - type: bind - src: /mount/path - dst: /local/ - - type: volume - src: myvol - dst: /local/ - - type: tmpfs - dst: /local/ -`, - }, - - { - name: "network", - resource: ` -apiVersion: v1beta1 -kind: Example -metadata: - annotations: - config.kubernetes.io/function: |- - container: - image: foo:v1.0.0 - network: - required: true -`, - expectedFn: ` -container: - image: foo:v1.0.0 - network: - required: true -`, - }, - - { - name: "path", - resource: ` -apiVersion: v1beta1 -kind: Example -metadata: - annotations: - config.kubernetes.io/function: |- - path: foo - container: - image: foo:v1.0.0 -`, - // path should be erased - expectedFn: ` -container: - image: foo:v1.0.0 -`, - }, - - { - name: "network", - resource: ` -apiVersion: v1beta1 -kind: Example -metadata: - annotations: - config.kubernetes.io/function: |- - network: foo - container: - image: foo:v1.0.0 -`, - // network should be erased - expectedFn: ` -container: - image: foo:v1.0.0 -`, - }, - - // legacy fn style - {name: "legacy fn meta", - resource: ` -apiVersion: v1beta1 -kind: Example -metadata: - configFn: - container: - image: foo:v1.0.0 -`, - expectedFn: ` -container: - image: foo:v1.0.0 -`, - }, - - // no fn - {name: "no fn", - resource: ` -apiVersion: v1beta1 -kind: Example -metadata: - annotations: {} -`, - missingFn: true, - }, - - // test network, etc... - } - - for i := range tests { - tt := tests[i] - t.Run(tt.name, func(t *testing.T) { - resource := yaml.MustParse(tt.resource) - fn := GetFunctionSpec(resource) - if tt.missingFn { - if !assert.Nil(t, fn) { - t.FailNow() - } - } else { - b, err := yaml.Marshal(fn) - if !assert.NoError(t, err) { - t.FailNow() - } - if !assert.Equal(t, - strings.TrimSpace(tt.expectedFn), - strings.TrimSpace(string(b))) { - t.FailNow() - } - } - }) - } -} - -func Test_GetContainerNetworkRequired(t *testing.T) { - tests := []struct { - input string - required bool - }{ - { - input: `apiVersion: v1 -kind: Foo -metadata: - name: foo - configFn: - container: - image: gcr.io/kustomize-functions/example-tshirt:v0.1.0 - network: - required: true -`, - required: true, - }, - { - input: `apiVersion: v1 -kind: Foo -metadata: - name: foo - configFn: - container: - image: gcr.io/kustomize-functions/example-tshirt:v0.1.0 - network: - required: false -`, - required: false, - }, - { - - input: `apiVersion: v1 -kind: Foo -metadata: - name: foo - configFn: - container: - image: gcr.io/kustomize-functions/example-tshirt:v0.1.0 -`, - required: false, - }, - { - input: `apiVersion: v1 -kind: Foo -metadata: - name: foo - annotations: - config.kubernetes.io/function: | - container: - image: gcr.io/kustomize-functions/example-tshirt:v0.1.0 - network: - required: true -`, - required: true, - }, - } - - for _, tc := range tests { - cfg, err := yaml.Parse(tc.input) - if !assert.NoError(t, err) { - return - } - fn := GetFunctionSpec(cfg) - assert.Equal(t, tc.required, fn.Container.Network.Required) - } -} - -func TestFilter_Filter_defaultNaming(t *testing.T) { - cfg, err := yaml.Parse(`apiVersion: apps/v1 -kind: Deployment -metadata: - name: foo - annotations: - config.kubernetes.io/path: 'foo/bar.yaml' -`) - if !assert.NoError(t, err) { - return - } - - input, err := (&kio.ByteReader{Reader: bytes.NewBufferString(``)}).Read() - if !assert.NoError(t, err) { - return - } - - called := false - result, err := (&ContainerFilter{ - SetFlowStyleForConfig: true, - Image: "example.com:version", - Config: cfg, - args: []string{"echo", `apiVersion: apps/v1 -kind: Deployment -metadata: - name: deployment-foo ---- -apiVersion: v1 -kind: Service -metadata: - name: service-foo -`}, - checkInput: func(s string) { - called = true - if !assert.Equal(t, `apiVersion: config.kubernetes.io/v1alpha1 -kind: ResourceList -items: [] -functionConfig: {apiVersion: apps/v1, kind: Deployment, metadata: {name: foo, annotations: { - config.kubernetes.io/path: 'foo/bar.yaml'}}} -`, s) { - t.FailNow() - } - }, - }).Filter(input) - if !assert.NoError(t, err) { - return - } - if !assert.True(t, called) { - return - } - - b := &bytes.Buffer{} - err = kio.ByteWriter{Writer: b, KeepReaderAnnotations: true}.Write(result) - if !assert.NoError(t, err) { - return - } - - assert.Equal(t, `apiVersion: apps/v1 -kind: Deployment -metadata: - name: deployment-foo - annotations: - config.kubernetes.io/index: '0' - config.kubernetes.io/path: 'foo/deployment_deployment-foo.yaml' ---- -apiVersion: v1 -kind: Service -metadata: - name: service-foo - annotations: - config.kubernetes.io/index: '1' - config.kubernetes.io/path: 'foo/service_service-foo.yaml' -`, b.String()) -} - -func TestFilter_Filter_defaultNamingFunctions(t *testing.T) { - cfg, err := yaml.Parse(`apiVersion: apps/v1 -kind: Deployment -metadata: - name: foo - annotations: - config.kubernetes.io/path: 'foo/functions/bar.yaml' -`) - if !assert.NoError(t, err) { - return - } - - input, err := (&kio.ByteReader{Reader: bytes.NewBufferString(``)}).Read() - if !assert.NoError(t, err) { - return - } - - called := false - result, err := (&ContainerFilter{ - SetFlowStyleForConfig: true, - Image: "example.com:version", - Config: cfg, - args: []string{"echo", `apiVersion: apps/v1 -kind: Deployment -metadata: - name: deployment-foo ---- -apiVersion: v1 -kind: Service -metadata: - name: service-foo -`}, - checkInput: func(s string) { - called = true - if !assert.Equal(t, `apiVersion: config.kubernetes.io/v1alpha1 -kind: ResourceList -items: [] -functionConfig: {apiVersion: apps/v1, kind: Deployment, metadata: {name: foo, annotations: { - config.kubernetes.io/path: 'foo/functions/bar.yaml'}}} -`, s) { - t.FailNow() - } - }, - }).Filter(input) - if !assert.NoError(t, err) { - return - } - if !assert.True(t, called) { - return - } - - b := &bytes.Buffer{} - err = kio.ByteWriter{Writer: b, KeepReaderAnnotations: true}.Write(result) - if !assert.NoError(t, err) { - return - } - - assert.Equal(t, `apiVersion: apps/v1 -kind: Deployment -metadata: - name: deployment-foo - annotations: - config.kubernetes.io/index: '0' - config.kubernetes.io/path: 'foo/deployment_deployment-foo.yaml' ---- -apiVersion: v1 -kind: Service -metadata: - name: service-foo - annotations: - config.kubernetes.io/index: '1' - config.kubernetes.io/path: 'foo/service_service-foo.yaml' -`, b.String()) -} - -func TestFilter_Filter_scopeMissingFromResource(t *testing.T) { - cfg, err := yaml.Parse(`apiVersion: apps/v1 -kind: Deployment -metadata: - name: foo - annotations: - config.kubernetes.io/path: 'foo/bar.yaml' -`) - if !assert.NoError(t, err) { - return - } - - input, err := (&kio.ByteReader{Reader: bytes.NewBufferString(` -apiVersion: apps/v1 -kind: Deployment -metadata: - name: deployment-foo ---- -apiVersion: v1 -kind: Service -metadata: - name: service-foo -`)}).Read() - if !assert.NoError(t, err) { - return - } - - // no resources match the scope - called := false - result, err := (&ContainerFilter{ - SetFlowStyleForConfig: true, - Image: "example.com:version", - Config: cfg, - args: []string{"sed", "s/Deployment/StatefulSet/g"}, - checkInput: func(s string) { - called = true - if !assert.Equal(t, `apiVersion: config.kubernetes.io/v1alpha1 -kind: ResourceList -items: [] -functionConfig: {apiVersion: apps/v1, kind: Deployment, metadata: {name: foo, annotations: { - config.kubernetes.io/path: 'foo/bar.yaml'}}} -`, s) { - t.FailNow() - } - }, - }).Filter(input) - if !assert.NoError(t, err) { - return - } - if !assert.True(t, called) { - return - } - - b := &bytes.Buffer{} - err = kio.ByteWriter{Writer: b, KeepReaderAnnotations: true}.Write(result) - if !assert.NoError(t, err) { - return - } - - // Resources should be preserved -- paths shouldn't be set by container - assert.Equal(t, `apiVersion: apps/v1 -kind: Deployment -metadata: - name: deployment-foo - annotations: - config.kubernetes.io/index: '0' ---- -apiVersion: v1 -kind: Service -metadata: - name: service-foo - annotations: - config.kubernetes.io/index: '1' -`, b.String()) -} - -func TestFilter_Filter_globalScope(t *testing.T) { - cfg, err := yaml.Parse(`apiVersion: apps/v1 -kind: Deployment -metadata: - name: foo - annotations: - config.kubernetes.io/path: 'foo/bar.yaml' -`) - if !assert.NoError(t, err) { - return - } - - input, err := (&kio.ByteReader{Reader: bytes.NewBufferString(` -apiVersion: apps/v1 -kind: Deployment -metadata: - name: deployment-foo ---- -apiVersion: v1 -kind: Service -metadata: - name: service-foo -`)}).Read() - if !assert.NoError(t, err) { - return - } - - // no resources match the scope - called := false - result, err := (&ContainerFilter{ - SetFlowStyleForConfig: true, - GlobalScope: true, - Image: "example.com:version", - Config: cfg, - args: []string{"sed", "s/Deployment/StatefulSet/g"}, - checkInput: func(s string) { - called = true - if !assert.Equal(t, `apiVersion: config.kubernetes.io/v1alpha1 -kind: ResourceList -items: -- apiVersion: apps/v1 - kind: Deployment - metadata: - name: deployment-foo - annotations: - config.kubernetes.io/index: '0' -- apiVersion: v1 - kind: Service - metadata: - name: service-foo - annotations: - config.kubernetes.io/index: '1' -functionConfig: {apiVersion: apps/v1, kind: Deployment, metadata: {name: foo, annotations: { - config.kubernetes.io/path: 'foo/bar.yaml'}}} -`, s) { - t.FailNow() - } - }, - }).Filter(input) - if !assert.NoError(t, err) { - return - } - if !assert.True(t, called) { - return - } - - b := &bytes.Buffer{} - err = kio.ByteWriter{Writer: b, KeepReaderAnnotations: true}.Write(result) - if !assert.NoError(t, err) { - return - } - - // Resources should be preserved -- paths shouldn't be set by container - assert.Equal(t, `apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: deployment-foo - annotations: - config.kubernetes.io/index: '0' - config.kubernetes.io/path: 'foo/statefulset_deployment-foo.yaml' ---- -apiVersion: v1 -kind: Service -metadata: - name: service-foo - annotations: - config.kubernetes.io/index: '1' - config.kubernetes.io/path: 'foo/service_service-foo.yaml' -`, b.String()) -} - -func TestFilter_Filter_scopeFunctionsDir(t *testing.T) { - // functions under "functions/" dir should be scoped to parent dir - cfg, err := yaml.Parse(`apiVersion: apps/v1 -kind: Deployment -metadata: - name: foo - annotations: - config.kubernetes.io/path: 'foo/functions/bar.yaml' -`) - if !assert.NoError(t, err) { - return - } - - input, err := (&kio.ByteReader{Reader: bytes.NewBufferString(` -apiVersion: apps/v1 -kind: Deployment -metadata: - name: deployment-foo - annotations: - config.kubernetes.io/path: 'foo/bar/d.yaml' ---- -apiVersion: v1 -kind: Service -metadata: - name: service-foo - annotations: - config.kubernetes.io/path: 'foo/bar/s.yaml' -`)}).Read() - if !assert.NoError(t, err) { - return - } - - // no resources match the scope - called := false - result, err := (&ContainerFilter{ - SetFlowStyleForConfig: true, - Image: "example.com:version", - Config: cfg, - args: []string{"sed", "s/Deployment/StatefulSet/g"}, - checkInput: func(s string) { - called = true - if !assert.Equal(t, `apiVersion: config.kubernetes.io/v1alpha1 -kind: ResourceList -items: -- apiVersion: apps/v1 - kind: Deployment - metadata: - name: deployment-foo - annotations: - config.kubernetes.io/path: 'foo/bar/d.yaml' - config.kubernetes.io/index: '0' -- apiVersion: v1 - kind: Service - metadata: - name: service-foo - annotations: - config.kubernetes.io/path: 'foo/bar/s.yaml' - config.kubernetes.io/index: '1' -functionConfig: {apiVersion: apps/v1, kind: Deployment, metadata: {name: foo, annotations: { - config.kubernetes.io/path: 'foo/functions/bar.yaml'}}} -`, s) { - t.FailNow() - } - }, - }).Filter(input) - if !assert.NoError(t, err) { - return - } - if !assert.True(t, called) { - return - } - - b := &bytes.Buffer{} - err = kio.ByteWriter{Writer: b, KeepReaderAnnotations: true}.Write(result) - if !assert.NoError(t, err) { - return - } - - // Resources should be modified - assert.Equal(t, `apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: deployment-foo - annotations: - config.kubernetes.io/path: 'foo/bar/d.yaml' - config.kubernetes.io/index: '0' ---- -apiVersion: v1 -kind: Service -metadata: - name: service-foo - annotations: - config.kubernetes.io/path: 'foo/bar/s.yaml' - config.kubernetes.io/index: '1' -`, b.String()) -} - -func TestFilter_Filter_scope_nested_resource(t *testing.T) { - // functions under "functions/" dir should be scoped to parent dir - cfg, err := yaml.Parse(`apiVersion: apps/v1 -kind: Deployment -metadata: - name: foo - annotations: - config.kubernetes.io/path: 'baz.yaml' -`) - if !assert.NoError(t, err) { - return - } - - input, err := (&kio.ByteReader{Reader: bytes.NewBufferString(` -apiVersion: apps/v1 -kind: Deployment -metadata: - name: deployment-foo - annotations: - config.kubernetes.io/path: 'foo/bar/d.yaml' ---- -apiVersion: v1 -kind: Service -metadata: - name: service-foo - annotations: - config.kubernetes.io/path: 'foo/bar/s.yaml' -`)}).Read() - if !assert.NoError(t, err) { - return - } - - // no resources match the scope - called := false - result, err := (&ContainerFilter{ - SetFlowStyleForConfig: true, - Image: "example.com:version", - Config: cfg, - args: []string{"sed", "s/Deployment/StatefulSet/g"}, - checkInput: func(s string) { - called = true - if !assert.Equal(t, `apiVersion: config.kubernetes.io/v1alpha1 -kind: ResourceList -items: -- apiVersion: apps/v1 - kind: Deployment - metadata: - name: deployment-foo - annotations: - config.kubernetes.io/path: 'foo/bar/d.yaml' - config.kubernetes.io/index: '0' -- apiVersion: v1 - kind: Service - metadata: - name: service-foo - annotations: - config.kubernetes.io/path: 'foo/bar/s.yaml' - config.kubernetes.io/index: '1' -functionConfig: {apiVersion: apps/v1, kind: Deployment, metadata: {name: foo, annotations: { - config.kubernetes.io/path: 'baz.yaml'}}} -`, s) { - t.FailNow() - } - }, - }).Filter(input) - if !assert.NoError(t, err) { - return - } - if !assert.True(t, called) { - return - } - - b := &bytes.Buffer{} - err = kio.ByteWriter{Writer: b, KeepReaderAnnotations: true}.Write(result) - if !assert.NoError(t, err) { - return - } - - // Resources should be modified - assert.Equal(t, `apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: deployment-foo - annotations: - config.kubernetes.io/path: 'foo/bar/d.yaml' - config.kubernetes.io/index: '0' ---- -apiVersion: v1 -kind: Service -metadata: - name: service-foo - annotations: - config.kubernetes.io/path: 'foo/bar/s.yaml' - config.kubernetes.io/index: '1' -`, b.String()) -} - -func TestFilter_Filter_scopeDir(t *testing.T) { - // functions under "functions/" dir should be scoped to parent dir - cfg, err := yaml.Parse(`apiVersion: apps/v1 -kind: Deployment -metadata: - name: foo - annotations: - config.kubernetes.io/path: 'foo/bar.yaml' -`) - if !assert.NoError(t, err) { - return - } - - input, err := (&kio.ByteReader{Reader: bytes.NewBufferString(` -apiVersion: apps/v1 -kind: Deployment -metadata: - name: deployment-foo - annotations: - config.kubernetes.io/path: 'foo/bar/d.yaml' ---- -apiVersion: v1 -kind: Service -metadata: - name: service-foo - annotations: - config.kubernetes.io/path: 'foo/bar/s.yaml' -`)}).Read() - if !assert.NoError(t, err) { - return - } - - // no resources match the scope - called := false - result, err := (&ContainerFilter{ - SetFlowStyleForConfig: true, - Image: "example.com:version", - Config: cfg, - args: []string{"sed", "s/Deployment/StatefulSet/g"}, - checkInput: func(s string) { - called = true - if !assert.Equal(t, `apiVersion: config.kubernetes.io/v1alpha1 -kind: ResourceList -items: -- apiVersion: apps/v1 - kind: Deployment - metadata: - name: deployment-foo - annotations: - config.kubernetes.io/path: 'foo/bar/d.yaml' - config.kubernetes.io/index: '0' -- apiVersion: v1 - kind: Service - metadata: - name: service-foo - annotations: - config.kubernetes.io/path: 'foo/bar/s.yaml' - config.kubernetes.io/index: '1' -functionConfig: {apiVersion: apps/v1, kind: Deployment, metadata: {name: foo, annotations: { - config.kubernetes.io/path: 'foo/bar.yaml'}}} -`, s) { - t.FailNow() - } - }, - }).Filter(input) - if !assert.NoError(t, err) { - return - } - if !assert.True(t, called) { - return - } - - b := &bytes.Buffer{} - err = kio.ByteWriter{Writer: b, KeepReaderAnnotations: true}.Write(result) - if !assert.NoError(t, err) { - return - } - - // Resources should be preserved - assert.Equal(t, `apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: deployment-foo - annotations: - config.kubernetes.io/path: 'foo/bar/d.yaml' - config.kubernetes.io/index: '0' ---- -apiVersion: v1 -kind: Service -metadata: - name: service-foo - annotations: - config.kubernetes.io/path: 'foo/bar/s.yaml' - config.kubernetes.io/index: '1' -`, b.String()) -} - -func TestContainerFilter_scope(t *testing.T) { - cf := &ContainerFilter{} - - fnR, err := yaml.Parse(`apiVersion: config.kubernetes.io/v1beta1 -kind: ConfigFunction -metadata: - name: config-function - annotations: - config.kubernetes.io/path: 'functions/bar.yaml' -`) - if !assert.NoError(t, err) { - return - } - - inRs := []*yaml.RNode{fnR} - inScopeRs, notInScopeRs, err := cf.scope(".", inRs) - if !assert.NoError(t, err) { - return - } - assert.Len(t, inScopeRs, 1, "Number of in-scope Resources") - assert.Len(t, notInScopeRs, 0, "Number of out-of-scope Resources") -} diff --git a/kyaml/runfn/runfn.go b/kyaml/runfn/runfn.go index 6802259b7..5043b37c6 100644 --- a/kyaml/runfn/runfn.go +++ b/kyaml/runfn/runfn.go @@ -14,17 +14,19 @@ import ( "sync/atomic" "sigs.k8s.io/kustomize/kyaml/errors" + "sigs.k8s.io/kustomize/kyaml/fn/runtime/container" + "sigs.k8s.io/kustomize/kyaml/fn/runtime/exec" + "sigs.k8s.io/kustomize/kyaml/fn/runtime/runtimeutil" + "sigs.k8s.io/kustomize/kyaml/fn/runtime/starlark" "sigs.k8s.io/kustomize/kyaml/kio" - "sigs.k8s.io/kustomize/kyaml/kio/filters" "sigs.k8s.io/kustomize/kyaml/kio/kioutil" - "sigs.k8s.io/kustomize/kyaml/starlark" "sigs.k8s.io/kustomize/kyaml/yaml" ) // RunFns runs the set of configuration functions in a local directory against // the Resources in that directory type RunFns struct { - StorageMounts []filters.StorageMount + StorageMounts []runtimeutil.StorageMount // Path is the path to the directory containing functions Path string @@ -63,6 +65,9 @@ type RunFns struct { // EnableStarlark will enable functions run as starlark scripts EnableStarlark bool + // EnableExec will enable exec functions + EnableExec bool + // DisableContainers will disable functions run as containers DisableContainers bool @@ -75,7 +80,7 @@ type RunFns struct { // functionFilterProvider provides a filter to perform the function. // this is a variable so it can be mocked in tests functionFilterProvider func( - filter filters.FunctionSpec, api *yaml.RNode) (kio.Filter, error) + filter runtimeutil.FunctionSpec, api *yaml.RNode) (kio.Filter, error) } // Execute runs the command @@ -173,7 +178,7 @@ func (r RunFns) runFunctions( // check for deferred function errors var errs []string for i := range fltrs { - cf, ok := fltrs[i].(filters.DeferFailureFunction) + cf, ok := fltrs[i].(runtimeutil.DeferFailureFunction) if !ok { continue } @@ -196,7 +201,7 @@ func (r RunFns) getFunctionsFromInput(nodes []*yaml.RNode) ([]kio.Filter, error) buff := &kio.PackageBuffer{} err := kio.Pipeline{ Inputs: []kio.Reader{&kio.PackageBuffer{Nodes: nodes}}, - Filters: []kio.Filter{&filters.IsReconcilerFilter{}}, + Filters: []kio.Filter{&runtimeutil.IsReconcilerFilter{}}, Outputs: []kio.Writer{buff}, }.Execute() if err != nil { @@ -235,7 +240,7 @@ func (r RunFns) getFunctionFilters(global bool, fns ...*yaml.RNode) ( var fltrs []kio.Filter for i := range fns { api := fns[i] - spec := filters.GetFunctionSpec(api) + spec := runtimeutil.GetFunctionSpec(api) if spec.Container.Network.Required { if !r.Network { // TODO(eddiezane): Provide error info about which function needs the network @@ -251,9 +256,9 @@ func (r RunFns) getFunctionFilters(global bool, fns ...*yaml.RNode) ( if c == nil { continue } - cf, ok := c.(*filters.ContainerFilter) + cf, ok := c.(*container.Filter) if global && ok { - cf.GlobalScope = true + cf.Exec.GlobalScope = true } fltrs = append(fltrs, c) } @@ -331,25 +336,25 @@ func (r *RunFns) init() { } // ffp provides function filters -func (r *RunFns) ffp(spec filters.FunctionSpec, api *yaml.RNode) (kio.Filter, error) { +func (r *RunFns) ffp(spec runtimeutil.FunctionSpec, api *yaml.RNode) (kio.Filter, error) { + var resultsFile string + if r.ResultsDir != "" { + resultsFile = filepath.Join(r.ResultsDir, fmt.Sprintf( + "results-%v.yaml", r.resultsCount)) + atomic.AddUint32(&r.resultsCount, 1) + } if !r.DisableContainers && spec.Container.Image != "" { - var resultsFile string // TODO: Add a test for this behavior - - if r.ResultsDir != "" { - resultsFile = filepath.Join(r.ResultsDir, fmt.Sprintf( - "results-%v.yaml", r.resultsCount)) - atomic.AddUint32(&r.resultsCount, 1) - } - return &filters.ContainerFilter{ + cf := &container.Filter{ Image: spec.Container.Image, - Config: api, Network: spec.Network, StorageMounts: r.StorageMounts, - GlobalScope: r.GlobalScope, - ResultsFile: resultsFile, - DeferFailure: spec.DeferFailure, - }, nil + } + cf.Exec.FunctionConfig = api + cf.Exec.GlobalScope = r.GlobalScope + cf.Exec.ResultsFile = resultsFile + cf.Exec.DeferFailure = spec.DeferFailure + return cf, nil } if r.EnableStarlark && spec.Starlark.Path != "" { // the script path is relative to the function config file @@ -369,11 +374,24 @@ func (r *RunFns) ffp(spec filters.FunctionSpec, api *yaml.RNode) (kio.Filter, er } p = path.Join(r.Path, path.Dir(p), spec.Starlark.Path) - return &starlark.Filter{ - Name: spec.Starlark.Name, - Path: p, - FunctionConfig: api, - }, nil + sf := &starlark.Filter{Name: spec.Starlark.Name, Path: p} + + sf.FunctionConfig = api + sf.GlobalScope = r.GlobalScope + sf.ResultsFile = resultsFile + sf.DeferFailure = spec.DeferFailure + return sf, nil } + + if r.EnableExec && spec.Exec.Path != "" { + ef := &exec.Filter{Path: spec.Exec.Path} + + ef.FunctionConfig = api + ef.GlobalScope = r.GlobalScope + ef.ResultsFile = resultsFile + ef.DeferFailure = spec.DeferFailure + return ef, nil + } + return nil, nil } diff --git a/kyaml/runfn/runfn_test.go b/kyaml/runfn/runfn_test.go index e725bcb58..787641333 100644 --- a/kyaml/runfn/runfn_test.go +++ b/kyaml/runfn/runfn_test.go @@ -16,6 +16,8 @@ import ( "github.com/stretchr/testify/assert" "sigs.k8s.io/kustomize/kyaml/copyutil" "sigs.k8s.io/kustomize/kyaml/errors" + "sigs.k8s.io/kustomize/kyaml/fn/runtime/container" + "sigs.k8s.io/kustomize/kyaml/fn/runtime/runtimeutil" "sigs.k8s.io/kustomize/kyaml/kio" "sigs.k8s.io/kustomize/kyaml/kio/filters" "sigs.k8s.io/kustomize/kyaml/yaml" @@ -48,8 +50,8 @@ func TestRunFns_init(t *testing.T) { api, err := yaml.Parse(`apiVersion: apps/v1 kind: `) - spec := filters.FunctionSpec{ - Container: filters.ContainerSpec{ + spec := runtimeutil.FunctionSpec{ + Container: runtimeutil.ContainerSpec{ Image: "example.com:version", }, } @@ -57,7 +59,9 @@ kind: return } filter, _ := instance.functionFilterProvider(spec, api) - assert.Equal(t, &filters.ContainerFilter{Image: "example.com:version", Config: api}, filter) + cf := &container.Filter{Image: "example.com:version"} + cf.Exec.FunctionConfig = api + assert.Equal(t, cf, filter) } func TestRunFns_Execute__initGlobalScope(t *testing.T) { @@ -76,8 +80,8 @@ kind: return } - spec := filters.FunctionSpec{ - Container: filters.ContainerSpec{ + spec := runtimeutil.FunctionSpec{ + Container: runtimeutil.ContainerSpec{ Image: "example.com:version", }, } @@ -85,8 +89,10 @@ kind: return } filter, _ := instance.functionFilterProvider(spec, api) - assert.Equal(t, &filters.ContainerFilter{ - Image: "example.com:version", Config: api, GlobalScope: true}, filter) + cf := &container.Filter{Image: "example.com:version"} + cf.Exec.FunctionConfig = api + cf.Exec.GlobalScope = true + assert.Equal(t, cf, filter) } func TestRunFns_Execute__initDefault(t *testing.T) { @@ -143,12 +149,12 @@ func TestRunFns_Execute__initDefault(t *testing.T) { }, { name: "explicit directories in mounts", - instance: RunFns{StorageMounts: []filters.StorageMount{{MountType: "volume", Src: "myvol", DstPath: "/local/"}}}, + instance: RunFns{StorageMounts: []runtimeutil.StorageMount{{MountType: "volume", Src: "myvol", DstPath: "/local/"}}}, expected: RunFns{ Output: os.Stdout, Input: os.Stdin, NoFunctionsFromInput: getFalse(), - StorageMounts: []filters.StorageMount{{MountType: "volume", Src: "myvol", DstPath: "/local/"}}, + StorageMounts: []runtimeutil.StorageMount{{MountType: "volume", Src: "myvol", DstPath: "/local/"}}, }, }, } @@ -751,7 +757,7 @@ replace: StatefulSet var fltrs []*TestFilter instance := RunFns{ Path: dir, - functionFilterProvider: func(f filters.FunctionSpec, node *yaml.RNode) (kio.Filter, error) { + functionFilterProvider: func(f runtimeutil.FunctionSpec, node *yaml.RNode) (kio.Filter, error) { tf := &TestFilter{ Exit: errors.Errorf("message: %s", f.Container.Image), } @@ -927,8 +933,8 @@ func setupTest(t *testing.T) string { // getFilterProvider fakes the creation of a filter, replacing the ContainerFiler with // a filter to s/kind: Deployment/kind: StatefulSet/g. // this can be used to simulate running a filter. -func getFilterProvider(t *testing.T) func(filters.FunctionSpec, *yaml.RNode) (kio.Filter, error) { - return func(f filters.FunctionSpec, node *yaml.RNode) (kio.Filter, error) { +func getFilterProvider(t *testing.T) func(runtimeutil.FunctionSpec, *yaml.RNode) (kio.Filter, error) { + return func(f runtimeutil.FunctionSpec, node *yaml.RNode) (kio.Filter, error) { // parse the filter from the input filter := yaml.YFilter{} b := &bytes.Buffer{} diff --git a/kyaml/starlark/starlark.go b/kyaml/starlark/starlark.go deleted file mode 100644 index e673a754f..000000000 --- a/kyaml/starlark/starlark.go +++ /dev/null @@ -1,251 +0,0 @@ -// Copyright 2019 The Kubernetes Authors. -// SPDX-License-Identifier: Apache-2.0 - -package starlark - -import ( - "fmt" - "io/ioutil" - "net/http" - "strconv" - - "github.com/qri-io/starlib/util" - "go.starlark.net/starlark" - "sigs.k8s.io/kustomize/kyaml/comments" - "sigs.k8s.io/kustomize/kyaml/errors" - "sigs.k8s.io/kustomize/kyaml/kio/filters" - "sigs.k8s.io/kustomize/kyaml/yaml" -) - -// Filter transforms a set of resources through the provided program -type Filter struct { - Name string - - // Program is a starlark script which will be run against the resources - Program string - - // URL is the url of a starlark program to fetch and run - URL string - - // Path is the path to a starlark program to read and run - Path string - - // FunctionConfig is the value to be provided for resourceList.functionConfig as specified by - // https://github.com/kubernetes-sigs/kustomize/blob/master/cmd/config/docs/api-conventions/functions-spec.md. - FunctionConfig *yaml.RNode -} - -func (sf *Filter) String() string { - return fmt.Sprintf("name: %v path: %v url: %v program: %v", sf.Name, sf.Path, sf.URL, sf.Program) -} - -func (sf *Filter) Filter(input []*yaml.RNode) ([]*yaml.RNode, error) { - if sf.URL != "" && sf.Path != "" || - sf.URL != "" && sf.Program != "" || - sf.Path != "" && sf.Program != "" { - return nil, errors.Errorf("Filter Path, Program and URL are mutually exclusive") - } - - // read the program from a file - if sf.Path != "" { - b, err := ioutil.ReadFile(sf.Path) - if err != nil { - return nil, err - } - sf.Program = string(b) - } - - // read the program from a URL - if sf.URL != "" { - err := func() error { - resp, err := http.Get(sf.URL) - if err != nil { - return err - } - defer resp.Body.Close() - b, err := ioutil.ReadAll(resp.Body) - if err != nil { - return err - } - sf.Program = string(b) - return nil - }() - if err != nil { - return nil, err - } - } - - // retain map of inputs to outputs by id so if the name is changed by the - // starlark program, we are able to match the same resources - value, ids, err := sf.inputToResourceList(input) - if err != nil { - return nil, errors.Wrap(err) - } - - // run the starlark as program as transformation function - thread := &starlark.Thread{Name: sf.Name} - - ctx := &Context{ - resourceList: value, - } - pd, err := ctx.predeclared() - if err != nil { - return nil, errors.Wrap(err) - } - _, err = starlark.ExecFile(thread, sf.Name, sf.Program, pd) - if err != nil { - return nil, errors.Wrap(err) - } - - results, err := sf.resourceListToOutput(value, ids) - if err != nil { - return nil, errors.Wrap(err) - } - - // starlark will serialize the resources sorting the fields alphabetically, - // format them to have a better ordering - return filters.FormatFilter{}.Filter(results) -} - -// tuple maps an input resource to the output resource -type tuple struct { - // in is the RNode provided to the starlark program - in *yaml.RNode - // out is the RNode emitted by the starlark program with the id matching in - out *yaml.RNode -} - -// inputToResourceList transforms input into a starlark.Value -func (sf *Filter) inputToResourceList( - input []*yaml.RNode) (starlark.Value, map[int]*tuple, error) { - var id int - ids := map[int]*tuple{} - - // convert into a ResourceList which will be converted to a starlark dictionary - // create the ResourceList - resourceList, err := yaml.Parse(`kind: ResourceList`) - if err != nil { - return nil, nil, errors.Wrap(err) - } - - // set the functionConfig - if sf.FunctionConfig != nil { - if err := resourceList.PipeE( - yaml.FieldSetter{Name: "functionConfig", Value: sf.FunctionConfig}); err != nil { - return nil, nil, err - } - } - - // the inputs should be provided as the list "items" - items, err := resourceList.Pipe(yaml.LookupCreate(yaml.SequenceNode, "items")) - if err != nil { - return nil, nil, errors.Wrap(err) - } - // add the input as items, give each resource an id - for i := range input { - item := input[i] - - // create an id for tracking the resource through the program - err := item.PipeE(yaml.SetAnnotation("config.k8s.io/id", fmt.Sprintf("%d", id))) - if err != nil { - return nil, nil, errors.Wrap(err) - } - ids[id] = &tuple{in: item} - id++ - - items.YNode().Content = append(items.YNode().Content, item.YNode()) - } - - // convert the ResourceList into a starlark dictionary by - // first converting it into a map[string]interface{} - value, err := nodeToValue(resourceList) - return value, ids, err -} - -// resourceListToOutput converts the output of the starlark program to the filter output -func (sf *Filter) resourceListToOutput( - value starlark.Value, ids map[int]*tuple) ([]*yaml.RNode, error) { - // convert the modified resourceList back into a slice of RNodes - // by first converting to a map[string]interface{} - out, err := util.Unmarshal(value) - if err != nil { - return nil, errors.Wrap(err) - } - o := out.(map[string]interface{}) - - // parse the function config - if _, found := o["functionConfig"]; found { - fc := (o["functionConfig"].(map[string]interface{})) - b, err := yaml.Marshal(fc) - if err != nil { - return nil, errors.Wrap(err) - } - sf.FunctionConfig, err = yaml.Parse(string(b)) - if err != nil { - return nil, errors.Wrap(err) - } - } - - // parse the items - // copy the items out of the ResourceList, and into the Filter output - var results []*yaml.RNode - it := (o["items"].([]interface{})) - for i := range it { - // convert the resource back to the native yaml form - b, err := yaml.Marshal(it[i]) - if err != nil { - return nil, errors.Wrap(err) - } - node, err := yaml.Parse(string(b)) - if err != nil { - return nil, errors.Wrap(err) - } - - // match it to an input - idS, err := node.Pipe(yaml.GetAnnotation("config.k8s.io/id")) - if err != nil { - return nil, errors.Wrap(err) - } - if idS == nil { - // no matching input -- new resource - results = append(results, node) - continue - } - - id, err := strconv.Atoi(idS.YNode().Value) - if err != nil { - return nil, errors.Wrap(err) - } - if match, found := ids[id]; found { - // matching resources - match.out = node - } else { - // no matching input with the same id -- new resource - // this may be an error case, the outputs probably shouldn't have ids - // assigned by the starlark program - results = append(results, node) - } - } - - // retain the comments instead of dropping them by copying them from the original inputs - for i := 0; i < len(ids); i++ { - v := ids[i] - if v.out == nil { - continue - } - if err := comments.CopyComments(v.in, v.out); err != nil { - return nil, errors.Wrap(err) - } - results = append(results, v.out) - } - - // delete the ids from resources, these were only to track through the starlark program - // and that is finished. - for i := range results { - err := results[i].PipeE(yaml.ClearAnnotation("config.k8s.io/id")) - if err != nil { - return nil, err - } - } - return results, nil -} diff --git a/plugin/someteam.example.com/v1/chartinflator/ChartInflator b/plugin/someteam.example.com/v1/chartinflator/ChartInflator index 37aa63d16..e71b475ca 100755 --- a/plugin/someteam.example.com/v1/chartinflator/ChartInflator +++ b/plugin/someteam.example.com/v1/chartinflator/ChartInflator @@ -17,7 +17,7 @@ # chartVersion: 9.0.1 # helmHome: /abs/path/to/helm/config # helmBin: /abs/path/to/helmBin -# releaseNam: nameOfHelmRelease +# releaseName: nameOfHelmRelease # releaseNamespace: namespaceWhereHelmWouldApply # # fetches the given chart from stable/$chartName, diff --git a/travis/kyaml-pre-commit.sh b/travis/kyaml-pre-commit.sh index 4f1f93f21..8d9c00403 100755 --- a/travis/kyaml-pre-commit.sh +++ b/travis/kyaml-pre-commit.sh @@ -18,6 +18,9 @@ functions/examples/validator-resource-requests functions/examples/application-cr " +# don't run e2e tests in CI by default +export KUSTOMIZE_DOCKER_E2E=${KUSTOMIZE_DOCKER_E2E:-"false"} + for target in $targets; do echo "----- Making $target -----" pushd .