From 5364b2198a90a2c528e124ae7c1a09370219f1f6 Mon Sep 17 00:00:00 2001 From: Phillip Wittrock Date: Sun, 8 Mar 2020 16:53:02 -0700 Subject: [PATCH] Add Starlark kio.Filter - Support for implementing filters in Starlark --- cmd/config/go.sum | 17 +++ kyaml/comments/comments.go | 56 +++++++ kyaml/go.mod | 2 + kyaml/go.sum | 25 +++ kyaml/starlark/doc.go | 26 ++++ kyaml/starlark/example_test.go | 93 ++++++++++++ kyaml/starlark/starlark.go | 180 ++++++++++++++++++++++ kyaml/starlark/starlark_test.go | 261 ++++++++++++++++++++++++++++++++ 8 files changed, 660 insertions(+) create mode 100644 kyaml/comments/comments.go create mode 100644 kyaml/starlark/doc.go create mode 100644 kyaml/starlark/example_test.go create mode 100644 kyaml/starlark/starlark.go create mode 100644 kyaml/starlark/starlark_test.go diff --git a/cmd/config/go.sum b/cmd/config/go.sum index a499dd06c..020858148 100644 --- a/cmd/config/go.sum +++ b/cmd/config/go.sum @@ -1,12 +1,18 @@ +github.com/360EntSecGroup-Skylar/excelize v1.4.1/go.mod h1:vnax29X2usfl7HHkBrX5EvSCJcmH3dT9luvxzu8iGAE= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= +github.com/PuerkitoBio/goquery v1.5.0/go.mod h1:qD2PgZ9lccMbQlc7eEOjaeRlFQON7xY8kdmcsrnKqMg= github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI= github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= +github.com/andybalholm/cascadia v1.0.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= @@ -16,6 +22,7 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= +github.com/dustmop/soup v1.1.2-0.20190516214245-38228baa104e/go.mod h1:CgNC6SGbT+Xb8wGGvzilttZL1mc5sQ/5KkcxsZttMIk= github.com/elazarl/goproxy v0.0.0-20170405201442-c4fc26588b6e/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= @@ -82,6 +89,7 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJ github.com/modern-go/reflect2 v0.0.0-20180320133207-05fbef0ca5da/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8= github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= github.com/olekukonko/tablewriter v0.0.4 h1:vHD/YYe1Wolo78koG299f7V/VAS08c6IpCLn+Ejf/w8= @@ -91,6 +99,7 @@ github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+W github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/paulmach/orb v0.1.3/go.mod h1:VFlX/8C+IQ1p6FTRRKzKoOPJnvEtA5G0Veuqwbu//Vk= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -100,6 +109,7 @@ github.com/posener/complete/v2 v2.0.1-alpha.12 h1:0wvkuDfHb5vSZlNBYgpEH4XQHpF46M github.com/posener/complete/v2 v2.0.1-alpha.12/go.mod h1://JlL91cS2JV7rOl6LVHrRqBXoBUecJu3ILQPgbJiMQ= github.com/posener/script v1.0.4 h1:nSuXW5ZdmFnQIueLB2s0qvs4oNsUloM1Zydzh75v42w= github.com/posener/script v1.0.4/go.mod h1:Rg3ijooqulo05aGLyGsHoLmIOUzHUVK19WVgrYBPU/E= +github.com/qri-io/starlib v0.4.2-0.20200213133954-ff2e8cd5ef8d/go.mod h1:7DPO4domFU579Ga6E61sB9VFNaniPVwJP5C4bBCu3wA= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= @@ -116,6 +126,7 @@ github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DM github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v0.0.0-20151208002404-e3a8ff8ce365/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.2.3-0.20181224173747-660f15d67dbb/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= @@ -123,10 +134,15 @@ github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljT github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca h1:1CFlNzQhALwjS9mBAUkycX616GzgsuYUOCHA5+HSlXI= github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +go.starlark.net v0.0.0-20190528202925-30ae18b8564f/go.mod h1:c1/X6cHgvdXj6pUlmWKMkuqRnW4K8x2vwt6JAaaircg= +go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5/go.mod h1:nmDLcffg48OtT/PSW0Hg7FvpRQsQh5OSqIylirxKC7o= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191004110552-13f9640d40b9 h1:rjwSpXsdiK0dV8/Naq3kAw9ymfAeJIyd0upUIElB+lI= @@ -137,6 +153,7 @@ golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191002063906-3421d5a6bb1c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= diff --git a/kyaml/comments/comments.go b/kyaml/comments/comments.go new file mode 100644 index 000000000..e7547f15e --- /dev/null +++ b/kyaml/comments/comments.go @@ -0,0 +1,56 @@ +// Copyright 2019 The Kubernetes Authors. +// SPDX-License-Identifier: Apache-2.0 + +package comments + +import ( + "sigs.k8s.io/kustomize/kyaml/openapi" + "sigs.k8s.io/kustomize/kyaml/yaml" + "sigs.k8s.io/kustomize/kyaml/yaml/walk" +) + +// CopyComments recursively copies the comments on fields in from to fields in to +func CopyComments(from, to *yaml.RNode) error { + // walk the fields copying comments + _, err := walk.Walker{ + Sources: []*yaml.RNode{from, to}, + Visitor: &copier{}, + VisitKeysAsScalars: true}.Walk() + return err +} + +// copier implements walk.Visitor, and copies comments to fields shared between 2 instances +// of a resource +type copier struct{} + +func (c *copier) VisitMap(s walk.Sources, _ *openapi.ResourceSchema) (*yaml.RNode, error) { + copy(s.Dest(), s.Origin()) + return s.Dest(), nil +} + +func (c *copier) VisitScalar(s walk.Sources, _ *openapi.ResourceSchema) (*yaml.RNode, error) { + copy(s.Dest(), s.Origin()) + return s.Dest(), nil +} + +func (c *copier) VisitList(s walk.Sources, _ *openapi.ResourceSchema, _ walk.ListKind) ( + *yaml.RNode, error) { + copy(s.Dest(), s.Origin()) + return s.Dest(), nil +} + +// copy copies the comment from one field to another +func copy(from, to *yaml.RNode) { + if from == nil || to == nil { + return + } + if from.YNode().LineComment != "" { + to.YNode().LineComment = from.YNode().LineComment + } + if from.YNode().HeadComment != "" { + to.YNode().HeadComment = from.YNode().HeadComment + } + if from.YNode().FootComment != "" { + to.YNode().FootComment = from.YNode().FootComment + } +} diff --git a/kyaml/go.mod b/kyaml/go.mod index 33eefaad1..fa1886b41 100644 --- a/kyaml/go.mod +++ b/kyaml/go.mod @@ -6,9 +6,11 @@ require ( github.com/davecgh/go-spew v1.1.1 github.com/go-errors/errors v1.0.1 github.com/go-openapi/spec v0.19.5 + github.com/qri-io/starlib v0.4.2-0.20200213133954-ff2e8cd5ef8d github.com/sergi/go-diff v1.1.0 github.com/stretchr/testify v1.4.0 github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca + go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5 golang.org/x/net v0.0.0-20191004110552-13f9640d40b9 // indirect gopkg.in/yaml.v2 v2.2.7 gopkg.in/yaml.v3 v3.0.0-20191120175047-4206685974f2 diff --git a/kyaml/go.sum b/kyaml/go.sum index 05b1b96aa..cdffe44f9 100644 --- a/kyaml/go.sum +++ b/kyaml/go.sum @@ -1,10 +1,21 @@ +github.com/360EntSecGroup-Skylar/excelize v1.4.1 h1:l55mJb6rkkaUzOpSsgEeKYtS6/0gHwBYyfo5Jcjv/Ks= +github.com/360EntSecGroup-Skylar/excelize v1.4.1/go.mod h1:vnax29X2usfl7HHkBrX5EvSCJcmH3dT9luvxzu8iGAE= +github.com/PuerkitoBio/goquery v1.5.0 h1:uGvmFXOA73IKluu/F84Xd1tt/z07GYm8X49XKHP7EJk= +github.com/PuerkitoBio/goquery v1.5.0/go.mod h1:qD2PgZ9lccMbQlc7eEOjaeRlFQON7xY8kdmcsrnKqMg= github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI= github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= +github.com/andybalholm/cascadia v1.0.0 h1:hOCXnnZ5A+3eVDX8pvgl4kofXv2ELss0bKcqRySc45o= +github.com/andybalholm/cascadia v1.0.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dustmop/soup v1.1.2-0.20190516214245-38228baa104e h1:44fmjqDtdCiUNlSjJVp+w1AOs6na3Y6Ai0aIeseFjkI= +github.com/dustmop/soup v1.1.2-0.20190516214245-38228baa104e/go.mod h1:CgNC6SGbT+Xb8wGGvzilttZL1mc5sQ/5KkcxsZttMIk= github.com/go-errors/errors v1.0.1 h1:LUHzmkK3GUKUrL/1gfBUxAHzcev3apQlezX/+O7ma6w= github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= @@ -25,23 +36,37 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e h1:hB2xlXdHp/pmPZq0y3QnmWAArdw9PqbmotexnWx/FU8= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw= +github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8= +github.com/paulmach/orb v0.1.3 h1:Wa1nzU269Zv7V9paVEY1COWW8FCqv4PC/KJRbJSimpM= +github.com/paulmach/orb v0.1.3/go.mod h1:VFlX/8C+IQ1p6FTRRKzKoOPJnvEtA5G0Veuqwbu//Vk= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/qri-io/starlib v0.4.2-0.20200213133954-ff2e8cd5ef8d h1:K6eOUihrFLdZjZnA4XlRp864fmWXv9YTIk7VPLhRacA= +github.com/qri-io/starlib v0.4.2-0.20200213133954-ff2e8cd5ef8d/go.mod h1:7DPO4domFU579Ga6E61sB9VFNaniPVwJP5C4bBCu3wA= github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.3-0.20181224173747-660f15d67dbb/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca h1:1CFlNzQhALwjS9mBAUkycX616GzgsuYUOCHA5+HSlXI= github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg= +go.starlark.net v0.0.0-20190528202925-30ae18b8564f/go.mod h1:c1/X6cHgvdXj6pUlmWKMkuqRnW4K8x2vwt6JAaaircg= +go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5 h1:+FNtrFTmVw0YZGpBGX56XDee331t6JAXeK2bcyhLOOc= +go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5/go.mod h1:nmDLcffg48OtT/PSW0Hg7FvpRQsQh5OSqIylirxKC7o= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297 h1:k7pJ2yAPLPgbskkFdhRCsA77k2fySZ1zf2zCjvQCiIM= golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191004110552-13f9640d40b9 h1:rjwSpXsdiK0dV8/Naq3kAw9ymfAeJIyd0upUIElB+lI= golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20191002063906-3421d5a6bb1c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= diff --git a/kyaml/starlark/doc.go b/kyaml/starlark/doc.go new file mode 100644 index 000000000..734ec3fdb --- /dev/null +++ b/kyaml/starlark/doc.go @@ -0,0 +1,26 @@ +// Copyright 2019 The Kubernetes Authors. +// SPDX-License-Identifier: Apache-2.0 + +// Package starlark contains a kio.Filter which can be applied to resources to transform +// them through starlark program. +// +// The resources are provided to the program through the global variable "resourceList". +// "resourceList" is a dictionary containing an "items" field with a list of resources. +// Changes to "resourceList" made by the starlark program will be reflected in the Filter output. +// +// After being run through the starlark program, the filter will copy the comments from the input +// resources to restore them after they are dropped due to the serialization. +// +// The Filter will also format the output so that output has the preferred field ordering +// rather than an alphabetical field ordering. +// +// The resourceList variable adheres to the kustomize function spec as specified by: +// https://github.com/kubernetes-sigs/kustomize/blob/master/cmd/config/docs/api-conventions/functions-spec.md +// +// All items in the resourceList are resources represented as starlark dictionaries/ +// The items in the resourceList respect the io spec specified by: +// https://github.com/kubernetes-sigs/kustomize/blob/master/cmd/config/docs/api-conventions/config-io.md +// +// The starlark language spec can be found here: +// https://github.com/google/starlark-go/blob/master/doc/spec.md +package starlark diff --git a/kyaml/starlark/example_test.go b/kyaml/starlark/example_test.go new file mode 100644 index 000000000..bf9a8b004 --- /dev/null +++ b/kyaml/starlark/example_test.go @@ -0,0 +1,93 @@ +// Copyright 2019 The Kubernetes Authors. +// SPDX-License-Identifier: Apache-2.0 + +package starlark_test + +import ( + "bytes" + "fmt" + "log" + + "sigs.k8s.io/kustomize/kyaml/kio" + "sigs.k8s.io/kustomize/kyaml/starlark" +) + +func ExampleFilter_Filter() { + // input contains the items that will provided to the starlark program + input := bytes.NewBufferString(` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: deployment-1 +spec: + template: + spec: + containers: + - name: nginx + image: nginx:1.8.1 # {"$ref": "#/definitions/io.k8s.cli.substitutions.image-1"} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: deployment-2 +spec: + template: + spec: + containers: + - name: nginx + image: nginx:1.7.9 # {"$ref": "#/definitions/io.k8s.cli.substitutions.image-2"} +`) + + // fltr transforms the input using a starlark program + fltr := &starlark.Filter{ + Name: "annotate", + Program: ` +def run(items): + for item in items: + item["metadata"]["annotations"]["foo"] = "bar" + +run(resourceList["items"]) +`, + } + + // output contains the transformed resources + output := &bytes.Buffer{} + + // run the fltr against the inputs using a kio.Pipeline + err := kio.Pipeline{ + Inputs: []kio.Reader{&kio.ByteReader{Reader: input}}, + Filters: []kio.Filter{fltr}, + Outputs: []kio.Writer{&kio.ByteWriter{Writer: output}}}.Execute() + if err != nil { + log.Fatal(err) + } + + fmt.Println(output.String()) + + // Output: + // apiVersion: apps/v1 + // kind: Deployment + // metadata: + // name: deployment-1 + // annotations: + // foo: bar + // spec: + // template: + // spec: + // containers: + // - name: nginx + // image: nginx:1.8.1 # {"$ref": "#/definitions/io.k8s.cli.substitutions.image-1"} + //--- + // apiVersion: apps/v1 + // kind: Deployment + // metadata: + // name: deployment-2 + // annotations: + // foo: bar + // spec: + // template: + // spec: + // containers: + // - name: nginx + // image: nginx:1.7.9 # {"$ref": "#/definitions/io.k8s.cli.substitutions.image-2"} +} diff --git a/kyaml/starlark/starlark.go b/kyaml/starlark/starlark.go new file mode 100644 index 000000000..d888ebd7a --- /dev/null +++ b/kyaml/starlark/starlark.go @@ -0,0 +1,180 @@ +// Copyright 2019 The Kubernetes Authors. +// SPDX-License-Identifier: Apache-2.0 + +package starlark + +import ( + "fmt" + "strconv" + + "github.com/qri-io/starlib/util" + "go.starlark.net/starlark" + "sigs.k8s.io/kustomize/kyaml/comments" + "sigs.k8s.io/kustomize/kyaml/errors" + "sigs.k8s.io/kustomize/kyaml/kio/filters" + "sigs.k8s.io/kustomize/kyaml/yaml" +) + +// Filter transforms a set of resources through the provided program +type Filter struct { + Name string + + // Program is a starlark script which will be run against the resources + Program string +} + +func (sf *Filter) Filter(input []*yaml.RNode) ([]*yaml.RNode, error) { + // retain map of inputs to outputs by id so if the name is changed by the + // starlark program, we are able to match the same resources + value, ids, err := sf.inputToResourceList(input) + if err != nil { + return nil, errors.Wrap(err) + } + + // run the starlark as program as transformation function + thread := &starlark.Thread{Name: sf.Name} + predeclared := starlark.StringDict{"resourceList": value} + _, err = starlark.ExecFile(thread, sf.Name, sf.Program, predeclared) + if err != nil { + return nil, errors.Wrap(err) + } + + results, err := sf.resourceListToOutput(value, ids) + if err != nil { + return nil, errors.Wrap(err) + } + + // starlark will serialize the resources sorting the fields alphabetically, + // format them to have a better ordering + return filters.FormatFilter{}.Filter(results) +} + +// tuple maps an input resource to the output resource +type tuple struct { + // in is the RNode provided to the starlark program + in *yaml.RNode + // out is the RNode emitted by the starlark program with the id matching in + out *yaml.RNode +} + +// inputToResourceList transforms input into a starlark.Value +func (sf *Filter) inputToResourceList( + input []*yaml.RNode) (starlark.Value, map[int]*tuple, error) { + var id int + ids := map[int]*tuple{} + + // convert into a ResourceList which will be converted to a starlark dictionary + // create the ResourceList + resourceList, err := yaml.Parse(`kind: ResourceList`) + if err != nil { + return nil, nil, errors.Wrap(err) + } + items, err := resourceList.Pipe(yaml.LookupCreate(yaml.SequenceNode, "items")) + if err != nil { + return nil, nil, errors.Wrap(err) + } + // add the input as items, give each resource an id + for i := range input { + item := input[i] + + // create an id for tracking the resource through the program + err := item.PipeE(yaml.SetAnnotation("config.k8s.io/id", fmt.Sprintf("%d", id))) + if err != nil { + return nil, nil, errors.Wrap(err) + } + ids[id] = &tuple{in: item} + id++ + + items.YNode().Content = append(items.YNode().Content, item.YNode()) + } + + // convert the ResourceList into a starlark dictionary by + // first converting it into a map[string]interface{} + s, err := resourceList.String() + if err != nil { + return nil, nil, errors.Wrap(err) + } + var in map[string]interface{} + if err := yaml.Unmarshal([]byte(s), &in); err != nil { + return nil, nil, errors.Wrap(err) + } + value, err := util.Marshal(in) + if err != nil { + return nil, nil, errors.Wrap(err) + } + return value, ids, err +} + +// resourceListToOutput converts the output of the starlark program to the filter output +func (sf *Filter) resourceListToOutput( + value starlark.Value, ids map[int]*tuple) ([]*yaml.RNode, error) { + // convert the modified resourceList back into a slice of RNodes + // by first converting to a map[string]interface{} + out, err := util.Unmarshal(value) + if err != nil { + return nil, errors.Wrap(err) + } + o := out.(map[string]interface{}) + it := (o["items"].([]interface{})) + + // copy the items out of the ResourceList, and into the Filter output + var results []*yaml.RNode + for i := range it { + // convert the resource back to the native yaml form + b, err := yaml.Marshal(it[i]) + if err != nil { + return nil, errors.Wrap(err) + } + node, err := yaml.Parse(string(b)) + if err != nil { + return nil, errors.Wrap(err) + } + + // match it to an input + idS, err := node.Pipe(yaml.GetAnnotation("config.k8s.io/id")) + if err != nil { + return nil, errors.Wrap(err) + } + if idS == nil { + // no matching input -- new resource + results = append(results, node) + continue + } + + id, err := strconv.Atoi(idS.YNode().Value) + if err != nil { + return nil, errors.Wrap(err) + } + if match, found := ids[id]; found { + // matching resources + match.out = node + } else { + // no matching input with the same id -- new resource + // this may be an error case, the outputs probably shouldn't have ids + // assigned by the starlark program + results = append(results, node) + } + } + + // retain the comments instead of dropping them by copying them from the original inputs + for i := 0; i < len(ids); i++ { + v := ids[i] + if v.out == nil { + continue + } + if err := comments.CopyComments(v.in, v.out); err != nil { + return nil, errors.Wrap(err) + } + results = append(results, v.out) + } + + // delete the ids from resources, these were only to track through the starlark program + // and that is finished. + for i := range results { + err := results[i].PipeE(yaml.ClearAnnotation("config.k8s.io/id")) + if err != nil { + return nil, err + } + } + return results, nil +} diff --git a/kyaml/starlark/starlark_test.go b/kyaml/starlark/starlark_test.go new file mode 100644 index 000000000..cb761aeb2 --- /dev/null +++ b/kyaml/starlark/starlark_test.go @@ -0,0 +1,261 @@ +// Copyright 2019 The Kubernetes Authors. +// SPDX-License-Identifier: Apache-2.0 + +package starlark + +import ( + "bytes" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "sigs.k8s.io/kustomize/kyaml/kio" +) + +func TestStarlarkFilter_Filter(t *testing.T) { + var tests = []struct { + name string + input string + script string + expected string + }{ + { + name: "add_annotation", + input: ` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nginx-deployment +spec: + template: + spec: + containers: + - name: nginx + # head comment + image: nginx:1.8.1 # {"$ref": "#/definitions/io.k8s.cli.substitutions.image"} +`, + script: ` +# set the foo annotation on each resource +def run(r): + for resource in r: + resource["metadata"]["annotations"]["foo"] = "bar" + +run(resourceList["items"]) +`, + expected: ` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nginx-deployment + annotations: + foo: bar +spec: + template: + spec: + containers: + - name: nginx + # head comment + image: nginx:1.8.1 # {"$ref": "#/definitions/io.k8s.cli.substitutions.image"} +`, + }, + { + name: "update_annotation", + input: ` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nginx-deployment + annotations: + foo: baz +spec: + template: + spec: + containers: + - name: nginx + # head comment + image: nginx:1.8.1 # {"$ref": "#/definitions/io.k8s.cli.substitutions.image"} +`, + script: ` +# set the foo annotation on each resource +def run(r): + for resource in r: + resource["metadata"]["annotations"]["foo"] = "bar" + +run(resourceList["items"]) +`, + expected: ` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nginx-deployment + annotations: + foo: bar +spec: + template: + spec: + containers: + - name: nginx + # head comment + image: nginx:1.8.1 # {"$ref": "#/definitions/io.k8s.cli.substitutions.image"} +`, + }, + { + name: "update_annotation_multiple", + input: ` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nginx-deployment-1 + annotations: + foo: baz +spec: + template: + spec: + containers: + - name: nginx + # head comment + image: nginx:1.8.1 # {"$ref": "#/definitions/io.k8s.cli.substitutions.image"} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nginx-deployment-2 +spec: + template: + spec: + containers: + - name: nginx + # head comment + image: nginx:1.8.1 # {"$ref": "#/definitions/io.k8s.cli.substitutions.image"} +`, + script: ` +# set the foo annotation on each resource +def run(r): + for resource in r: + resource["metadata"]["annotations"]["foo"] = "bar" + +run(resourceList["items"]) +`, + expected: ` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nginx-deployment-1 + annotations: + foo: bar +spec: + template: + spec: + containers: + - name: nginx + # head comment + image: nginx:1.8.1 # {"$ref": "#/definitions/io.k8s.cli.substitutions.image"} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nginx-deployment-2 + annotations: + foo: bar +spec: + template: + spec: + containers: + - name: nginx + # head comment + image: nginx:1.8.1 # {"$ref": "#/definitions/io.k8s.cli.substitutions.image"}`, + }, + { + name: "add_resource", + input: ` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nginx-deployment-1 +spec: + template: + spec: + containers: + - name: nginx + # head comment + image: nginx:1.8.1 # {"$ref": "#/definitions/io.k8s.cli.substitutions.image"} +`, + script: ` +def run(r): + d = { + "apiVersion": "apps/v1", + "kind": "Deployment", + "metadata": { + "name": "nginx-deployment-2", + }, +} + r.append(d) +run(resourceList["items"]) +`, + expected: ` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nginx-deployment-2 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nginx-deployment-1 +spec: + template: + spec: + containers: + - name: nginx + # head comment + image: nginx:1.8.1 # {"$ref": "#/definitions/io.k8s.cli.substitutions.image"} +`, + }, + { + name: "remove_resource", + input: ` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nginx-deployment-1 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nginx-deployment-2 +`, + script: ` +def run(r): + r.pop() +run(resourceList["items"]) +`, + expected: ` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nginx-deployment-1 +`, + }, + } + for i := range tests { + test := tests[i] + t.Run(test.name, func(t *testing.T) { + r := &kio.ByteReader{Reader: bytes.NewBufferString(test.input)} + o := &bytes.Buffer{} + w := &kio.ByteWriter{Writer: o} + f := &Filter{Name: test.name, Program: test.script} + p := kio.Pipeline{ + Inputs: []kio.Reader{r}, + Filters: []kio.Filter{f}, + Outputs: []kio.Writer{w}, + } + err := p.Execute() + if !assert.NoError(t, err) { + t.FailNow() + } + if !assert.Equal(t, strings.TrimSpace(test.expected), strings.TrimSpace(o.String())) { + t.FailNow() + } + }) + } +}