Merge remote-tracking branch 'upstream/master' into release-cmd/config-v0.0

This commit is contained in:
Phillip Wittrock
2020-01-16 16:24:35 -08:00
45 changed files with 1651 additions and 177 deletions

View File

@@ -18,7 +18,7 @@ verify-kustomize: \
test-examples-kustomize-against-latest
.PHONY: verify-kustomize-e2e
verify-kustomize-e2e: test-examples-e2e-kustomize-against-HEAD
verify-kustomize-e2e: test-examples-e2e-kustomize
# Other builds in this repo might want a different linter version.
# Without one Makefile to rule them all, the different makes
@@ -202,8 +202,16 @@ test-examples-kustomize-against-HEAD: $(MYGOBIN)/kustomize $(MYGOBIN)/mdrip
./hack/testExamplesAgainstKustomize.sh HEAD
.PHONY:
test-examples-e2e-kustomize-against-HEAD: $(MYGOBIN)/kustomize $(MYGOBIN)/mdrip $(MYGOBIN)/resource
./hack/testExamplesE2EAgainstKustomize.sh HEAD
test-examples-e2e-kustomize: $(MYGOBIN)/mdrip
( \
set -e; \
/bin/rm -f $(MYGOBIN)/kustomize; \
echo "Installing kustomize from ."; \
cd kustomize; go install .; cd ..; \
echo "Installing resource from ."; \
cd cmd/resource; go install .; cd ../..; \
./hack/testExamplesE2EAgainstKustomize.sh .; \
)
.PHONY:
test-examples-kustomize-against-latest: $(MYGOBIN)/mdrip

View File

@@ -44,7 +44,7 @@ type kustomizeSearch struct {
// /register: not implemented, but meant as an endpoint for adding new
// kustomization files to the corpus.
func NewKustomizeSearch(ctx context.Context) (*kustomizeSearch, error) {
idx, err := index.NewKustomizeIndex(ctx)
idx, err := index.NewKustomizeIndex(ctx, "kustomize")
if err != nil {
return nil, err
}

View File

@@ -9,6 +9,8 @@ import (
"os"
"time"
"sigs.k8s.io/kustomize/api/internal/crawl/utils"
"sigs.k8s.io/kustomize/api/internal/crawl/crawler"
"sigs.k8s.io/kustomize/api/internal/crawl/crawler/github"
"sigs.k8s.io/kustomize/api/internal/crawl/doc"
@@ -26,6 +28,7 @@ const (
)
type CrawlMode int
const (
CrawlUnknown CrawlMode = iota
// Crawl all the kustomization files in all the repositories of a Github user
@@ -125,13 +128,13 @@ func main() {
// seen tracks the IDs of all the documents in the index.
// This helps avoid indexing a given document multiple times.
seen := crawler.NewSeenMap()
seen := utils.NewSeenMap()
mode := NewCrawlMode(*modePtr)
ghCrawlerConstructor := func(user, repo string) crawler.Crawler {
if user != "" {
return github.NewCrawler(githubToken, retryCount, clientCache,
return github.NewCrawler(githubToken, retryCount, clientCache,
github.QueryWith(
github.Filename("kustomization.yaml"),
github.Filename("kustomization.yml"),

View File

@@ -5,12 +5,21 @@ import (
"flag"
"fmt"
"log"
"path/filepath"
"net/http"
"os"
"sort"
"time"
"sigs.k8s.io/kustomize/api/internal/crawl/crawler/github"
"sigs.k8s.io/kustomize/api/internal/crawl/doc"
"sigs.k8s.io/kustomize/api/internal/crawl/index"
"sigs.k8s.io/kustomize/api/konfig"
)
const (
githubAccessTokenVar = "GITHUB_ACCESS_TOKEN"
retryCount = 3
)
// iterateArr adds each item in arr into countMap.
@@ -25,17 +34,6 @@ func iterateArr(arr []string, countMap map[string]int) {
}
// isKustomizationFile determines whether a file path is a kustomization file
func isKustomizationFile(path string) bool {
basename := filepath.Base(path)
for _, name := range konfig.RecognizedKustomizationFileNames() {
if basename == name {
return true
}
}
return false
}
// SortMapKeyByValue takes a map as its input, sorts its keys according to their values
// in the map, and outputs the sorted keys as a slice.
func SortMapKeyByValue(m map[string]int) []string {
@@ -44,10 +42,129 @@ func SortMapKeyByValue(m map[string]int) []string {
keys = append(keys, key)
}
// sort keys according to their values in the map m
sort.Slice(keys, func(i, j int) bool {return m[keys[i]] > m[keys[j]]})
sort.Slice(keys, func(i, j int) bool { return m[keys[i]] > m[keys[j]] })
return keys
}
func GeneratorOrTransformerStats(ctx context.Context,
docs []*doc.Document, isGenerator bool, idx *index.KustomizeIndex) {
fieldName := "generators"
if !isGenerator {
fieldName = "transformers"
}
// allReferredDocs includes all the documents referred in the field
allReferredDocs := doc.NewUniqueDocuments()
// docUsingGeneratorCount counts the number of the kustomization files using generators or transformers
docCount := 0
// collect all the documents referred in the field
for _, d := range docs {
kdoc := doc.KustomizationDocument{
Document: *d,
}
referredDocs, err := kdoc.GetResources(false, !isGenerator, isGenerator)
if err != nil {
log.Printf("failed to parse the %s field of the Document (%s): %v",
fieldName, d.Path(), err)
}
if len(referredDocs) > 0 {
docCount++
allReferredDocs.AddDocuments(referredDocs)
}
}
fileCount, dirCount, fileTypeDocs, dirTypeDocs := DocumentTypeSummary(ctx, allReferredDocs.Documents())
// check whether any of the files are not in the index
nonExistFileCount := ExistInIndex(idx, fileTypeDocs, fieldName + " file ")
// check whether any of the dirs are not in the index
nonExistDirCount := ExistInIndex(idx, dirTypeDocs, fieldName + " dir ")
GitRepositorySummary(fileTypeDocs, fieldName + " files")
GitRepositorySummary(dirTypeDocs, fieldName + " dirs")
fmt.Printf("%d kustomization files use %s: %d %s are files and %d %s are dirs.\n",
docCount, fieldName, fileCount, fieldName, dirCount, fieldName)
fmt.Printf("%d %s files do not exist in the index\n", nonExistFileCount, fieldName)
fmt.Printf("%d %s dirs do not exist in the index\n", nonExistDirCount, fieldName)
}
// GitRepositorySummary counts the distribution of docs:
// 1) how many git repositories are these docs from?
// 2) how many docs are from each git repository?
func GitRepositorySummary(docs []*doc.Document, msgPrefix string) {
m := make(map[string]int)
for _, d := range docs {
if _, ok := m[d.RepositoryURL]; ok {
m[d.RepositoryURL]++
} else {
m[d.RepositoryURL] = 1
}
}
sortedKeys := SortMapKeyByValue(m)
for _, k := range sortedKeys {
fmt.Printf("%d %s are from %s\n", m[k], msgPrefix, k)
}
}
// ExistInIndex goes through each Document in docs, and check whether it is in the index or not.
// It returns the number of documents which does not exist in the index.
func ExistInIndex(idx *index.KustomizeIndex, docs []*doc.Document, msgPrefix string) int {
nonExistCount := 0
for _, d := range docs {
exists, err := idx.Exists(d.ID())
if err != nil {
log.Println(err)
}
if !exists {
log.Printf("%s (%s) does not exist in the index", msgPrefix, d.Path())
nonExistCount++
}
}
return nonExistCount
}
// DocumentTypeSummary goes through each doc in docs, and determines whether it is a file or dir.
func DocumentTypeSummary(ctx context.Context, docs []*doc.Document) (
fileCount, dirCount int, files, dirs []*doc.Document) {
githubToken := os.Getenv(githubAccessTokenVar)
if githubToken == "" {
log.Fatalf("Must set the variable '%s' to make github requests.\n",
githubAccessTokenVar)
}
ghCrawler := github.NewCrawler(githubToken, retryCount, &http.Client{}, github.QueryWith())
for _, d := range docs {
oldFilePath := d.FilePath
if err := ghCrawler.FetchDocument(ctx, d); err != nil {
log.Printf("FetchDocument failed on %s: %v", d.Path(), err)
continue
}
if d.FilePath == oldFilePath {
fileCount++
files = append(files, d)
} else {
dirCount++
dirs = append(dirs, d)
}
}
return fileCount, dirCount, files, dirs
}
// ExistInSlice checks where target exits in items.
func ExistInSlice(items []string, target string) bool {
for _, item := range items {
if item == target {
return true
}
}
return false
}
func main() {
topKindsPtr := flag.Int(
"kinds", -1,
@@ -64,10 +181,12 @@ If you only want to list the 10 most popular identifiers, set the flag to 10.`)
`the number of kustomize features to be listed according to their popularities.
By default, all the features will be listed.
If you only want to list the 10 most popular features, set the flag to 10.`)
indexNamePtr := flag.String(
"index", "kustomize", "The name of the ElasticSearch index.")
flag.Parse()
ctx := context.Background()
idx, err := index.NewKustomizeIndex(ctx)
idx, err := index.NewKustomizeIndex(ctx, *indexNamePtr)
if err != nil {
log.Fatalf("Could not create an index: %v\n", err)
}
@@ -85,6 +204,12 @@ If you only want to list the 10 most popular features, set the flag to 10.`)
// ids tracks the unique IDs of the documents in the index
ids := make(map[string]struct{})
// generatorDocs includes all the docs using generators
generatorDocs := make([]*doc.Document, 0)
// transformersDocs includes all the docs using transformers
transformersDocs := make([]*doc.Document, 0)
// get all the documents in the index
query := []byte(`{ "query":{ "match_all":{} } }`)
it := idx.IterateQuery(query, 10000, 60*time.Second)
@@ -94,21 +219,28 @@ If you only want to list the 10 most popular features, set the flag to 10.`)
if _, ok := ids[hit.ID]; !ok {
ids[hit.ID] = struct{}{}
} else {
fmt.Printf("Found duplicate ID (%s)\n", hit.ID)
log.Printf("Found duplicate ID (%s)\n", hit.ID)
}
count++
iterateArr(hit.Document.Kinds, kindsMap)
iterateArr(hit.Document.Identifiers, identifiersMap)
if isKustomizationFile(hit.Document.FilePath) {
if doc.IsKustomizationFile(hit.Document.FilePath) {
kustomizationFilecount++
iterateArr(hit.Document.Identifiers, kustomizeIdentifiersMap)
if ExistInSlice(hit.Document.Identifiers, "generators") {
generatorDocs = append(generatorDocs, hit.Document.Copy())
}
if ExistInSlice(hit.Document.Identifiers, "transformers") {
transformersDocs = append(transformersDocs, hit.Document.Copy())
}
}
}
}
if err := it.Err(); err != nil {
fmt.Printf("Error iterating: %v\n", err)
log.Fatalf("Error iterating: %v\n", err)
}
sortedKindsMapKeys := SortMapKeyByValue(kindsMap)
@@ -147,4 +279,7 @@ There are %d documents in the kustomize index.
kustomizeFeatureCount++
}
}
GeneratorOrTransformerStats(ctx, generatorDocs, true, idx)
GeneratorOrTransformerStats(ctx, transformersDocs, false, idx)
}

View File

@@ -11,8 +11,13 @@ spec:
image: gcr.io/haiyanmeng-gke-dev/kustomize_stats:v1
imagePullPolicy: Always
command: ["/kustomize_stats"]
args: ["--kinds=50", "--identifiers=50", "--kustomize-features=50"]
args: ["--index=kustomize", "--kinds=50", "--identifiers=50", "--kustomize-features=50"]
env:
- name: GITHUB_ACCESS_TOKEN
valueFrom:
secretKeyRef:
name: github-access-token
key: token
- name: ELASTICSEARCH_URL
valueFrom:
configMapKeyRef:

View File

@@ -10,6 +10,8 @@ import (
"os"
"sync"
"sigs.k8s.io/kustomize/api/internal/crawl/utils"
"sigs.k8s.io/kustomize/api/internal/crawl/index"
_ "github.com/gomodule/redigo/redis"
@@ -29,7 +31,7 @@ type Crawler interface {
// Crawl returns when it is done processing. This method does not take
// ownership of the channel. The channel is write only, and it
// designates where the crawler should forward the documents.
Crawl(ctx context.Context, output chan<- CrawledDocument, seen SeenMap) error
Crawl(ctx context.Context, output chan<- CrawledDocument, seen utils.SeenMap) error
// Get the document data given the FilePath, Repo, and Ref/Tag/Branch.
FetchDocument(context.Context, *doc.Document) error
@@ -43,25 +45,15 @@ type CrawledDocument interface {
ID() string
GetDocument() *doc.Document
// Get all the Documents directly referred in a Document.
GetResources() ([]*doc.Document, error)
// For a Document representing a non-kustomization file, an empty slice will be returned.
// For a Document representing a kustomization file:
// the `includeResources` parameter determines whether the documents referred in the `resources` field are returned or not;
// the `includeTransformers` parameter determines whether the documents referred in the `transformers` field are returned or not;
// the `includeGenerators` parameter determines whether the documents referred in the `generators` field are returned or not.
GetResources(includeResources, includeTransformers, includeGenerators bool) ([]*doc.Document, error)
WasCached() bool
}
type SeenMap map[string]struct{}
func (seen SeenMap) Seen(item string) bool {
_, ok := seen[item]
return ok
}
func (seen SeenMap) Add(item string) {
seen[item] = struct{}{}
}
func NewSeenMap() SeenMap {
return make(map[string]struct{})
}
type CrawlSeed []*doc.Document
type IndexFunc func(CrawledDocument, index.Mode) error
@@ -84,18 +76,18 @@ func findMatch(d *doc.Document, crawlers []Crawler) Crawler {
}
func addBranches(cdoc CrawledDocument, match Crawler, indx IndexFunc,
seen SeenMap, stack *CrawlSeed) {
seen utils.SeenMap, stack *CrawlSeed) {
seen.Add(cdoc.ID())
// Insert into index
if err := indx(cdoc, index.InsertOrUpdate); err != nil {
logger.Printf("Failed to insert or update %s %s: %v",
cdoc.GetDocument().RepositoryURL, cdoc.GetDocument().FilePath, err)
logger.Printf("Failed to insert or update doc(%s): %v",
cdoc.GetDocument().Path(), err)
return
}
deps, err := cdoc.GetResources()
deps, err := cdoc.GetResources(true, false, false)
if err != nil {
logger.Println(err)
return
@@ -110,7 +102,7 @@ func addBranches(cdoc CrawledDocument, match Crawler, indx IndexFunc,
}
func doCrawl(ctx context.Context, docsPtr *CrawlSeed, crawlers []Crawler, conv Converter, indx IndexFunc,
seen SeenMap, stack *CrawlSeed) {
seen utils.SeenMap, stack *CrawlSeed) {
UpdatedDocCount := 0
seenDocCount := 0
@@ -131,7 +123,7 @@ func doCrawl(ctx context.Context, docsPtr *CrawlSeed, crawlers []Crawler, conv C
*docsPtr = (*docsPtr)[:(len(*docsPtr) - 1)]
crawledDocCount++
logger.Printf("Crawling doc %d: %s %s", crawledDocCount, tail.RepositoryURL, tail.FilePath)
logger.Printf("Crawling doc %d: %s", crawledDocCount, tail.Path())
if seen.Seen(tail.ID()) {
logger.Printf("this doc has been seen before")
@@ -140,7 +132,7 @@ func doCrawl(ctx context.Context, docsPtr *CrawlSeed, crawlers []Crawler, conv C
}
if tail.WasCached() {
logger.Printf("%s %s is cached already", tail.RepositoryURL, tail.FilePath)
logger.Printf("doc(%s) is cached already", tail.Path())
cachedDocCount++
continue
}
@@ -161,10 +153,8 @@ func doCrawl(ctx context.Context, docsPtr *CrawlSeed, crawlers []Crawler, conv C
// `bases` field.
seen.Add(tail.ID())
if err := match.FetchDocument(ctx, tail); err != nil {
logger.Printf("FetchDocument failed on %s %s: %v",
tail.RepositoryURL, tail.FilePath, err)
logger.Printf("FetchDocument failed on doc(%s): %v", tail.Path(), err)
FetchDocumentErrCount++
// delete the document from the index
cdoc := &doc.KustomizationDocument{
@@ -172,16 +162,14 @@ func doCrawl(ctx context.Context, docsPtr *CrawlSeed, crawlers []Crawler, conv C
}
seen.Add(cdoc.ID())
if err := indx(cdoc, index.Delete); err != nil {
logger.Printf("Failed to delete %s %s: %v",
cdoc.RepositoryURL, cdoc.FilePath, err)
logger.Printf("Failed to delete doc(%s): %v", cdoc.Path(), err)
}
deleteDocCount++
continue
}
if err := match.SetCreated(ctx, tail); err != nil {
logger.Printf("SetCreated failed on %s %s: %v",
tail.RepositoryURL, tail.FilePath, err)
logger.Printf("SetCreated failed on doc(%s): %v", tail.Path(), err)
SetCreatedErrCount++
}
@@ -189,8 +177,7 @@ func doCrawl(ctx context.Context, docsPtr *CrawlSeed, crawlers []Crawler, conv C
// If conv returns an error, cdoc can still be added into the index so that
// cdoc.Document can be searched.
if err != nil {
logger.Printf("conv failed on %s %s: %v",
tail.RepositoryURL, tail.FilePath, err)
logger.Printf("conv failed on doc(%s): %v", tail.Path(), err)
convErrCount++
}
@@ -211,7 +198,7 @@ func doCrawl(ctx context.Context, docsPtr *CrawlSeed, crawlers []Crawler, conv C
// CrawlFromSeed updates all the documents in seed, and crawls all the new
// documents referred in the seed.
func CrawlFromSeed(ctx context.Context, seed CrawlSeed, crawlers []Crawler,
conv Converter, indx IndexFunc, seen SeenMap) {
conv Converter, indx IndexFunc, seen utils.SeenMap) {
// stack tracks the documents directly referred in other documents.
stack := make(CrawlSeed, 0)
@@ -247,7 +234,7 @@ func CrawlFromSeed(ctx context.Context, seed CrawlSeed, crawlers []Crawler,
// from the seed will be processed before any other documents from the
// crawlers.
func CrawlGithubRunner(ctx context.Context, output chan<- CrawledDocument,
crawlers []Crawler, seen SeenMap) []error {
crawlers []Crawler, seen utils.SeenMap) []error {
errs := make([]error, len(crawlers))
wg := sync.WaitGroup{}
@@ -291,7 +278,7 @@ func CrawlGithubRunner(ctx context.Context, output chan<- CrawledDocument,
// CrawlGithub crawls all the kustomization files on Github.
func CrawlGithub(ctx context.Context, crawlers []Crawler, conv Converter,
indx IndexFunc, seen SeenMap) {
indx IndexFunc, seen utils.SeenMap) {
// stack tracks the documents directly referred in other documents.
stack := make(CrawlSeed, 0)

View File

@@ -12,6 +12,8 @@ import (
"testing"
"time"
"sigs.k8s.io/kustomize/api/internal/crawl/utils"
"sigs.k8s.io/kustomize/api/internal/crawl/index"
"sigs.k8s.io/kustomize/api/internal/crawl/doc"
@@ -76,7 +78,7 @@ func newCrawler(matchPrefix string, err error,
// Crawl implements the Crawler interface for testing.
func (c testCrawler) Crawl(_ context.Context,
output chan<- CrawledDocument, _ SeenMap) error {
output chan<- CrawledDocument, _ utils.SeenMap) error {
for i, d := range c.docs {
isResource := true
@@ -182,7 +184,7 @@ func TestCrawlGithubRunner(t *testing.T) {
defer close(output)
defer wg.Done()
seen := NewSeenMap()
seen := utils.NewSeenMap()
errs := CrawlGithubRunner(context.Background(),
output, test.tc, seen)
@@ -324,7 +326,7 @@ resources:
visited[d.ID()]++
return nil
},
NewSeenMap(),
utils.NewSeenMap(),
)
if lv, lc := len(visited), len(tc.corpus); lv != lc {
t.Errorf("error: %d of %d documents visited.", lv, lc)

View File

@@ -16,6 +16,8 @@ import (
"strings"
"time"
"sigs.k8s.io/kustomize/api/internal/crawl/utils"
"sigs.k8s.io/kustomize/api/internal/crawl/crawler"
"sigs.k8s.io/kustomize/api/internal/crawl/doc"
"sigs.k8s.io/kustomize/api/internal/crawl/httpclient"
@@ -68,7 +70,7 @@ func (gc githubCrawler) DefaultBranch(repo string) string {
// Implements crawler.Crawler.
func (gc githubCrawler) Crawl(ctx context.Context,
output chan<- crawler.CrawledDocument, seen crawler.SeenMap) error {
output chan<- crawler.CrawledDocument, seen utils.SeenMap) error {
noETagClient := GhClient{
RequestConfig: gc.client.RequestConfig,
@@ -195,9 +197,9 @@ func (gc githubCrawler) Match(d *doc.Document) bool {
type RangeQueryResult struct {
totalDocCnt uint64
seenDocCnt uint64
newDocCnt uint64
errorCnt uint64
seenDocCnt uint64
newDocCnt uint64
errorCnt uint64
}
func (r *RangeQueryResult) Add(other RangeQueryResult) {
@@ -209,7 +211,7 @@ func (r *RangeQueryResult) Add(other RangeQueryResult) {
func (r *RangeQueryResult) String() string {
return fmt.Sprintf("got %d files from API. "+
"%d have been seen before. %d are new and sent to the output channel." +
"%d have been seen before. %d are new and sent to the output channel."+
" %d have kustomizationResultAdapter errors.",
r.totalDocCnt, r.seenDocCnt, r.newDocCnt, r.errorCnt)
}
@@ -217,7 +219,7 @@ func (r *RangeQueryResult) String() string {
// processQuery follows all of the pages in a query, and updates/adds the
// documents from the crawl to the datastore/index.
func processQuery(ctx context.Context, gcl GhClient, query string,
output chan<- crawler.CrawledDocument, seen crawler.SeenMap,
output chan<- crawler.CrawledDocument, seen utils.SeenMap,
branchMap map[string]string) (RangeQueryResult, error) {
queryPages := make(chan GhResponseInfo)
@@ -271,7 +273,7 @@ func processQuery(ctx context.Context, gcl GhClient, query string,
return result, errs
}
func kustomizationResultAdapter(gcl GhClient, k GhFileSpec, seen crawler.SeenMap,
func kustomizationResultAdapter(gcl GhClient, k GhFileSpec, seen utils.SeenMap,
branchMap map[string]string) (crawler.CrawledDocument, error) {
url := gcl.ReposRequest(k.Repository.FullName)
defaultBranch, err := gcl.GetDefaultBranch(url, k.Repository.URL, branchMap)

View File

@@ -117,7 +117,7 @@ type RequestConfig struct {
// understand why the request object is useful.
func (rc RequestConfig) CodeSearchRequestWith(query Query) request {
vals := url.Values{
"sort": []string{"indexed"},
"sort": []string{"indexed"},
"order": []string{"desc"},
}
req := rc.makeRequest("search/code", query, vals)

View File

@@ -3,6 +3,7 @@ package doc
import (
"fmt"
"log"
"path/filepath"
"sort"
"strings"
@@ -51,15 +52,21 @@ func (doc *KustomizationDocument) String() string {
doc.IsSame, doc.Kinds, len(doc.Identifiers), len(doc.Values))
}
// Implements the CrawlerDocument interface.
func (doc *KustomizationDocument) GetResources() ([]*Document, error) {
isResource := true
for _, suffix := range konfig.RecognizedKustomizationFileNames() {
if strings.HasSuffix(doc.FilePath, "/"+suffix) {
isResource = false
// IsKustomizationFile determines whether a file path is a kustomization file
func IsKustomizationFile(path string) bool {
basename := filepath.Base(path)
for _, name := range konfig.RecognizedKustomizationFileNames() {
if basename == name {
return true
}
}
if isResource {
return false
}
// Implements the CrawlerDocument interface.
func (doc *KustomizationDocument) GetResources(
includeResources, includeTransformers, includeGenerators bool) ([]*Document, error) {
if !IsKustomizationFile(doc.FilePath) {
return []*Document{}, nil
}
@@ -77,20 +84,42 @@ func (doc *KustomizationDocument) GetResources() ([]*Document, error) {
}
k.FixKustomizationPostUnmarshalling()
res := make([]*Document, 0, len(k.Resources))
for _, r := range k.Resources {
res := make([]*Document, 0)
if includeResources {
resourceDocs := doc.CollectDocuments(k.Resources)
res = append(res, resourceDocs...)
}
if includeGenerators {
generatorDocs := doc.CollectDocuments(k.Generators)
res = append(res, generatorDocs...)
}
if includeTransformers {
transformerDocs := doc.CollectDocuments(k.Transformers)
res = append(res, transformerDocs...)
}
return res, nil
}
// CollectDocuments construct a Document for each path in paths, and return
// a slice of Document pointers.
func (doc *KustomizationDocument) CollectDocuments(paths []string) []*Document {
docs := make([]*Document, 0, len(paths))
for _, r := range paths {
if strings.TrimSpace(r) == "" {
continue
}
next, err := doc.Document.FromRelativePath(r)
if err != nil {
log.Printf("GetResources error: %v\n", err)
log.Printf("CollectDocuments error: %v\n", err)
continue
}
res = append(res, &next)
docs = append(docs, &next)
}
return res, nil
return docs
}
func (doc *KustomizationDocument) readBytes() ([]map[string]interface{}, error) {

View File

@@ -189,11 +189,13 @@ metadata:
}
}
type TestStructForGetResources struct {
doc KustomizationDocument
resources []*Document
}
func TestGetResources(t *testing.T) {
tests := []struct {
doc KustomizationDocument
resources []*Document
}{
tests := []TestStructForGetResources{
{
doc: KustomizationDocument{
Document: Document{
@@ -248,9 +250,12 @@ resources:
resources: []*Document{},
},
}
runTest(t, tests, true, false, false)
}
func runTest(t *testing.T, tests []TestStructForGetResources, includeResources, includeTransformers, includeGenerators bool) {
for _, test := range tests {
res, err := test.doc.GetResources()
res, err := test.doc.GetResources(includeResources, includeTransformers, includeGenerators)
if err != nil {
t.Errorf("Unexpected error: %v\n", err)
continue
@@ -284,3 +289,73 @@ resources:
}
}
}
func TestGetResourcesAndGenerators(t *testing.T) {
tests := []TestStructForGetResources{
{
doc: KustomizationDocument{
Document: Document{
RepositoryURL: "sigs.k8s.io/kustomize",
FilePath: "some/path/to/kdir/kustomization.yaml",
DocumentData: `
resources:
- file.yaml
generators:
- gen.yaml
transformers:
- tr.yaml
`},
},
resources: []*Document{
{
RepositoryURL: "sigs.k8s.io/kustomize",
FilePath: "some/path/to/kdir/gen.yaml",
},
{
RepositoryURL: "sigs.k8s.io/kustomize",
FilePath: "some/path/to/kdir/file.yaml",
},
},
},
}
runTest(t, tests, true, false, true)
}
func TestGetResourcesAndGeneratorsAndTransformers(t *testing.T) {
tests := []TestStructForGetResources{
{
doc: KustomizationDocument{
Document: Document{
RepositoryURL: "sigs.k8s.io/kustomize",
FilePath: "some/path/to/kdir/kustomization.yaml",
DocumentData: `
resources:
- file.yaml
generators:
- gen.yaml
transformers:
- tr.yaml
`},
},
resources: []*Document{
{
RepositoryURL: "sigs.k8s.io/kustomize",
FilePath: "some/path/to/kdir/tr.yaml",
},
{
RepositoryURL: "sigs.k8s.io/kustomize",
FilePath: "some/path/to/kdir/gen.yaml",
},
{
RepositoryURL: "sigs.k8s.io/kustomize",
FilePath: "some/path/to/kdir/file.yaml",
},
},
},
}
runTest(t, tests, true, true, true)
}

View File

@@ -35,6 +35,11 @@ func (doc *Document) Copy() *Document {
}
}
func (doc *Document) Path() string {
return fmt.Sprintf("repoURL: %s filePath: %s branch: %s",
doc.RepositoryURL, doc.FilePath, doc.DefaultBranch)
}
// Implements the CrawlerDocument interface.
func (doc *Document) WasCached() bool {
return doc.IsSame

View File

@@ -65,7 +65,7 @@ func TestFromRelativePath(t *testing.T) {
func TestDocument_RepositoryFullName(t *testing.T) {
testCases := []struct {
doc Document
doc Document
expectedRepositoryFullName string
}{
{
@@ -108,4 +108,4 @@ func TestDocument_RepositoryFullName(t *testing.T) {
returnedRepositoryFullName)
}
}
}
}

View File

@@ -0,0 +1,36 @@
package doc
import (
"sigs.k8s.io/kustomize/api/internal/crawl/utils"
)
// UniqueDocuments make sure a Document with a given ID appears only once
type UniqueDocuments struct {
docs []*Document
docIDs utils.SeenMap
}
func NewUniqueDocuments() UniqueDocuments {
return UniqueDocuments{
docs: []*Document{},
docIDs: utils.NewSeenMap(),
}
}
func (uds *UniqueDocuments) Add(d *Document) {
if uds.docIDs.Seen(d.ID()) {
return
}
uds.docs = append(uds.docs, d)
uds.docIDs.Add(d.ID())
}
func (uds *UniqueDocuments) AddDocuments(docs []*Document) {
for _, d := range docs {
uds.Add(d)
}
}
func (uds *UniqueDocuments) Documents() []*Document {
return uds.docs
}

View File

@@ -18,6 +18,7 @@ const (
)
type Mode int
const (
InsertOrUpdate = iota
Delete

View File

@@ -1,6 +1,6 @@
Find out the largest value of the `creationTime` field:
```
curl -X POST "${ElasticSearchURL}:9200/kustomize/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
curl -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
{
"aggs" : {
"max_creationTime" : { "max" : { "field" : "creationTime" } }
@@ -11,7 +11,7 @@ curl -X POST "${ElasticSearchURL}:9200/kustomize/_search?size=0&pretty" -H 'Cont
Find out the smallest value of the `creationTime` field:
```
curl -X POST "${ElasticSearchURL}:9200/kustomize/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
curl -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
{
"aggs" : {
"min_creationTime" : { "min" : { "field" : "creationTime" } }
@@ -22,7 +22,7 @@ curl -X POST "${ElasticSearchURL}:9200/kustomize/_search?size=0&pretty" -H 'Cont
Find out the smallest value of the `creationTime` field of all the kustomization files:
```
curl -X POST "${ElasticSearchURL}:9200/kustomize/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
curl -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
{
"query": {
"bool": {
@@ -40,7 +40,7 @@ curl -X POST "${ElasticSearchURL}:9200/kustomize/_search?size=0&pretty" -H 'Cont
Find out the smallest value of the `creationTime` field of all kustomize resource files:
```
curl -X POST "${ElasticSearchURL}:9200/kustomize/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
curl -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
{
"query": {
"bool": {
@@ -58,7 +58,7 @@ curl -X POST "${ElasticSearchURL}:9200/kustomize/_search?size=0&pretty" -H 'Cont
Query all the documents whose `creationTime` <= `2016-07-29T17:38:26.000Z`:
```
curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type: application/json' -d'
curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
{
"query": {
"range": {
@@ -73,7 +73,7 @@ curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type
Query all the documents whose `creationTime` falls within the specific range:
```
curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type: application/json' -d'
curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
{
"query": {
"range": {
@@ -89,7 +89,7 @@ curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type
Aggregate how many new kustomization files were added into Github each month:
```
curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
{
"query": {
"bool": {
@@ -112,7 +112,7 @@ curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?size=0&pretty" -H 'Conte
Aggregate how many new kustomize resource files were added into Github each month:
```
curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
{
"query": {
"bool": {
@@ -135,7 +135,7 @@ curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?size=0&pretty" -H 'Conte
Aggregate how many new kustomization files were added into Github each year:
```
curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
{
"query": {
"bool": {
@@ -158,7 +158,7 @@ curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?size=0&pretty" -H 'Conte
Aggregate how many new kustomize resource files were added into Github each year:
```
curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
{
"query": {
"bool": {

View File

@@ -1,6 +1,6 @@
Count distinct values of the `defaultBranch` field:
```
curl -X POST "${ElasticSearchURL}:9200/kustomize/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
curl -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
{
"aggs" : {
"defaultBranch_count" : {
@@ -17,7 +17,7 @@ curl -X POST "${ElasticSearchURL}:9200/kustomize/_search?size=0&pretty" -H 'Cont
List all the github branches where kustomization files and kustomize resource files live,
and how many kustomization files and kustomize resource files live in each branch:
```
curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type: application/json' -d'
curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
{
"aggs" : {
"defaultBranch" : {

View File

@@ -1,7 +1,7 @@
Count the documents whose `document` field is empty (The reason why the `document` field
of a document is empty is because of empty documents):
```
curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type: application/json' -d'
curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
{
"size": 10000,
"query": {
@@ -19,7 +19,7 @@ curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type
Find all the documents having the `creationTime` field set:
```
curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type: application/json' -d'
curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
{
"query": {
"exists": {
@@ -32,7 +32,7 @@ curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type
Find all the documents whose `creationTime` field is not set:
```
curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type: application/json' -d'
curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
{
"size": 10000,
"query": {

View File

@@ -1,7 +1,7 @@
Count the documents in the index whose `repositoryUrl` field starts with
`https://github.com/`:
```
curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type: application/json' -d'
curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
{
"query": {
"bool": {
@@ -17,7 +17,7 @@ curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type
Count the documents in the index whose `repositoryUrl` field does not start with
`https://github.com/`:
```
curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type: application/json' -d'
curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
{
"query": {
"bool": {
@@ -33,7 +33,7 @@ curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type
Search all the documents matching the given `repositoryUrl` and `filePath`, and return
a version for each search hit:
```
curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type: application/json' -d'
curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
{
"size": 10000,
"version": true,
@@ -52,7 +52,7 @@ curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type
Search all the documents whose filePath ends with one of these following three filenames:
`kustomization.yaml`, `kustomization.yml`, `kustomization`:
```
curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type: application/json' -d'
curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
{
"query": {
"bool": {
@@ -68,7 +68,7 @@ curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type
Search all the documents whose filePath does not end with any of these following
three filenames: `kustomization.yaml`, `kustomization.yml`, `kustomization`:
```
curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type: application/json' -d'
curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
{
"query": {
"bool": {

View File

@@ -10,10 +10,10 @@ curl "${ElasticSearchURL}:9200/_cat/indices?v"
Get the mapping of the index:
```
curl -X GET "${ElasticSearchURL}:9200/kustomize/_mapping?pretty"
curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_mapping?pretty"
```
Delete the kustomize index from the ElasticSearch cluster (**Use this command with caution**):
```
curl -X DELETE "${ElasticSearchURL}:9200/kustomize?pretty"
curl -X DELETE "${ElasticSearchURL}:9200/${INDEXNAME}?pretty"
```

View File

@@ -1,6 +1,6 @@
Count distinct values of the `repositoryUrl` field:
```
curl -X POST "${ElasticSearchURL}:9200/kustomize/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
curl -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
{
"aggs" : {
"repositoryUrl_count" : {
@@ -16,7 +16,7 @@ curl -X POST "${ElasticSearchURL}:9200/kustomize/_search?size=0&pretty" -H 'Cont
Count how many Github repositories include kustomization files:
```
curl -X POST "${ElasticSearchURL}:9200/kustomize/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
curl -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
{
"query": {
"bool": {
@@ -39,7 +39,7 @@ curl -X POST "${ElasticSearchURL}:9200/kustomize/_search?size=0&pretty" -H 'Cont
Count how many Github repositories include kustomize resource files:
```
curl -X POST "${ElasticSearchURL}:9200/kustomize/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
curl -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
{
"query": {
"bool": {
@@ -64,7 +64,7 @@ List all the github repositories including kustomization files and kustomize res
and how many kustomization files and kustomize resource files each github repository includes
(the github repository including the most kustomization files is listed first):
```
curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
{
"aggs" : {
"repositoryUrl" : {
@@ -80,7 +80,7 @@ curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?size=0&pretty" -H 'Conte
List the top 20 Github repositories including the most amount of kustomization files:
```
curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
{
"query": {
"bool": {
@@ -103,7 +103,7 @@ curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?size=0&pretty" -H 'Conte
List the top 20 Github repositories including the most amount of kustomize resource files:
```
curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
{
"query": {
"bool": {

View File

@@ -1,6 +1,6 @@
Search for all the kustomize resource files including a Deployment object:
```
curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type: application/json' -d'
curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
{
"query": {
"match" : {
@@ -16,7 +16,7 @@ curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type
Search for all the kustomize resource files including a Deployment object, but only
including the `kinds` field in the result:
```
curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type: application/json' -d'
curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
{
"_source": {
"includes": ["kinds"]
@@ -35,7 +35,7 @@ curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type
Search for all the kustomize resource files including both a Deployment object and
a Service object:
```
curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type: application/json' -d'
curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
{
"query": {
"match" : {
@@ -52,7 +52,7 @@ curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type
Count the number of documents including Deployment and the number of documents
including Service:
```
curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type: application/json' -d'
curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
{
"size": 0,
"aggs" : {
@@ -71,7 +71,7 @@ curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type
Search for all the kustomization files involving CRDs:
```
curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type: application/json' -d'
curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
{
"size": 10000,
"query": {
@@ -87,7 +87,7 @@ curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type
Search for all the kustomization files defining configMapGenerator:
```
curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type: application/json' -d'
curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
{
"size": 10000,
"query": {
@@ -103,7 +103,7 @@ curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type
Search for all the documents having a `kind` field:
```
curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type: application/json' -d'
curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
{
"query": {
"bool": {
@@ -118,7 +118,7 @@ curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type
Search for all the kuostmization files having a `kind` field:
```
curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type: application/json' -d'
curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
{
"query": {
"bool": {
@@ -134,7 +134,7 @@ curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type
Search for all the kustomization files defining the `generatorOptions:disableNameSuffixHash` feature:
```
curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type: application/json' -d'
curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
{
"query": {
"match" : {

View File

@@ -0,0 +1,16 @@
package utils
type SeenMap map[string]struct{}
func (seen SeenMap) Seen(item string) bool {
_, ok := seen[item]
return ok
}
func (seen SeenMap) Add(item string) {
seen[item] = struct{}{}
}
func NewSeenMap() SeenMap {
return make(map[string]struct{})
}

View File

@@ -45,6 +45,7 @@ Advanced Documentation Topics:
// Export commands publicly for composition
var (
Annotate = commands.AnnotateCommand
Cat = commands.CatCommand
Count = commands.CountCommand
CreateSetter = commands.CreateSetterCommand
@@ -58,6 +59,8 @@ var (
Sink = commands.SinkCommand
Source = commands.SourceCommand
Tree = commands.TreeCommand
Wrap = commands.WrapCommand
XArgs = commands.XArgsCommand
StackOnError = &commands.StackOnError
ExitOnError = &commands.ExitOnError
@@ -89,6 +92,7 @@ func NewConfigCommand(name string) *cobra.Command {
name = strings.TrimSpace(name + " config")
commands.ExitOnError = true
root.AddCommand(commands.AnnotateCommand(name))
root.AddCommand(commands.GrepCommand(name))
root.AddCommand(commands.TreeCommand(name))
root.AddCommand(commands.CatCommand(name))

View File

@@ -0,0 +1,18 @@
## annotate
[Alpha] Set an annotation on Resources.
### Synopsis
[Alpha] Set an annotation on Resources.
DIR:
Path to local directory.
### Examples
kustomize config annotate my-dir/ --kv foo=bar
kustomize config annotate my-dir/ --kv foo=bar --kv a=b
kustomize config annotate my-dir/ --kv foo=bar --kind Deployment --name foo

View File

@@ -0,0 +1,103 @@
// Copyright 2019 The Kubernetes Authors.
// SPDX-License-Identifier: Apache-2.0
package commands
import (
"strings"
"github.com/spf13/cobra"
"sigs.k8s.io/kustomize/cmd/config/internal/generateddocs/commands"
"sigs.k8s.io/kustomize/kyaml/errors"
"sigs.k8s.io/kustomize/kyaml/kio"
"sigs.k8s.io/kustomize/kyaml/yaml"
)
// NewAnnotateRunner returns a command runner.
func NewAnnotateRunner(parent string) *AnnotateRunner {
r := &AnnotateRunner{}
c := &cobra.Command{
Use: "annotate [DIR]",
Args: cobra.MaximumNArgs(1),
Short: commands.AnnotateShort,
Long: commands.AnnotateLong,
Example: commands.AnnotateExamples,
RunE: r.runE,
}
fixDocs(parent, c)
r.Command = c
c.Flags().StringVar(&r.Kind, "kind", "", "Resource kind to annotate")
c.Flags().StringVar(&r.ApiVersion, "apiVersion", "", "Resource apiVersion to annotate")
c.Flags().StringVar(&r.Name, "name", "", "Resource name to annotate")
c.Flags().StringVar(&r.Namespace, "namespace", "", "Resource namespace to annotate")
c.Flags().StringSliceVar(&r.Values, "kv", []string{}, "annotation as KEY=VALUE")
return r
}
func AnnotateCommand(parent string) *cobra.Command {
return NewAnnotateRunner(parent).Command
}
type AnnotateRunner struct {
Command *cobra.Command
Values []string
Kind string
Name string
ApiVersion string
Namespace string
Path string
}
func (r *AnnotateRunner) runE(c *cobra.Command, args []string) error {
var input []kio.Reader
var output []kio.Writer
if len(args) == 0 {
rw := &kio.ByteReadWriter{Reader: c.InOrStdin(), Writer: c.OutOrStdout()}
input = []kio.Reader{rw}
output = []kio.Writer{rw}
} else {
rw := &kio.LocalPackageReadWriter{PackagePath: args[0], NoDeleteFiles: true}
input = []kio.Reader{rw}
output = []kio.Writer{rw}
}
return handleError(c, kio.Pipeline{
Inputs: input,
Filters: []kio.Filter{r},
Outputs: output,
}.Execute())
}
func (r *AnnotateRunner) Filter(nodes []*yaml.RNode) ([]*yaml.RNode, error) {
for i := range nodes {
n := nodes[i]
m, err := n.GetMeta()
if err != nil {
return nil, err
}
if r.Kind != "" && r.Kind != m.Kind {
continue
}
if r.ApiVersion != "" && r.ApiVersion != m.APIVersion {
continue
}
if r.Namespace != "" && r.Namespace != m.Namespace {
continue
}
if r.Name != "" && r.Name != m.Name {
continue
}
for i := range r.Values {
// split key, value pairs
kv := strings.SplitN(r.Values[i], "=", 2)
if len(kv) != 2 {
return nil, errors.Errorf("must specify --kv as KEY=VALUE: %s", r.Values[i])
}
if err := n.PipeE(yaml.SetAnnotation(kv[0], kv[1])); err != nil {
return nil, err
}
}
}
return nodes, nil
}

View File

@@ -0,0 +1,488 @@
// Copyright 2019 The Kubernetes Authors.
// SPDX-License-Identifier: Apache-2.0
package commands
import (
"bytes"
"io/ioutil"
"os"
"path/filepath"
"strings"
"testing"
"github.com/stretchr/testify/assert"
"sigs.k8s.io/kustomize/kyaml/kio"
)
func TestAnnotateCommand(t *testing.T) {
var tests = []struct {
name string
args []string
expected string
}{
{
name: "single value",
args: []string{"--kv", "a=b"},
expected: expectedSingleValue,
},
{
name: "multi value",
args: []string{"--kv", "a=b", "--kv", "c=d"},
expected: expectedMultiValue,
},
{
name: "filter kind",
args: []string{"--kv", "a=b", "--kind", "Service"},
expected: expectedFilterKindService,
},
{
name: "filter apiVersion",
args: []string{"--kv", "a=b", "--apiVersion", "v1"},
expected: expectedFilterApiVersionV1,
},
{
name: "filter name",
args: []string{"--kv", "a=b", "--name", "bar"},
expected: expectedFilterNameBar,
},
{
name: "filter namespace",
args: []string{"--kv", "a=b", "--namespace", "bar"},
expected: expectedFilterNamespaceBar,
},
}
for i := range tests {
tt := tests[i]
t.Run(tt.name, func(t *testing.T) {
d := initTestDir(t)
defer os.RemoveAll(d)
a := NewAnnotateRunner("")
a.Command.SetArgs(append([]string{d}, tt.args...))
a.Command.SilenceUsage = true
a.Command.SilenceErrors = true
err := a.Command.Execute()
if !assert.NoError(t, err) {
t.FailNow()
}
actual := &bytes.Buffer{}
err = kio.Pipeline{
Inputs: []kio.Reader{kio.LocalPackageReader{PackagePath: d}},
Outputs: []kio.Writer{kio.ByteWriter{Writer: actual, KeepReaderAnnotations: true}},
}.Execute()
if !assert.NoError(t, err) {
t.FailNow()
}
if !assert.Equal(t,
strings.TrimSpace(tt.expected),
strings.TrimSpace(actual.String())) {
t.FailNow()
}
})
}
}
func initTestDir(t *testing.T) string {
d, err := ioutil.TempDir("", "kustomize-annotate-test")
if !assert.NoError(t, err) {
t.FailNow()
}
err = ioutil.WriteFile(filepath.Join(d, "f1.yaml"), []byte(f1Input), 0600)
if !assert.NoError(t, err) {
defer os.RemoveAll(d)
t.FailNow()
}
err = ioutil.WriteFile(filepath.Join(d, "f2.yaml"), []byte(f2Input), 0600)
if !assert.NoError(t, err) {
defer os.RemoveAll(d)
t.FailNow()
}
return d
}
var (
f1Input = `
kind: Deployment
metadata:
labels:
app: nginx2
name: foo
annotations:
app: nginx2
spec:
replicas: 1
---
kind: Service
metadata:
name: foo
annotations:
app: nginx
spec:
selector:
app: nginx
`
f2Input = `
apiVersion: v1
kind: Abstraction
metadata:
name: foo
configFn:
container:
image: gcr.io/example/reconciler:v1
annotations:
config.kubernetes.io/local-config: "true"
namespace: bar
spec:
replicas: 3
---
apiVersion: apps/v1
kind: Deployment
metadata:
labels:
app: nginx
name: bar
annotations:
app: nginx
namespace: foo
spec:
replicas: 3
`
expectedSingleValue = `kind: Deployment
metadata:
labels:
app: nginx2
name: foo
annotations:
app: nginx2
a: 'b'
config.kubernetes.io/index: '0'
config.kubernetes.io/path: 'f1.yaml'
spec:
replicas: 1
---
kind: Service
metadata:
name: foo
annotations:
app: nginx
a: 'b'
config.kubernetes.io/index: '1'
config.kubernetes.io/path: 'f1.yaml'
spec:
selector:
app: nginx
---
apiVersion: v1
kind: Abstraction
metadata:
name: foo
configFn:
container:
image: gcr.io/example/reconciler:v1
annotations:
config.kubernetes.io/local-config: "true"
a: 'b'
config.kubernetes.io/index: '0'
config.kubernetes.io/path: 'f2.yaml'
namespace: bar
spec:
replicas: 3
---
apiVersion: apps/v1
kind: Deployment
metadata:
labels:
app: nginx
name: bar
annotations:
app: nginx
a: 'b'
config.kubernetes.io/index: '1'
config.kubernetes.io/path: 'f2.yaml'
namespace: foo
spec:
replicas: 3
`
expectedMultiValue = `kind: Deployment
metadata:
labels:
app: nginx2
name: foo
annotations:
app: nginx2
a: 'b'
c: 'd'
config.kubernetes.io/index: '0'
config.kubernetes.io/path: 'f1.yaml'
spec:
replicas: 1
---
kind: Service
metadata:
name: foo
annotations:
app: nginx
a: 'b'
c: 'd'
config.kubernetes.io/index: '1'
config.kubernetes.io/path: 'f1.yaml'
spec:
selector:
app: nginx
---
apiVersion: v1
kind: Abstraction
metadata:
name: foo
configFn:
container:
image: gcr.io/example/reconciler:v1
annotations:
config.kubernetes.io/local-config: "true"
a: 'b'
c: 'd'
config.kubernetes.io/index: '0'
config.kubernetes.io/path: 'f2.yaml'
namespace: bar
spec:
replicas: 3
---
apiVersion: apps/v1
kind: Deployment
metadata:
labels:
app: nginx
name: bar
annotations:
app: nginx
a: 'b'
c: 'd'
config.kubernetes.io/index: '1'
config.kubernetes.io/path: 'f2.yaml'
namespace: foo
spec:
replicas: 3
`
expectedFilterKindService = `kind: Deployment
metadata:
labels:
app: nginx2
name: foo
annotations:
app: nginx2
config.kubernetes.io/index: '0'
config.kubernetes.io/path: 'f1.yaml'
spec:
replicas: 1
---
kind: Service
metadata:
name: foo
annotations:
app: nginx
a: 'b'
config.kubernetes.io/index: '1'
config.kubernetes.io/path: 'f1.yaml'
spec:
selector:
app: nginx
---
apiVersion: v1
kind: Abstraction
metadata:
name: foo
configFn:
container:
image: gcr.io/example/reconciler:v1
annotations:
config.kubernetes.io/local-config: "true"
config.kubernetes.io/index: '0'
config.kubernetes.io/path: 'f2.yaml'
namespace: bar
spec:
replicas: 3
---
apiVersion: apps/v1
kind: Deployment
metadata:
labels:
app: nginx
name: bar
annotations:
app: nginx
config.kubernetes.io/index: '1'
config.kubernetes.io/path: 'f2.yaml'
namespace: foo
spec:
replicas: 3
`
expectedFilterApiVersionV1 = `kind: Deployment
metadata:
labels:
app: nginx2
name: foo
annotations:
app: nginx2
config.kubernetes.io/index: '0'
config.kubernetes.io/path: 'f1.yaml'
spec:
replicas: 1
---
kind: Service
metadata:
name: foo
annotations:
app: nginx
config.kubernetes.io/index: '1'
config.kubernetes.io/path: 'f1.yaml'
spec:
selector:
app: nginx
---
apiVersion: v1
kind: Abstraction
metadata:
name: foo
configFn:
container:
image: gcr.io/example/reconciler:v1
annotations:
config.kubernetes.io/local-config: "true"
a: 'b'
config.kubernetes.io/index: '0'
config.kubernetes.io/path: 'f2.yaml'
namespace: bar
spec:
replicas: 3
---
apiVersion: apps/v1
kind: Deployment
metadata:
labels:
app: nginx
name: bar
annotations:
app: nginx
config.kubernetes.io/index: '1'
config.kubernetes.io/path: 'f2.yaml'
namespace: foo
spec:
replicas: 3
`
expectedFilterNameBar = `kind: Deployment
metadata:
labels:
app: nginx2
name: foo
annotations:
app: nginx2
config.kubernetes.io/index: '0'
config.kubernetes.io/path: 'f1.yaml'
spec:
replicas: 1
---
kind: Service
metadata:
name: foo
annotations:
app: nginx
config.kubernetes.io/index: '1'
config.kubernetes.io/path: 'f1.yaml'
spec:
selector:
app: nginx
---
apiVersion: v1
kind: Abstraction
metadata:
name: foo
configFn:
container:
image: gcr.io/example/reconciler:v1
annotations:
config.kubernetes.io/local-config: "true"
config.kubernetes.io/index: '0'
config.kubernetes.io/path: 'f2.yaml'
namespace: bar
spec:
replicas: 3
---
apiVersion: apps/v1
kind: Deployment
metadata:
labels:
app: nginx
name: bar
annotations:
app: nginx
a: 'b'
config.kubernetes.io/index: '1'
config.kubernetes.io/path: 'f2.yaml'
namespace: foo
spec:
replicas: 3
`
expectedFilterNamespaceBar = `kind: Deployment
metadata:
labels:
app: nginx2
name: foo
annotations:
app: nginx2
config.kubernetes.io/index: '0'
config.kubernetes.io/path: 'f1.yaml'
spec:
replicas: 1
---
kind: Service
metadata:
name: foo
annotations:
app: nginx
config.kubernetes.io/index: '1'
config.kubernetes.io/path: 'f1.yaml'
spec:
selector:
app: nginx
---
apiVersion: v1
kind: Abstraction
metadata:
name: foo
configFn:
container:
image: gcr.io/example/reconciler:v1
annotations:
config.kubernetes.io/local-config: "true"
a: 'b'
config.kubernetes.io/index: '0'
config.kubernetes.io/path: 'f2.yaml'
namespace: bar
spec:
replicas: 3
---
apiVersion: apps/v1
kind: Deployment
metadata:
labels:
app: nginx
name: bar
annotations:
app: nginx
config.kubernetes.io/index: '1'
config.kubernetes.io/path: 'f2.yaml'
namespace: foo
spec:
replicas: 3
`
)

View File

@@ -4,6 +4,20 @@
// Code generated by "mdtogo"; DO NOT EDIT.
package commands
var AnnotateShort = `[Alpha] Set an annotation on Resources.`
var AnnotateLong = `
[Alpha] Set an annotation on Resources.
DIR:
Path to local directory.
`
var AnnotateExamples = `
kustomize config annotate my-dir/ --kv foo=bar
kustomize config annotate my-dir/ --kv foo=bar --kv a=b
kustomize config annotate my-dir/ --kv foo=bar --kind Deployment --name foo`
var CatShort = `[Alpha] Print Resource Config from a local directory.`
var CatLong = `
[Alpha] Print Resource Config from a local directory.

View File

@@ -11,11 +11,11 @@ require (
k8s.io/component-base v0.17.0 // indirect
k8s.io/kubectl v0.0.0-20191219154910-1528d4eea6dd
sigs.k8s.io/controller-runtime v0.4.0
sigs.k8s.io/kustomize/kstatus v0.0.0-20200109211150-9555095de939
sigs.k8s.io/kustomize/kstatus v0.0.1
sigs.k8s.io/kustomize/kyaml v0.0.2
)
replace (
sigs.k8s.io/kustomize/kstatus v0.0.0-20200109211150-9555095de939 => ../../kstatus
sigs.k8s.io/kustomize/kstatus v0.0.1 => ../../kstatus
sigs.k8s.io/kustomize/kyaml v0.0.0 => ../../kyaml
)

View File

@@ -6,6 +6,7 @@ package kubectlcobra
import (
"fmt"
"sort"
"strings"
"k8s.io/apimachinery/pkg/runtime/schema"
@@ -81,3 +82,99 @@ func (i *Inventory) String() string {
i.GroupKind.Group, fieldSeparator,
i.GroupKind.Kind)
}
// InventorySet encapsulates a grouping of unique Inventory
// structs. Organizes the Inventory structs with a map,
// which ensures there are no duplicates. Allows set
// operations such as merging sets and subtracting sets.
type InventorySet struct {
set map[string]*Inventory
}
// NewInventorySet returns a pointer to an InventorySet
// struct grouping the passed Inventory items.
func NewInventorySet(items []*Inventory) *InventorySet {
invSet := InventorySet{set: map[string]*Inventory{}}
invSet.AddItems(items)
return &invSet
}
// GetItems returns the set of pointers to Inventory
// structs.
func (is *InventorySet) GetItems() []*Inventory {
items := []*Inventory{}
for _, item := range is.set {
items = append(items, item)
}
return items
}
// AddItems adds Inventory structs to the set which
// are not already in the set.
func (is *InventorySet) AddItems(items []*Inventory) {
for _, item := range items {
if item != nil {
is.set[item.String()] = item
}
}
}
// DeleteItem removes an Inventory struct from the
// set if it exists in the set. Returns true if the
// Inventory item was deleted, false if it did not exist
// in the set.
func (is *InventorySet) DeleteItem(item *Inventory) bool {
if item == nil {
return false
}
if _, ok := is.set[item.String()]; ok {
delete(is.set, item.String())
return true
}
return false
}
// Merge combines the unique set of Inventory items from the
// current set with the passed "other" set, returning a new
// set or error. Returns an error if the passed set to merge
// is nil.
func (is *InventorySet) Merge(other *InventorySet) (*InventorySet, error) {
if other == nil {
return nil, fmt.Errorf("InventorySet to merge is nil.")
}
// Copy the current InventorySet into result
result := NewInventorySet(is.GetItems())
result.AddItems(other.GetItems())
return result, nil
}
// Subtract removes the Inventory items in the "other" set from the
// current set, returning a new set. This does not modify the current
// set. Returns an error if the passed set to subtract is nil.
func (is *InventorySet) Subtract(other *InventorySet) (*InventorySet, error) {
if other == nil {
return nil, fmt.Errorf("InventorySet to subtract is nil.")
}
// Copy the current InventorySet into result
result := NewInventorySet(is.GetItems())
// Remove each item in "other" which exists in "result"
for _, item := range other.GetItems() {
result.DeleteItem(item)
}
return result, nil
}
// String returns a string describing set of Inventory structs.
func (is *InventorySet) String() string {
strs := []string{}
for _, item := range is.GetItems() {
strs = append(strs, item.String())
}
sort.Strings(strs)
return strings.Join(strs, ", ")
}
// Size returns the number of Inventory structs in the set.
func (is *InventorySet) Size() int {
return len(is.set)
}

View File

@@ -216,3 +216,274 @@ func TestParseInventory(t *testing.T) {
}
}
}
var inventory1 = Inventory{
Namespace: "test-namespace",
Name: "test-inv-1",
GroupKind: schema.GroupKind{
Group: "apps",
Kind: "Deployment",
},
}
var inventory2 = Inventory{
Namespace: "test-namespace",
Name: "test-inv-2",
GroupKind: schema.GroupKind{
Group: "",
Kind: "Pod",
},
}
var inventory3 = Inventory{
Namespace: "test-namespace",
Name: "test-inv-3",
GroupKind: schema.GroupKind{
Group: "",
Kind: "Service",
},
}
var inventory4 = Inventory{
Namespace: "test-namespace",
Name: "test-inv-4",
GroupKind: schema.GroupKind{
Group: "apps",
Kind: "DaemonSet",
},
}
func TestNewInventorySet(t *testing.T) {
tests := []struct {
items []*Inventory
expectedStr string
expectedSize int
}{
{
items: []*Inventory{},
expectedStr: "",
expectedSize: 0,
},
{
items: []*Inventory{&inventory1},
expectedStr: "test-namespace_test-inv-1_apps_Deployment",
expectedSize: 1,
},
{
items: []*Inventory{&inventory1, &inventory2},
expectedStr: "test-namespace_test-inv-1_apps_Deployment, test-namespace_test-inv-2__Pod",
expectedSize: 2,
},
}
for _, test := range tests {
invSet := NewInventorySet(test.items)
actualStr := invSet.String()
actualSize := invSet.Size()
if test.expectedStr != actualStr {
t.Errorf("Expected InventorySet (%s), got (%s)\n", test.expectedStr, actualStr)
}
if test.expectedSize != actualSize {
t.Errorf("Expected InventorySet size (%d), got (%d)\n", test.expectedSize, actualSize)
}
actualItems := invSet.GetItems()
if len(test.items) != len(actualItems) {
t.Errorf("Expected num inventory items (%d), got (%d)\n", len(test.items), len(actualItems))
}
}
}
func TestInventorySetAddItems(t *testing.T) {
tests := []struct {
initialItems []*Inventory
addItems []*Inventory
expectedItems []*Inventory
}{
// Adding no items to empty inventory set.
{
initialItems: []*Inventory{},
addItems: []*Inventory{},
expectedItems: []*Inventory{},
},
// Adding item to empty inventory set.
{
initialItems: []*Inventory{},
addItems: []*Inventory{&inventory1},
expectedItems: []*Inventory{&inventory1},
},
// Adding no items does not change the inventory set
{
initialItems: []*Inventory{&inventory1},
addItems: []*Inventory{},
expectedItems: []*Inventory{&inventory1},
},
// Adding an item which alread exists does not increase size.
{
initialItems: []*Inventory{&inventory1, &inventory2},
addItems: []*Inventory{&inventory1},
expectedItems: []*Inventory{&inventory1, &inventory2},
},
{
initialItems: []*Inventory{&inventory1, &inventory2},
addItems: []*Inventory{&inventory3, &inventory4},
expectedItems: []*Inventory{&inventory1, &inventory2, &inventory3, &inventory4},
},
}
for _, test := range tests {
invSet := NewInventorySet(test.initialItems)
invSet.AddItems(test.addItems)
if len(test.expectedItems) != invSet.Size() {
t.Errorf("Expected num inventory items (%d), got (%d)\n", len(test.expectedItems), invSet.Size())
}
}
}
func TestInventorySetDeleteItem(t *testing.T) {
tests := []struct {
initialItems []*Inventory
deleteItem *Inventory
expected bool
expectedItems []*Inventory
}{
{
initialItems: []*Inventory{},
deleteItem: nil,
expected: false,
expectedItems: []*Inventory{},
},
{
initialItems: []*Inventory{},
deleteItem: &inventory1,
expected: false,
expectedItems: []*Inventory{},
},
{
initialItems: []*Inventory{&inventory2},
deleteItem: &inventory1,
expected: false,
expectedItems: []*Inventory{&inventory2},
},
{
initialItems: []*Inventory{&inventory1},
deleteItem: &inventory1,
expected: true,
expectedItems: []*Inventory{},
},
{
initialItems: []*Inventory{&inventory1, &inventory2},
deleteItem: &inventory1,
expected: true,
expectedItems: []*Inventory{&inventory2},
},
}
for _, test := range tests {
invSet := NewInventorySet(test.initialItems)
actual := invSet.DeleteItem(test.deleteItem)
if test.expected != actual {
t.Errorf("Expected return value (%t), got (%t)\n", test.expected, actual)
}
if len(test.expectedItems) != invSet.Size() {
t.Errorf("Expected num inventory items (%d), got (%d)\n", len(test.expectedItems), invSet.Size())
}
}
}
func TestInventorySetMerge(t *testing.T) {
tests := []struct {
set1 []*Inventory
set2 []*Inventory
merged []*Inventory
}{
{
set1: []*Inventory{},
set2: []*Inventory{},
merged: []*Inventory{},
},
{
set1: []*Inventory{},
set2: []*Inventory{&inventory1},
merged: []*Inventory{&inventory1},
},
{
set1: []*Inventory{&inventory1},
set2: []*Inventory{},
merged: []*Inventory{&inventory1},
},
{
set1: []*Inventory{&inventory1, &inventory2},
set2: []*Inventory{&inventory1},
merged: []*Inventory{&inventory1, &inventory2},
},
{
set1: []*Inventory{&inventory1, &inventory2},
set2: []*Inventory{&inventory1, &inventory2},
merged: []*Inventory{&inventory1, &inventory2},
},
{
set1: []*Inventory{&inventory1, &inventory2},
set2: []*Inventory{&inventory3, &inventory4},
merged: []*Inventory{&inventory1, &inventory2, &inventory3, &inventory4},
},
}
for _, test := range tests {
invSet1 := NewInventorySet(test.set1)
invSet2 := NewInventorySet(test.set2)
expected := NewInventorySet(test.merged)
merged, _ := invSet1.Merge(invSet2)
if expected.Size() != merged.Size() {
t.Errorf("Expected merged inventory set size (%d), got (%d)\n", expected.Size(), merged.Size())
}
}
}
func TestInventorySetSubtract(t *testing.T) {
tests := []struct {
initialItems []*Inventory
subtractItems []*Inventory
expected []*Inventory
}{
{
initialItems: []*Inventory{},
subtractItems: []*Inventory{},
expected: []*Inventory{},
},
{
initialItems: []*Inventory{},
subtractItems: []*Inventory{&inventory1},
expected: []*Inventory{},
},
{
initialItems: []*Inventory{&inventory1},
subtractItems: []*Inventory{},
expected: []*Inventory{&inventory1},
},
{
initialItems: []*Inventory{&inventory1, &inventory2},
subtractItems: []*Inventory{&inventory1},
expected: []*Inventory{&inventory2},
},
{
initialItems: []*Inventory{&inventory1, &inventory2},
subtractItems: []*Inventory{&inventory1, &inventory2},
expected: []*Inventory{},
},
{
initialItems: []*Inventory{&inventory1, &inventory2},
subtractItems: []*Inventory{&inventory3, &inventory4},
expected: []*Inventory{&inventory1, &inventory2},
},
}
for _, test := range tests {
invInitialItems := NewInventorySet(test.initialItems)
invSubtractItems := NewInventorySet(test.subtractItems)
expected := NewInventorySet(test.expected)
actual, _ := invInitialItems.Subtract(invSubtractItems)
if expected.Size() != actual.Size() {
t.Errorf("Expected subtracted inventory set size (%d), got (%d)\n", expected.Size(), actual.Size())
}
}
}

View File

@@ -0,0 +1,113 @@
[base]: ../../docs/glossary.md#base
[config]: https://github.com/kinflate/example-hello
[gitops]: ../../docs/glossary.md#gitops
[hello]: https://github.com/monopole/hello
[kustomization]: ../../docs/glossary.md#kustomization
[original]: https://github.com/kinflate/example-hello
[overlay]: ../../docs/glossary.md#overlay
[overlays]: ../../docs/glossary.md#overlay
[patch]: ../../docs/glossary.md#patch
[variant]: ../../docs/glossary.md#variant
[variants]: ../../docs/glossary.md#variant
# Demo: hello world
Steps:
1. Clone an existing configuration as a [base].
1. Customize it.
First define a place to work:
<!-- @makeWorkplace @testE2EAgainstLatestRelease-->
```
DEMO_HOME=$(mktemp -d)
```
Alternatively, use
> ```
> DEMO_HOME=~/hello
> ```
## Establish the base
Let's run the [hello] service.
To keep this document shorter, the base resources are
off in a supplemental data directory rather than
declared here as HERE documents. Download them:
<!-- @downloadBase @testE2EAgainstLatestRelease-->
```
BASE=$DEMO_HOME/base
mkdir -p $BASE
curl -s -o "$BASE/#1.yaml" "https://raw.githubusercontent.com\
/kubernetes-sigs/kustomize\
/master/examples/alphaTestExamples/helloWorld\
/{configMap,deployment,grouping,kustomization,service}.yaml"
```
### The Base Kustomization
The `base` directory has a [kustomization] file:
<!-- @showKustomization @testE2EAgainstLatestRelease -->
```
more $BASE/kustomization.yaml
```
### Customize the base
A first customization step could be to change the _app
label_ applied to all resources:
<!-- @addLabel @testE2EAgainstLatestRelease -->
```
sed -i.bak 's/app: hello/app: my-hello/' \
$BASE/kustomization.yaml
```
To do end to end tests using kustomize, go through the following section. You should have GOPATH set up and "kind" installed(https://github.com/kubernetes-sigs/kind).
<!-- @setGoBin @testE2EAgainstLatestRelease -->
```
MYGOBIN=$GOPATH/bin
```
Delete any existing kind cluster and create a new one. By default the name of the cluster is "kind"
<!-- @deleteAndCreateKindCluster @testE2EAgainstLatestRelease -->
```
kind delete cluster;
kind create cluster;
```
Use the kustomize binary in MYGOBIN to apply a deployment, fetch the status and verify the status.
<!-- @e2eTestUsingKustomize @testE2EAgainstLatestRelease -->
```
export KUSTOMIZE_ENABLE_ALPHA_COMMANDS=true
$MYGOBIN/kustomize resources apply $BASE --status;
status=$(mktemp);
$MYGOBIN/resource status fetch $BASE > $status
test 1 == \
$(grep "the-deployment" $status | grep "Deployment is available. Replicas: 3" | wc -l); \
echo $?
test 1 == \
$(grep "the-map" $status | grep "Resource is always ready" | wc -l); \
echo $?
test 1 == \
$(grep "the-service" $status | grep "Service is ready" | wc -l); \
echo $?
```
Clean-up the cluster
<!-- @createKindCluster @testE2EAgainstLatestRelease -->
```
kind delete cluster;
```

View File

@@ -0,0 +1,7 @@
apiVersion: v1
kind: ConfigMap
metadata:
name: the-map
data:
altGreeting: "Good Morning!"
enableRisky: "false"

View File

@@ -0,0 +1,30 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: the-deployment
spec:
replicas: 3
template:
metadata:
labels:
deployment: hello
spec:
containers:
- name: the-container
image: monopole/hello:1
command: ["/hello",
"--port=8080",
"--enableRiskyFeature=$(ENABLE_RISKY)"]
ports:
- containerPort: 8080
env:
- name: ALT_GREETING
valueFrom:
configMapKeyRef:
name: the-map
key: altGreeting
- name: ENABLE_RISKY
valueFrom:
configMapKeyRef:
name: the-map
key: enableRisky

View File

@@ -0,0 +1,6 @@
apiVersion: v1
kind: ConfigMap
metadata:
name: inventory-map
labels:
"kustomize.config.k8s.io/inventory-id": "hello-world-app"

View File

@@ -0,0 +1,10 @@
# Example configuration for the webserver
# at https://github.com/monopole/hello
commonLabels:
app: hello
resources:
- deployment.yaml
- service.yaml
- configMap.yaml
- grouping.yaml

View File

@@ -0,0 +1,12 @@
kind: Service
apiVersion: v1
metadata:
name: the-service
spec:
selector:
deployment: hello
type: LoadBalancer
ports:
- protocol: TCP
port: 8666
targetPort: 8080

View File

@@ -22,7 +22,7 @@ Steps:
First define a place to work:
<!-- @makeWorkplace @testAgainstLatestRelease @testE2EAgainstLatestRelease-->
<!-- @makeWorkplace @testAgainstLatestRelease -->
```
DEMO_HOME=$(mktemp -d)
```
@@ -44,7 +44,7 @@ To keep this document shorter, the base resources are
off in a supplemental data directory rather than
declared here as HERE documents. Download them:
<!-- @downloadBase @testAgainstLatestRelease @testE2EAgainstLatestRelease-->
<!-- @downloadBase @testAgainstLatestRelease -->
```
BASE=$DEMO_HOME/base
mkdir -p $BASE
@@ -309,31 +309,3 @@ To deploy, pipe the above commands to kubectl apply:
> kustomize build $OVERLAYS/production |\
> kubectl apply -f -
> ```
[Alpha] To do end to end tests using kustomize, use the following commands on any folder. You should have GOPATH set up and "kind" installed(https://github.com/kubernetes-sigs/kind).
<!-- @e2eTest @testE2EAgainstLatestRelease -->
```
MYGOBIN=$GOPATH/bin
kind delete cluster;
kind create cluster;
$MYGOBIN/kustomize build $BASE | kubectl apply -f -;
status=$(mktemp);
$MYGOBIN/resource status events $BASE #Waits for all transient events to finish
$MYGOBIN/resource status fetch $BASE > $status
test 1 == \
$(grep "apps/v1/Deployment" $status | grep "Deployment is available. Replicas: 3" | wc -l); \
echo $?
test 1 == \
$(grep "v1/ConfigMap" $status | grep "Resource is always ready" | wc -l); \
echo $?
test 1 == \
$(grep "v1/Service" $status | grep "Service is ready" | wc -l); \
echo $?
$MYGOBIN/kustomize build $BASE | kubectl delete -f -;
kind delete cluster;
```

View File

@@ -8,6 +8,6 @@ set -o errexit
set -o pipefail
mdrip --blockTimeOut 60m0s --mode test \
--label testE2EAgainstLatestRelease examples
--label testE2EAgainstLatestRelease examples/alphaTestExamples
echo "Example e2e tests passed against ${version}."
echo "Example e2e tests passed against ."

View File

@@ -18,3 +18,8 @@ exclude (
github.com/russross/blackfriday v2.0.0+incompatible
sigs.k8s.io/kustomize/api v0.2.0
)
replace (
sigs.k8s.io/kustomize/cmd/kubectl v0.0.3 => ../cmd/kubectl
sigs.k8s.io/kustomize/kstatus v0.0.1 => ../kstatus
)

View File

@@ -24,6 +24,8 @@ type flagsAndArgs struct {
EnvFileSource string
// Type of secret to create
Type string
// Namespace of secret
Namespace string
}
// Validate validates required fields are set to support structured generation.

View File

@@ -86,6 +86,11 @@ func newCmdAddSecret(
"type",
"Opaque",
"Specify the secret type this can be 'Opaque' (default), or 'kubernetes.io/tls'")
cmd.Flags().StringVar(
&flags.Namespace,
"namespace",
"",
"Specify the namespace of the secret")
return cmd
}
@@ -97,7 +102,7 @@ func addSecret(
ldr ifc.KvLoader,
k *types.Kustomization,
flags flagsAndArgs, kf ifc.KunstructuredFactory) error {
args := findOrMakeSecretArgs(k, flags.Name, flags.Type)
args := findOrMakeSecretArgs(k, flags.Name, flags.Namespace, flags.Type)
mergeFlagsIntoGeneratorArgs(&args.GeneratorArgs, flags)
// Validate by trying to create corev1.secret.
_, err := kf.MakeSecret(ldr, k.GeneratorOptions, args)
@@ -107,16 +112,17 @@ func addSecret(
return nil
}
func findOrMakeSecretArgs(m *types.Kustomization, name, secretType string) *types.SecretArgs {
func findOrMakeSecretArgs(m *types.Kustomization, name, namespace, secretType string) *types.SecretArgs {
for i, v := range m.SecretGenerator {
if name == v.Name {
if name == v.Name && namespace == v.Namespace {
return &m.SecretGenerator[i]
}
}
// secret not found, create new one and add it to the kustomization file.
secret := &types.SecretArgs{
GeneratorArgs: types.GeneratorArgs{Name: name},
Type: secretType}
GeneratorArgs: types.GeneratorArgs{Name: name, Namespace: namespace},
Type: secretType,
}
m.SecretGenerator = append(m.SecretGenerator, *secret)
return &m.SecretGenerator[len(m.SecretGenerator)-1]
}

View File

@@ -28,6 +28,7 @@ func TestNewCmdAddSecretIsNotNil(t *testing.T) {
func TestMakeSecretArgs(t *testing.T) {
secretName := "test-secret-name"
namespace := "test-secret-namespace"
kustomization := &types.Kustomization{
NamePrefix: "test-name-prefix",
@@ -38,7 +39,7 @@ func TestMakeSecretArgs(t *testing.T) {
if len(kustomization.SecretGenerator) != 0 {
t.Fatal("Initial kustomization should not have any secrets")
}
args := findOrMakeSecretArgs(kustomization, secretName, secretType)
args := findOrMakeSecretArgs(kustomization, secretName, namespace, secretType)
if args == nil {
t.Fatalf("args should always be non-nil")
@@ -52,7 +53,7 @@ func TestMakeSecretArgs(t *testing.T) {
t.Fatalf("Pointer address for newly inserted secret generator should be same")
}
args2 := findOrMakeSecretArgs(kustomization, secretName, secretType)
args2 := findOrMakeSecretArgs(kustomization, secretName, namespace, secretType)
if args2 != args {
t.Fatalf("should have returned an existing args with name: %v", secretName)
@@ -65,7 +66,7 @@ func TestMakeSecretArgs(t *testing.T) {
func TestMergeFlagsIntoSecretArgs_LiteralSources(t *testing.T) {
k := &types.Kustomization{}
args := findOrMakeSecretArgs(k, "foo", "forbidden")
args := findOrMakeSecretArgs(k, "foo", "bar", "forbidden")
mergeFlagsIntoGeneratorArgs(
&args.GeneratorArgs,
flagsAndArgs{LiteralSources: []string{"k1=v1"}})
@@ -82,7 +83,7 @@ func TestMergeFlagsIntoSecretArgs_LiteralSources(t *testing.T) {
func TestMergeFlagsIntoSecretArgs_FileSources(t *testing.T) {
k := &types.Kustomization{}
args := findOrMakeSecretArgs(k, "foo", "forbidden")
args := findOrMakeSecretArgs(k, "foo", "bar", "forbidden")
mergeFlagsIntoGeneratorArgs(
&args.GeneratorArgs,
flagsAndArgs{FileSources: []string{"file1"}})
@@ -99,7 +100,7 @@ func TestMergeFlagsIntoSecretArgs_FileSources(t *testing.T) {
func TestMergeFlagsIntoSecretArgs_EnvSource(t *testing.T) {
k := &types.Kustomization{}
args := findOrMakeSecretArgs(k, "foo", "forbidden")
args := findOrMakeSecretArgs(k, "foo", "bar", "forbidden")
mergeFlagsIntoGeneratorArgs(
&args.GeneratorArgs,
flagsAndArgs{EnvFileSource: "env1"})

View File

@@ -40,10 +40,11 @@ function parseYaml {
local file=$1
while read -r line
do
local k=${line%:*}
local k=${line%%:*}
local v=${line#*:}
[ "$k" == "chartName" ] && chartName=$v
[ "$k" == "chartRepo" ] && chartRepo=$v
[ "$k" == "chartHome" ] && chartHome=$v
[ "$k" == "chartRelease" ] && chartRelease=$v
[ "$k" == "chartVersion" ] && chartVersion=$v
@@ -56,6 +57,7 @@ function parseYaml {
# Trim leading space
chartName="${chartName#"${chartName%%[![:space:]]*}"}"
chartRepo="${chartRepo#"${chartRepo%%[![:space:]]*}"}"
chartHome="${chartHome#"${chartHome%%[![:space:]]*}"}"
chartRelease="${chartRelease#"${chartRelease%%[![:space:]]*}"}"
chartVersion="${chartVersion#"${chartVersion%%[![:space:]]*}"}"
@@ -84,6 +86,14 @@ if [ -z "$chartRelease" ]; then
chartRelease="stable"
fi
# The repo to pull the chart from
if [ -n "$chartRepo" ]; then
chartRepoArg="--repo=$chartRepo"
chartNameArg="$chartName"
else
chartNameArg="$chartRelease/$chartName"
fi
# Set version only if specified
if [ ! -z "$chartVersion" ]; then
chartVersionArg="--version=$chartVersion"
@@ -114,9 +124,10 @@ doHelm init --client-only >& /dev/null
if [ ! -d "$chartHome/$chartName" ]; then
doHelm fetch $chartVersionArg \
$chartRepoArg \
--untar \
--untardir $chartHome \
${chartRelease}/$chartName
$chartNameArg
fi
doHelm template \

View File

@@ -6,7 +6,7 @@
# kyaml version
export kyaml_major=0
export kyaml_minor=0
export kyaml_patch=7
export kyaml_patch=8
# kstatus version
export kstatus_major=0
@@ -21,7 +21,7 @@ export api_patch=2
# cmd/config version
export cmd_config_major=0
export cmd_config_minor=0
export cmd_config_patch=8
export cmd_config_patch=9
# cmd/kubectl version
export cmd_kubectl_major=0