mirror of
https://github.com/kubernetes-sigs/kustomize.git
synced 2026-06-28 17:28:18 +00:00
Merge remote-tracking branch 'upstream/master' into release-cmd/config-v0.0
This commit is contained in:
14
Makefile
14
Makefile
@@ -18,7 +18,7 @@ verify-kustomize: \
|
||||
test-examples-kustomize-against-latest
|
||||
|
||||
.PHONY: verify-kustomize-e2e
|
||||
verify-kustomize-e2e: test-examples-e2e-kustomize-against-HEAD
|
||||
verify-kustomize-e2e: test-examples-e2e-kustomize
|
||||
|
||||
# Other builds in this repo might want a different linter version.
|
||||
# Without one Makefile to rule them all, the different makes
|
||||
@@ -202,8 +202,16 @@ test-examples-kustomize-against-HEAD: $(MYGOBIN)/kustomize $(MYGOBIN)/mdrip
|
||||
./hack/testExamplesAgainstKustomize.sh HEAD
|
||||
|
||||
.PHONY:
|
||||
test-examples-e2e-kustomize-against-HEAD: $(MYGOBIN)/kustomize $(MYGOBIN)/mdrip $(MYGOBIN)/resource
|
||||
./hack/testExamplesE2EAgainstKustomize.sh HEAD
|
||||
test-examples-e2e-kustomize: $(MYGOBIN)/mdrip
|
||||
( \
|
||||
set -e; \
|
||||
/bin/rm -f $(MYGOBIN)/kustomize; \
|
||||
echo "Installing kustomize from ."; \
|
||||
cd kustomize; go install .; cd ..; \
|
||||
echo "Installing resource from ."; \
|
||||
cd cmd/resource; go install .; cd ../..; \
|
||||
./hack/testExamplesE2EAgainstKustomize.sh .; \
|
||||
)
|
||||
|
||||
.PHONY:
|
||||
test-examples-kustomize-against-latest: $(MYGOBIN)/mdrip
|
||||
|
||||
@@ -44,7 +44,7 @@ type kustomizeSearch struct {
|
||||
// /register: not implemented, but meant as an endpoint for adding new
|
||||
// kustomization files to the corpus.
|
||||
func NewKustomizeSearch(ctx context.Context) (*kustomizeSearch, error) {
|
||||
idx, err := index.NewKustomizeIndex(ctx)
|
||||
idx, err := index.NewKustomizeIndex(ctx, "kustomize")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -9,6 +9,8 @@ import (
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"sigs.k8s.io/kustomize/api/internal/crawl/utils"
|
||||
|
||||
"sigs.k8s.io/kustomize/api/internal/crawl/crawler"
|
||||
"sigs.k8s.io/kustomize/api/internal/crawl/crawler/github"
|
||||
"sigs.k8s.io/kustomize/api/internal/crawl/doc"
|
||||
@@ -26,6 +28,7 @@ const (
|
||||
)
|
||||
|
||||
type CrawlMode int
|
||||
|
||||
const (
|
||||
CrawlUnknown CrawlMode = iota
|
||||
// Crawl all the kustomization files in all the repositories of a Github user
|
||||
@@ -125,13 +128,13 @@ func main() {
|
||||
|
||||
// seen tracks the IDs of all the documents in the index.
|
||||
// This helps avoid indexing a given document multiple times.
|
||||
seen := crawler.NewSeenMap()
|
||||
seen := utils.NewSeenMap()
|
||||
|
||||
mode := NewCrawlMode(*modePtr)
|
||||
|
||||
ghCrawlerConstructor := func(user, repo string) crawler.Crawler {
|
||||
if user != "" {
|
||||
return github.NewCrawler(githubToken, retryCount, clientCache,
|
||||
return github.NewCrawler(githubToken, retryCount, clientCache,
|
||||
github.QueryWith(
|
||||
github.Filename("kustomization.yaml"),
|
||||
github.Filename("kustomization.yml"),
|
||||
|
||||
@@ -5,12 +5,21 @@ import (
|
||||
"flag"
|
||||
"fmt"
|
||||
"log"
|
||||
"path/filepath"
|
||||
"net/http"
|
||||
"os"
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
"sigs.k8s.io/kustomize/api/internal/crawl/crawler/github"
|
||||
|
||||
"sigs.k8s.io/kustomize/api/internal/crawl/doc"
|
||||
|
||||
"sigs.k8s.io/kustomize/api/internal/crawl/index"
|
||||
"sigs.k8s.io/kustomize/api/konfig"
|
||||
)
|
||||
|
||||
const (
|
||||
githubAccessTokenVar = "GITHUB_ACCESS_TOKEN"
|
||||
retryCount = 3
|
||||
)
|
||||
|
||||
// iterateArr adds each item in arr into countMap.
|
||||
@@ -25,17 +34,6 @@ func iterateArr(arr []string, countMap map[string]int) {
|
||||
|
||||
}
|
||||
|
||||
// isKustomizationFile determines whether a file path is a kustomization file
|
||||
func isKustomizationFile(path string) bool {
|
||||
basename := filepath.Base(path)
|
||||
for _, name := range konfig.RecognizedKustomizationFileNames() {
|
||||
if basename == name {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// SortMapKeyByValue takes a map as its input, sorts its keys according to their values
|
||||
// in the map, and outputs the sorted keys as a slice.
|
||||
func SortMapKeyByValue(m map[string]int) []string {
|
||||
@@ -44,10 +42,129 @@ func SortMapKeyByValue(m map[string]int) []string {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
// sort keys according to their values in the map m
|
||||
sort.Slice(keys, func(i, j int) bool {return m[keys[i]] > m[keys[j]]})
|
||||
sort.Slice(keys, func(i, j int) bool { return m[keys[i]] > m[keys[j]] })
|
||||
return keys
|
||||
}
|
||||
|
||||
func GeneratorOrTransformerStats(ctx context.Context,
|
||||
docs []*doc.Document, isGenerator bool, idx *index.KustomizeIndex) {
|
||||
|
||||
fieldName := "generators"
|
||||
if !isGenerator {
|
||||
fieldName = "transformers"
|
||||
}
|
||||
|
||||
// allReferredDocs includes all the documents referred in the field
|
||||
allReferredDocs := doc.NewUniqueDocuments()
|
||||
|
||||
// docUsingGeneratorCount counts the number of the kustomization files using generators or transformers
|
||||
docCount := 0
|
||||
|
||||
// collect all the documents referred in the field
|
||||
for _, d := range docs {
|
||||
kdoc := doc.KustomizationDocument{
|
||||
Document: *d,
|
||||
}
|
||||
referredDocs, err := kdoc.GetResources(false, !isGenerator, isGenerator)
|
||||
if err != nil {
|
||||
log.Printf("failed to parse the %s field of the Document (%s): %v",
|
||||
fieldName, d.Path(), err)
|
||||
}
|
||||
if len(referredDocs) > 0 {
|
||||
docCount++
|
||||
allReferredDocs.AddDocuments(referredDocs)
|
||||
}
|
||||
}
|
||||
|
||||
fileCount, dirCount, fileTypeDocs, dirTypeDocs := DocumentTypeSummary(ctx, allReferredDocs.Documents())
|
||||
|
||||
// check whether any of the files are not in the index
|
||||
nonExistFileCount := ExistInIndex(idx, fileTypeDocs, fieldName + " file ")
|
||||
// check whether any of the dirs are not in the index
|
||||
nonExistDirCount := ExistInIndex(idx, dirTypeDocs, fieldName + " dir ")
|
||||
|
||||
GitRepositorySummary(fileTypeDocs, fieldName + " files")
|
||||
GitRepositorySummary(dirTypeDocs, fieldName + " dirs")
|
||||
|
||||
fmt.Printf("%d kustomization files use %s: %d %s are files and %d %s are dirs.\n",
|
||||
docCount, fieldName, fileCount, fieldName, dirCount, fieldName)
|
||||
fmt.Printf("%d %s files do not exist in the index\n", nonExistFileCount, fieldName)
|
||||
fmt.Printf("%d %s dirs do not exist in the index\n", nonExistDirCount, fieldName)
|
||||
}
|
||||
|
||||
// GitRepositorySummary counts the distribution of docs:
|
||||
// 1) how many git repositories are these docs from?
|
||||
// 2) how many docs are from each git repository?
|
||||
func GitRepositorySummary(docs []*doc.Document, msgPrefix string) {
|
||||
m := make(map[string]int)
|
||||
for _, d := range docs {
|
||||
if _, ok := m[d.RepositoryURL]; ok {
|
||||
m[d.RepositoryURL]++
|
||||
} else {
|
||||
m[d.RepositoryURL] = 1
|
||||
}
|
||||
}
|
||||
sortedKeys := SortMapKeyByValue(m)
|
||||
for _, k := range sortedKeys {
|
||||
fmt.Printf("%d %s are from %s\n", m[k], msgPrefix, k)
|
||||
}
|
||||
}
|
||||
|
||||
// ExistInIndex goes through each Document in docs, and check whether it is in the index or not.
|
||||
// It returns the number of documents which does not exist in the index.
|
||||
func ExistInIndex(idx *index.KustomizeIndex, docs []*doc.Document, msgPrefix string) int {
|
||||
nonExistCount := 0
|
||||
for _, d := range docs {
|
||||
exists, err := idx.Exists(d.ID())
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
}
|
||||
if !exists {
|
||||
log.Printf("%s (%s) does not exist in the index", msgPrefix, d.Path())
|
||||
nonExistCount++
|
||||
}
|
||||
}
|
||||
return nonExistCount
|
||||
}
|
||||
|
||||
// DocumentTypeSummary goes through each doc in docs, and determines whether it is a file or dir.
|
||||
func DocumentTypeSummary(ctx context.Context, docs []*doc.Document) (
|
||||
fileCount, dirCount int, files, dirs []*doc.Document) {
|
||||
githubToken := os.Getenv(githubAccessTokenVar)
|
||||
if githubToken == "" {
|
||||
log.Fatalf("Must set the variable '%s' to make github requests.\n",
|
||||
githubAccessTokenVar)
|
||||
}
|
||||
ghCrawler := github.NewCrawler(githubToken, retryCount, &http.Client{}, github.QueryWith())
|
||||
|
||||
for _, d := range docs {
|
||||
oldFilePath := d.FilePath
|
||||
if err := ghCrawler.FetchDocument(ctx, d); err != nil {
|
||||
log.Printf("FetchDocument failed on %s: %v", d.Path(), err)
|
||||
continue
|
||||
}
|
||||
|
||||
if d.FilePath == oldFilePath {
|
||||
fileCount++
|
||||
files = append(files, d)
|
||||
} else {
|
||||
dirCount++
|
||||
dirs = append(dirs, d)
|
||||
}
|
||||
}
|
||||
return fileCount, dirCount, files, dirs
|
||||
}
|
||||
|
||||
// ExistInSlice checks where target exits in items.
|
||||
func ExistInSlice(items []string, target string) bool {
|
||||
for _, item := range items {
|
||||
if item == target {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func main() {
|
||||
topKindsPtr := flag.Int(
|
||||
"kinds", -1,
|
||||
@@ -64,10 +181,12 @@ If you only want to list the 10 most popular identifiers, set the flag to 10.`)
|
||||
`the number of kustomize features to be listed according to their popularities.
|
||||
By default, all the features will be listed.
|
||||
If you only want to list the 10 most popular features, set the flag to 10.`)
|
||||
indexNamePtr := flag.String(
|
||||
"index", "kustomize", "The name of the ElasticSearch index.")
|
||||
flag.Parse()
|
||||
|
||||
ctx := context.Background()
|
||||
idx, err := index.NewKustomizeIndex(ctx)
|
||||
idx, err := index.NewKustomizeIndex(ctx, *indexNamePtr)
|
||||
if err != nil {
|
||||
log.Fatalf("Could not create an index: %v\n", err)
|
||||
}
|
||||
@@ -85,6 +204,12 @@ If you only want to list the 10 most popular features, set the flag to 10.`)
|
||||
// ids tracks the unique IDs of the documents in the index
|
||||
ids := make(map[string]struct{})
|
||||
|
||||
// generatorDocs includes all the docs using generators
|
||||
generatorDocs := make([]*doc.Document, 0)
|
||||
|
||||
// transformersDocs includes all the docs using transformers
|
||||
transformersDocs := make([]*doc.Document, 0)
|
||||
|
||||
// get all the documents in the index
|
||||
query := []byte(`{ "query":{ "match_all":{} } }`)
|
||||
it := idx.IterateQuery(query, 10000, 60*time.Second)
|
||||
@@ -94,21 +219,28 @@ If you only want to list the 10 most popular features, set the flag to 10.`)
|
||||
if _, ok := ids[hit.ID]; !ok {
|
||||
ids[hit.ID] = struct{}{}
|
||||
} else {
|
||||
fmt.Printf("Found duplicate ID (%s)\n", hit.ID)
|
||||
log.Printf("Found duplicate ID (%s)\n", hit.ID)
|
||||
}
|
||||
|
||||
count++
|
||||
iterateArr(hit.Document.Kinds, kindsMap)
|
||||
iterateArr(hit.Document.Identifiers, identifiersMap)
|
||||
|
||||
if isKustomizationFile(hit.Document.FilePath) {
|
||||
if doc.IsKustomizationFile(hit.Document.FilePath) {
|
||||
kustomizationFilecount++
|
||||
iterateArr(hit.Document.Identifiers, kustomizeIdentifiersMap)
|
||||
if ExistInSlice(hit.Document.Identifiers, "generators") {
|
||||
generatorDocs = append(generatorDocs, hit.Document.Copy())
|
||||
}
|
||||
if ExistInSlice(hit.Document.Identifiers, "transformers") {
|
||||
transformersDocs = append(transformersDocs, hit.Document.Copy())
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if err := it.Err(); err != nil {
|
||||
fmt.Printf("Error iterating: %v\n", err)
|
||||
log.Fatalf("Error iterating: %v\n", err)
|
||||
}
|
||||
|
||||
sortedKindsMapKeys := SortMapKeyByValue(kindsMap)
|
||||
@@ -147,4 +279,7 @@ There are %d documents in the kustomize index.
|
||||
kustomizeFeatureCount++
|
||||
}
|
||||
}
|
||||
|
||||
GeneratorOrTransformerStats(ctx, generatorDocs, true, idx)
|
||||
GeneratorOrTransformerStats(ctx, transformersDocs, false, idx)
|
||||
}
|
||||
|
||||
@@ -11,8 +11,13 @@ spec:
|
||||
image: gcr.io/haiyanmeng-gke-dev/kustomize_stats:v1
|
||||
imagePullPolicy: Always
|
||||
command: ["/kustomize_stats"]
|
||||
args: ["--kinds=50", "--identifiers=50", "--kustomize-features=50"]
|
||||
args: ["--index=kustomize", "--kinds=50", "--identifiers=50", "--kustomize-features=50"]
|
||||
env:
|
||||
- name: GITHUB_ACCESS_TOKEN
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: github-access-token
|
||||
key: token
|
||||
- name: ELASTICSEARCH_URL
|
||||
valueFrom:
|
||||
configMapKeyRef:
|
||||
|
||||
@@ -10,6 +10,8 @@ import (
|
||||
"os"
|
||||
"sync"
|
||||
|
||||
"sigs.k8s.io/kustomize/api/internal/crawl/utils"
|
||||
|
||||
"sigs.k8s.io/kustomize/api/internal/crawl/index"
|
||||
|
||||
_ "github.com/gomodule/redigo/redis"
|
||||
@@ -29,7 +31,7 @@ type Crawler interface {
|
||||
// Crawl returns when it is done processing. This method does not take
|
||||
// ownership of the channel. The channel is write only, and it
|
||||
// designates where the crawler should forward the documents.
|
||||
Crawl(ctx context.Context, output chan<- CrawledDocument, seen SeenMap) error
|
||||
Crawl(ctx context.Context, output chan<- CrawledDocument, seen utils.SeenMap) error
|
||||
|
||||
// Get the document data given the FilePath, Repo, and Ref/Tag/Branch.
|
||||
FetchDocument(context.Context, *doc.Document) error
|
||||
@@ -43,25 +45,15 @@ type CrawledDocument interface {
|
||||
ID() string
|
||||
GetDocument() *doc.Document
|
||||
// Get all the Documents directly referred in a Document.
|
||||
GetResources() ([]*doc.Document, error)
|
||||
// For a Document representing a non-kustomization file, an empty slice will be returned.
|
||||
// For a Document representing a kustomization file:
|
||||
// the `includeResources` parameter determines whether the documents referred in the `resources` field are returned or not;
|
||||
// the `includeTransformers` parameter determines whether the documents referred in the `transformers` field are returned or not;
|
||||
// the `includeGenerators` parameter determines whether the documents referred in the `generators` field are returned or not.
|
||||
GetResources(includeResources, includeTransformers, includeGenerators bool) ([]*doc.Document, error)
|
||||
WasCached() bool
|
||||
}
|
||||
|
||||
type SeenMap map[string]struct{}
|
||||
|
||||
func (seen SeenMap) Seen(item string) bool {
|
||||
_, ok := seen[item]
|
||||
return ok
|
||||
}
|
||||
|
||||
func (seen SeenMap) Add(item string) {
|
||||
seen[item] = struct{}{}
|
||||
}
|
||||
|
||||
func NewSeenMap() SeenMap {
|
||||
return make(map[string]struct{})
|
||||
}
|
||||
|
||||
type CrawlSeed []*doc.Document
|
||||
|
||||
type IndexFunc func(CrawledDocument, index.Mode) error
|
||||
@@ -84,18 +76,18 @@ func findMatch(d *doc.Document, crawlers []Crawler) Crawler {
|
||||
}
|
||||
|
||||
func addBranches(cdoc CrawledDocument, match Crawler, indx IndexFunc,
|
||||
seen SeenMap, stack *CrawlSeed) {
|
||||
seen utils.SeenMap, stack *CrawlSeed) {
|
||||
|
||||
seen.Add(cdoc.ID())
|
||||
|
||||
// Insert into index
|
||||
if err := indx(cdoc, index.InsertOrUpdate); err != nil {
|
||||
logger.Printf("Failed to insert or update %s %s: %v",
|
||||
cdoc.GetDocument().RepositoryURL, cdoc.GetDocument().FilePath, err)
|
||||
logger.Printf("Failed to insert or update doc(%s): %v",
|
||||
cdoc.GetDocument().Path(), err)
|
||||
return
|
||||
}
|
||||
|
||||
deps, err := cdoc.GetResources()
|
||||
deps, err := cdoc.GetResources(true, false, false)
|
||||
if err != nil {
|
||||
logger.Println(err)
|
||||
return
|
||||
@@ -110,7 +102,7 @@ func addBranches(cdoc CrawledDocument, match Crawler, indx IndexFunc,
|
||||
}
|
||||
|
||||
func doCrawl(ctx context.Context, docsPtr *CrawlSeed, crawlers []Crawler, conv Converter, indx IndexFunc,
|
||||
seen SeenMap, stack *CrawlSeed) {
|
||||
seen utils.SeenMap, stack *CrawlSeed) {
|
||||
|
||||
UpdatedDocCount := 0
|
||||
seenDocCount := 0
|
||||
@@ -131,7 +123,7 @@ func doCrawl(ctx context.Context, docsPtr *CrawlSeed, crawlers []Crawler, conv C
|
||||
*docsPtr = (*docsPtr)[:(len(*docsPtr) - 1)]
|
||||
|
||||
crawledDocCount++
|
||||
logger.Printf("Crawling doc %d: %s %s", crawledDocCount, tail.RepositoryURL, tail.FilePath)
|
||||
logger.Printf("Crawling doc %d: %s", crawledDocCount, tail.Path())
|
||||
|
||||
if seen.Seen(tail.ID()) {
|
||||
logger.Printf("this doc has been seen before")
|
||||
@@ -140,7 +132,7 @@ func doCrawl(ctx context.Context, docsPtr *CrawlSeed, crawlers []Crawler, conv C
|
||||
}
|
||||
|
||||
if tail.WasCached() {
|
||||
logger.Printf("%s %s is cached already", tail.RepositoryURL, tail.FilePath)
|
||||
logger.Printf("doc(%s) is cached already", tail.Path())
|
||||
cachedDocCount++
|
||||
continue
|
||||
}
|
||||
@@ -161,10 +153,8 @@ func doCrawl(ctx context.Context, docsPtr *CrawlSeed, crawlers []Crawler, conv C
|
||||
// `bases` field.
|
||||
seen.Add(tail.ID())
|
||||
|
||||
|
||||
if err := match.FetchDocument(ctx, tail); err != nil {
|
||||
logger.Printf("FetchDocument failed on %s %s: %v",
|
||||
tail.RepositoryURL, tail.FilePath, err)
|
||||
logger.Printf("FetchDocument failed on doc(%s): %v", tail.Path(), err)
|
||||
FetchDocumentErrCount++
|
||||
// delete the document from the index
|
||||
cdoc := &doc.KustomizationDocument{
|
||||
@@ -172,16 +162,14 @@ func doCrawl(ctx context.Context, docsPtr *CrawlSeed, crawlers []Crawler, conv C
|
||||
}
|
||||
seen.Add(cdoc.ID())
|
||||
if err := indx(cdoc, index.Delete); err != nil {
|
||||
logger.Printf("Failed to delete %s %s: %v",
|
||||
cdoc.RepositoryURL, cdoc.FilePath, err)
|
||||
logger.Printf("Failed to delete doc(%s): %v", cdoc.Path(), err)
|
||||
}
|
||||
deleteDocCount++
|
||||
continue
|
||||
}
|
||||
|
||||
if err := match.SetCreated(ctx, tail); err != nil {
|
||||
logger.Printf("SetCreated failed on %s %s: %v",
|
||||
tail.RepositoryURL, tail.FilePath, err)
|
||||
logger.Printf("SetCreated failed on doc(%s): %v", tail.Path(), err)
|
||||
SetCreatedErrCount++
|
||||
}
|
||||
|
||||
@@ -189,8 +177,7 @@ func doCrawl(ctx context.Context, docsPtr *CrawlSeed, crawlers []Crawler, conv C
|
||||
// If conv returns an error, cdoc can still be added into the index so that
|
||||
// cdoc.Document can be searched.
|
||||
if err != nil {
|
||||
logger.Printf("conv failed on %s %s: %v",
|
||||
tail.RepositoryURL, tail.FilePath, err)
|
||||
logger.Printf("conv failed on doc(%s): %v", tail.Path(), err)
|
||||
convErrCount++
|
||||
}
|
||||
|
||||
@@ -211,7 +198,7 @@ func doCrawl(ctx context.Context, docsPtr *CrawlSeed, crawlers []Crawler, conv C
|
||||
// CrawlFromSeed updates all the documents in seed, and crawls all the new
|
||||
// documents referred in the seed.
|
||||
func CrawlFromSeed(ctx context.Context, seed CrawlSeed, crawlers []Crawler,
|
||||
conv Converter, indx IndexFunc, seen SeenMap) {
|
||||
conv Converter, indx IndexFunc, seen utils.SeenMap) {
|
||||
|
||||
// stack tracks the documents directly referred in other documents.
|
||||
stack := make(CrawlSeed, 0)
|
||||
@@ -247,7 +234,7 @@ func CrawlFromSeed(ctx context.Context, seed CrawlSeed, crawlers []Crawler,
|
||||
// from the seed will be processed before any other documents from the
|
||||
// crawlers.
|
||||
func CrawlGithubRunner(ctx context.Context, output chan<- CrawledDocument,
|
||||
crawlers []Crawler, seen SeenMap) []error {
|
||||
crawlers []Crawler, seen utils.SeenMap) []error {
|
||||
|
||||
errs := make([]error, len(crawlers))
|
||||
wg := sync.WaitGroup{}
|
||||
@@ -291,7 +278,7 @@ func CrawlGithubRunner(ctx context.Context, output chan<- CrawledDocument,
|
||||
|
||||
// CrawlGithub crawls all the kustomization files on Github.
|
||||
func CrawlGithub(ctx context.Context, crawlers []Crawler, conv Converter,
|
||||
indx IndexFunc, seen SeenMap) {
|
||||
indx IndexFunc, seen utils.SeenMap) {
|
||||
// stack tracks the documents directly referred in other documents.
|
||||
stack := make(CrawlSeed, 0)
|
||||
|
||||
|
||||
@@ -12,6 +12,8 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"sigs.k8s.io/kustomize/api/internal/crawl/utils"
|
||||
|
||||
"sigs.k8s.io/kustomize/api/internal/crawl/index"
|
||||
|
||||
"sigs.k8s.io/kustomize/api/internal/crawl/doc"
|
||||
@@ -76,7 +78,7 @@ func newCrawler(matchPrefix string, err error,
|
||||
|
||||
// Crawl implements the Crawler interface for testing.
|
||||
func (c testCrawler) Crawl(_ context.Context,
|
||||
output chan<- CrawledDocument, _ SeenMap) error {
|
||||
output chan<- CrawledDocument, _ utils.SeenMap) error {
|
||||
|
||||
for i, d := range c.docs {
|
||||
isResource := true
|
||||
@@ -182,7 +184,7 @@ func TestCrawlGithubRunner(t *testing.T) {
|
||||
defer close(output)
|
||||
defer wg.Done()
|
||||
|
||||
seen := NewSeenMap()
|
||||
seen := utils.NewSeenMap()
|
||||
errs := CrawlGithubRunner(context.Background(),
|
||||
output, test.tc, seen)
|
||||
|
||||
@@ -324,7 +326,7 @@ resources:
|
||||
visited[d.ID()]++
|
||||
return nil
|
||||
},
|
||||
NewSeenMap(),
|
||||
utils.NewSeenMap(),
|
||||
)
|
||||
if lv, lc := len(visited), len(tc.corpus); lv != lc {
|
||||
t.Errorf("error: %d of %d documents visited.", lv, lc)
|
||||
|
||||
@@ -16,6 +16,8 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"sigs.k8s.io/kustomize/api/internal/crawl/utils"
|
||||
|
||||
"sigs.k8s.io/kustomize/api/internal/crawl/crawler"
|
||||
"sigs.k8s.io/kustomize/api/internal/crawl/doc"
|
||||
"sigs.k8s.io/kustomize/api/internal/crawl/httpclient"
|
||||
@@ -68,7 +70,7 @@ func (gc githubCrawler) DefaultBranch(repo string) string {
|
||||
|
||||
// Implements crawler.Crawler.
|
||||
func (gc githubCrawler) Crawl(ctx context.Context,
|
||||
output chan<- crawler.CrawledDocument, seen crawler.SeenMap) error {
|
||||
output chan<- crawler.CrawledDocument, seen utils.SeenMap) error {
|
||||
|
||||
noETagClient := GhClient{
|
||||
RequestConfig: gc.client.RequestConfig,
|
||||
@@ -195,9 +197,9 @@ func (gc githubCrawler) Match(d *doc.Document) bool {
|
||||
|
||||
type RangeQueryResult struct {
|
||||
totalDocCnt uint64
|
||||
seenDocCnt uint64
|
||||
newDocCnt uint64
|
||||
errorCnt uint64
|
||||
seenDocCnt uint64
|
||||
newDocCnt uint64
|
||||
errorCnt uint64
|
||||
}
|
||||
|
||||
func (r *RangeQueryResult) Add(other RangeQueryResult) {
|
||||
@@ -209,7 +211,7 @@ func (r *RangeQueryResult) Add(other RangeQueryResult) {
|
||||
|
||||
func (r *RangeQueryResult) String() string {
|
||||
return fmt.Sprintf("got %d files from API. "+
|
||||
"%d have been seen before. %d are new and sent to the output channel." +
|
||||
"%d have been seen before. %d are new and sent to the output channel."+
|
||||
" %d have kustomizationResultAdapter errors.",
|
||||
r.totalDocCnt, r.seenDocCnt, r.newDocCnt, r.errorCnt)
|
||||
}
|
||||
@@ -217,7 +219,7 @@ func (r *RangeQueryResult) String() string {
|
||||
// processQuery follows all of the pages in a query, and updates/adds the
|
||||
// documents from the crawl to the datastore/index.
|
||||
func processQuery(ctx context.Context, gcl GhClient, query string,
|
||||
output chan<- crawler.CrawledDocument, seen crawler.SeenMap,
|
||||
output chan<- crawler.CrawledDocument, seen utils.SeenMap,
|
||||
branchMap map[string]string) (RangeQueryResult, error) {
|
||||
|
||||
queryPages := make(chan GhResponseInfo)
|
||||
@@ -271,7 +273,7 @@ func processQuery(ctx context.Context, gcl GhClient, query string,
|
||||
return result, errs
|
||||
}
|
||||
|
||||
func kustomizationResultAdapter(gcl GhClient, k GhFileSpec, seen crawler.SeenMap,
|
||||
func kustomizationResultAdapter(gcl GhClient, k GhFileSpec, seen utils.SeenMap,
|
||||
branchMap map[string]string) (crawler.CrawledDocument, error) {
|
||||
url := gcl.ReposRequest(k.Repository.FullName)
|
||||
defaultBranch, err := gcl.GetDefaultBranch(url, k.Repository.URL, branchMap)
|
||||
|
||||
@@ -117,7 +117,7 @@ type RequestConfig struct {
|
||||
// understand why the request object is useful.
|
||||
func (rc RequestConfig) CodeSearchRequestWith(query Query) request {
|
||||
vals := url.Values{
|
||||
"sort": []string{"indexed"},
|
||||
"sort": []string{"indexed"},
|
||||
"order": []string{"desc"},
|
||||
}
|
||||
req := rc.makeRequest("search/code", query, vals)
|
||||
|
||||
@@ -3,6 +3,7 @@ package doc
|
||||
import (
|
||||
"fmt"
|
||||
"log"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
@@ -51,15 +52,21 @@ func (doc *KustomizationDocument) String() string {
|
||||
doc.IsSame, doc.Kinds, len(doc.Identifiers), len(doc.Values))
|
||||
}
|
||||
|
||||
// Implements the CrawlerDocument interface.
|
||||
func (doc *KustomizationDocument) GetResources() ([]*Document, error) {
|
||||
isResource := true
|
||||
for _, suffix := range konfig.RecognizedKustomizationFileNames() {
|
||||
if strings.HasSuffix(doc.FilePath, "/"+suffix) {
|
||||
isResource = false
|
||||
// IsKustomizationFile determines whether a file path is a kustomization file
|
||||
func IsKustomizationFile(path string) bool {
|
||||
basename := filepath.Base(path)
|
||||
for _, name := range konfig.RecognizedKustomizationFileNames() {
|
||||
if basename == name {
|
||||
return true
|
||||
}
|
||||
}
|
||||
if isResource {
|
||||
return false
|
||||
}
|
||||
|
||||
// Implements the CrawlerDocument interface.
|
||||
func (doc *KustomizationDocument) GetResources(
|
||||
includeResources, includeTransformers, includeGenerators bool) ([]*Document, error) {
|
||||
if !IsKustomizationFile(doc.FilePath) {
|
||||
return []*Document{}, nil
|
||||
}
|
||||
|
||||
@@ -77,20 +84,42 @@ func (doc *KustomizationDocument) GetResources() ([]*Document, error) {
|
||||
}
|
||||
k.FixKustomizationPostUnmarshalling()
|
||||
|
||||
res := make([]*Document, 0, len(k.Resources))
|
||||
for _, r := range k.Resources {
|
||||
res := make([]*Document, 0)
|
||||
|
||||
if includeResources {
|
||||
resourceDocs := doc.CollectDocuments(k.Resources)
|
||||
res = append(res, resourceDocs...)
|
||||
}
|
||||
|
||||
if includeGenerators {
|
||||
generatorDocs := doc.CollectDocuments(k.Generators)
|
||||
res = append(res, generatorDocs...)
|
||||
}
|
||||
|
||||
if includeTransformers {
|
||||
transformerDocs := doc.CollectDocuments(k.Transformers)
|
||||
res = append(res, transformerDocs...)
|
||||
}
|
||||
|
||||
return res, nil
|
||||
}
|
||||
|
||||
// CollectDocuments construct a Document for each path in paths, and return
|
||||
// a slice of Document pointers.
|
||||
func (doc *KustomizationDocument) CollectDocuments(paths []string) []*Document {
|
||||
docs := make([]*Document, 0, len(paths))
|
||||
for _, r := range paths {
|
||||
if strings.TrimSpace(r) == "" {
|
||||
continue
|
||||
}
|
||||
next, err := doc.Document.FromRelativePath(r)
|
||||
if err != nil {
|
||||
log.Printf("GetResources error: %v\n", err)
|
||||
log.Printf("CollectDocuments error: %v\n", err)
|
||||
continue
|
||||
}
|
||||
res = append(res, &next)
|
||||
docs = append(docs, &next)
|
||||
}
|
||||
|
||||
return res, nil
|
||||
return docs
|
||||
}
|
||||
|
||||
func (doc *KustomizationDocument) readBytes() ([]map[string]interface{}, error) {
|
||||
|
||||
@@ -189,11 +189,13 @@ metadata:
|
||||
}
|
||||
}
|
||||
|
||||
type TestStructForGetResources struct {
|
||||
doc KustomizationDocument
|
||||
resources []*Document
|
||||
}
|
||||
|
||||
func TestGetResources(t *testing.T) {
|
||||
tests := []struct {
|
||||
doc KustomizationDocument
|
||||
resources []*Document
|
||||
}{
|
||||
tests := []TestStructForGetResources{
|
||||
{
|
||||
doc: KustomizationDocument{
|
||||
Document: Document{
|
||||
@@ -248,9 +250,12 @@ resources:
|
||||
resources: []*Document{},
|
||||
},
|
||||
}
|
||||
runTest(t, tests, true, false, false)
|
||||
}
|
||||
|
||||
func runTest(t *testing.T, tests []TestStructForGetResources, includeResources, includeTransformers, includeGenerators bool) {
|
||||
for _, test := range tests {
|
||||
res, err := test.doc.GetResources()
|
||||
res, err := test.doc.GetResources(includeResources, includeTransformers, includeGenerators)
|
||||
if err != nil {
|
||||
t.Errorf("Unexpected error: %v\n", err)
|
||||
continue
|
||||
@@ -284,3 +289,73 @@ resources:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetResourcesAndGenerators(t *testing.T) {
|
||||
tests := []TestStructForGetResources{
|
||||
{
|
||||
doc: KustomizationDocument{
|
||||
Document: Document{
|
||||
RepositoryURL: "sigs.k8s.io/kustomize",
|
||||
FilePath: "some/path/to/kdir/kustomization.yaml",
|
||||
DocumentData: `
|
||||
resources:
|
||||
- file.yaml
|
||||
|
||||
generators:
|
||||
- gen.yaml
|
||||
|
||||
transformers:
|
||||
- tr.yaml
|
||||
`},
|
||||
},
|
||||
resources: []*Document{
|
||||
{
|
||||
RepositoryURL: "sigs.k8s.io/kustomize",
|
||||
FilePath: "some/path/to/kdir/gen.yaml",
|
||||
},
|
||||
{
|
||||
RepositoryURL: "sigs.k8s.io/kustomize",
|
||||
FilePath: "some/path/to/kdir/file.yaml",
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
runTest(t, tests, true, false, true)
|
||||
}
|
||||
|
||||
func TestGetResourcesAndGeneratorsAndTransformers(t *testing.T) {
|
||||
tests := []TestStructForGetResources{
|
||||
{
|
||||
doc: KustomizationDocument{
|
||||
Document: Document{
|
||||
RepositoryURL: "sigs.k8s.io/kustomize",
|
||||
FilePath: "some/path/to/kdir/kustomization.yaml",
|
||||
DocumentData: `
|
||||
resources:
|
||||
- file.yaml
|
||||
|
||||
generators:
|
||||
- gen.yaml
|
||||
|
||||
transformers:
|
||||
- tr.yaml
|
||||
`},
|
||||
},
|
||||
resources: []*Document{
|
||||
{
|
||||
RepositoryURL: "sigs.k8s.io/kustomize",
|
||||
FilePath: "some/path/to/kdir/tr.yaml",
|
||||
},
|
||||
{
|
||||
RepositoryURL: "sigs.k8s.io/kustomize",
|
||||
FilePath: "some/path/to/kdir/gen.yaml",
|
||||
},
|
||||
{
|
||||
RepositoryURL: "sigs.k8s.io/kustomize",
|
||||
FilePath: "some/path/to/kdir/file.yaml",
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
runTest(t, tests, true, true, true)
|
||||
}
|
||||
|
||||
@@ -35,6 +35,11 @@ func (doc *Document) Copy() *Document {
|
||||
}
|
||||
}
|
||||
|
||||
func (doc *Document) Path() string {
|
||||
return fmt.Sprintf("repoURL: %s filePath: %s branch: %s",
|
||||
doc.RepositoryURL, doc.FilePath, doc.DefaultBranch)
|
||||
}
|
||||
|
||||
// Implements the CrawlerDocument interface.
|
||||
func (doc *Document) WasCached() bool {
|
||||
return doc.IsSame
|
||||
|
||||
@@ -65,7 +65,7 @@ func TestFromRelativePath(t *testing.T) {
|
||||
|
||||
func TestDocument_RepositoryFullName(t *testing.T) {
|
||||
testCases := []struct {
|
||||
doc Document
|
||||
doc Document
|
||||
expectedRepositoryFullName string
|
||||
}{
|
||||
{
|
||||
@@ -108,4 +108,4 @@ func TestDocument_RepositoryFullName(t *testing.T) {
|
||||
returnedRepositoryFullName)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
36
api/internal/crawl/doc/unique_doc.go
Normal file
36
api/internal/crawl/doc/unique_doc.go
Normal file
@@ -0,0 +1,36 @@
|
||||
package doc
|
||||
|
||||
import (
|
||||
"sigs.k8s.io/kustomize/api/internal/crawl/utils"
|
||||
)
|
||||
|
||||
// UniqueDocuments make sure a Document with a given ID appears only once
|
||||
type UniqueDocuments struct {
|
||||
docs []*Document
|
||||
docIDs utils.SeenMap
|
||||
}
|
||||
|
||||
func NewUniqueDocuments() UniqueDocuments {
|
||||
return UniqueDocuments{
|
||||
docs: []*Document{},
|
||||
docIDs: utils.NewSeenMap(),
|
||||
}
|
||||
}
|
||||
|
||||
func (uds *UniqueDocuments) Add(d *Document) {
|
||||
if uds.docIDs.Seen(d.ID()) {
|
||||
return
|
||||
}
|
||||
uds.docs = append(uds.docs, d)
|
||||
uds.docIDs.Add(d.ID())
|
||||
}
|
||||
|
||||
func (uds *UniqueDocuments) AddDocuments(docs []*Document) {
|
||||
for _, d := range docs {
|
||||
uds.Add(d)
|
||||
}
|
||||
}
|
||||
|
||||
func (uds *UniqueDocuments) Documents() []*Document {
|
||||
return uds.docs
|
||||
}
|
||||
@@ -18,6 +18,7 @@ const (
|
||||
)
|
||||
|
||||
type Mode int
|
||||
|
||||
const (
|
||||
InsertOrUpdate = iota
|
||||
Delete
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
Find out the largest value of the `creationTime` field:
|
||||
```
|
||||
curl -X POST "${ElasticSearchURL}:9200/kustomize/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
|
||||
curl -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
|
||||
{
|
||||
"aggs" : {
|
||||
"max_creationTime" : { "max" : { "field" : "creationTime" } }
|
||||
@@ -11,7 +11,7 @@ curl -X POST "${ElasticSearchURL}:9200/kustomize/_search?size=0&pretty" -H 'Cont
|
||||
|
||||
Find out the smallest value of the `creationTime` field:
|
||||
```
|
||||
curl -X POST "${ElasticSearchURL}:9200/kustomize/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
|
||||
curl -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
|
||||
{
|
||||
"aggs" : {
|
||||
"min_creationTime" : { "min" : { "field" : "creationTime" } }
|
||||
@@ -22,7 +22,7 @@ curl -X POST "${ElasticSearchURL}:9200/kustomize/_search?size=0&pretty" -H 'Cont
|
||||
|
||||
Find out the smallest value of the `creationTime` field of all the kustomization files:
|
||||
```
|
||||
curl -X POST "${ElasticSearchURL}:9200/kustomize/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
|
||||
curl -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
|
||||
{
|
||||
"query": {
|
||||
"bool": {
|
||||
@@ -40,7 +40,7 @@ curl -X POST "${ElasticSearchURL}:9200/kustomize/_search?size=0&pretty" -H 'Cont
|
||||
|
||||
Find out the smallest value of the `creationTime` field of all kustomize resource files:
|
||||
```
|
||||
curl -X POST "${ElasticSearchURL}:9200/kustomize/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
|
||||
curl -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
|
||||
{
|
||||
"query": {
|
||||
"bool": {
|
||||
@@ -58,7 +58,7 @@ curl -X POST "${ElasticSearchURL}:9200/kustomize/_search?size=0&pretty" -H 'Cont
|
||||
|
||||
Query all the documents whose `creationTime` <= `2016-07-29T17:38:26.000Z`:
|
||||
```
|
||||
curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type: application/json' -d'
|
||||
curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
|
||||
{
|
||||
"query": {
|
||||
"range": {
|
||||
@@ -73,7 +73,7 @@ curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type
|
||||
|
||||
Query all the documents whose `creationTime` falls within the specific range:
|
||||
```
|
||||
curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type: application/json' -d'
|
||||
curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
|
||||
{
|
||||
"query": {
|
||||
"range": {
|
||||
@@ -89,7 +89,7 @@ curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type
|
||||
|
||||
Aggregate how many new kustomization files were added into Github each month:
|
||||
```
|
||||
curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
|
||||
curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
|
||||
{
|
||||
"query": {
|
||||
"bool": {
|
||||
@@ -112,7 +112,7 @@ curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?size=0&pretty" -H 'Conte
|
||||
|
||||
Aggregate how many new kustomize resource files were added into Github each month:
|
||||
```
|
||||
curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
|
||||
curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
|
||||
{
|
||||
"query": {
|
||||
"bool": {
|
||||
@@ -135,7 +135,7 @@ curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?size=0&pretty" -H 'Conte
|
||||
|
||||
Aggregate how many new kustomization files were added into Github each year:
|
||||
```
|
||||
curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
|
||||
curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
|
||||
{
|
||||
"query": {
|
||||
"bool": {
|
||||
@@ -158,7 +158,7 @@ curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?size=0&pretty" -H 'Conte
|
||||
|
||||
Aggregate how many new kustomize resource files were added into Github each year:
|
||||
```
|
||||
curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
|
||||
curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
|
||||
{
|
||||
"query": {
|
||||
"bool": {
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
Count distinct values of the `defaultBranch` field:
|
||||
```
|
||||
curl -X POST "${ElasticSearchURL}:9200/kustomize/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
|
||||
curl -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
|
||||
{
|
||||
"aggs" : {
|
||||
"defaultBranch_count" : {
|
||||
@@ -17,7 +17,7 @@ curl -X POST "${ElasticSearchURL}:9200/kustomize/_search?size=0&pretty" -H 'Cont
|
||||
List all the github branches where kustomization files and kustomize resource files live,
|
||||
and how many kustomization files and kustomize resource files live in each branch:
|
||||
```
|
||||
curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type: application/json' -d'
|
||||
curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
|
||||
{
|
||||
"aggs" : {
|
||||
"defaultBranch" : {
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
Count the documents whose `document` field is empty (The reason why the `document` field
|
||||
of a document is empty is because of empty documents):
|
||||
```
|
||||
curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type: application/json' -d'
|
||||
curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
|
||||
{
|
||||
"size": 10000,
|
||||
"query": {
|
||||
@@ -19,7 +19,7 @@ curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type
|
||||
|
||||
Find all the documents having the `creationTime` field set:
|
||||
```
|
||||
curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type: application/json' -d'
|
||||
curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
|
||||
{
|
||||
"query": {
|
||||
"exists": {
|
||||
@@ -32,7 +32,7 @@ curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type
|
||||
|
||||
Find all the documents whose `creationTime` field is not set:
|
||||
```
|
||||
curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type: application/json' -d'
|
||||
curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
|
||||
{
|
||||
"size": 10000,
|
||||
"query": {
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
Count the documents in the index whose `repositoryUrl` field starts with
|
||||
`https://github.com/`:
|
||||
```
|
||||
curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type: application/json' -d'
|
||||
curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
|
||||
{
|
||||
"query": {
|
||||
"bool": {
|
||||
@@ -17,7 +17,7 @@ curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type
|
||||
Count the documents in the index whose `repositoryUrl` field does not start with
|
||||
`https://github.com/`:
|
||||
```
|
||||
curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type: application/json' -d'
|
||||
curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
|
||||
{
|
||||
"query": {
|
||||
"bool": {
|
||||
@@ -33,7 +33,7 @@ curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type
|
||||
Search all the documents matching the given `repositoryUrl` and `filePath`, and return
|
||||
a version for each search hit:
|
||||
```
|
||||
curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type: application/json' -d'
|
||||
curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
|
||||
{
|
||||
"size": 10000,
|
||||
"version": true,
|
||||
@@ -52,7 +52,7 @@ curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type
|
||||
Search all the documents whose filePath ends with one of these following three filenames:
|
||||
`kustomization.yaml`, `kustomization.yml`, `kustomization`:
|
||||
```
|
||||
curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type: application/json' -d'
|
||||
curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
|
||||
{
|
||||
"query": {
|
||||
"bool": {
|
||||
@@ -68,7 +68,7 @@ curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type
|
||||
Search all the documents whose filePath does not end with any of these following
|
||||
three filenames: `kustomization.yaml`, `kustomization.yml`, `kustomization`:
|
||||
```
|
||||
curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type: application/json' -d'
|
||||
curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
|
||||
{
|
||||
"query": {
|
||||
"bool": {
|
||||
|
||||
@@ -10,10 +10,10 @@ curl "${ElasticSearchURL}:9200/_cat/indices?v"
|
||||
|
||||
Get the mapping of the index:
|
||||
```
|
||||
curl -X GET "${ElasticSearchURL}:9200/kustomize/_mapping?pretty"
|
||||
curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_mapping?pretty"
|
||||
```
|
||||
|
||||
Delete the kustomize index from the ElasticSearch cluster (**Use this command with caution**):
|
||||
```
|
||||
curl -X DELETE "${ElasticSearchURL}:9200/kustomize?pretty"
|
||||
curl -X DELETE "${ElasticSearchURL}:9200/${INDEXNAME}?pretty"
|
||||
```
|
||||
@@ -1,6 +1,6 @@
|
||||
Count distinct values of the `repositoryUrl` field:
|
||||
```
|
||||
curl -X POST "${ElasticSearchURL}:9200/kustomize/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
|
||||
curl -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
|
||||
{
|
||||
"aggs" : {
|
||||
"repositoryUrl_count" : {
|
||||
@@ -16,7 +16,7 @@ curl -X POST "${ElasticSearchURL}:9200/kustomize/_search?size=0&pretty" -H 'Cont
|
||||
|
||||
Count how many Github repositories include kustomization files:
|
||||
```
|
||||
curl -X POST "${ElasticSearchURL}:9200/kustomize/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
|
||||
curl -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
|
||||
{
|
||||
"query": {
|
||||
"bool": {
|
||||
@@ -39,7 +39,7 @@ curl -X POST "${ElasticSearchURL}:9200/kustomize/_search?size=0&pretty" -H 'Cont
|
||||
|
||||
Count how many Github repositories include kustomize resource files:
|
||||
```
|
||||
curl -X POST "${ElasticSearchURL}:9200/kustomize/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
|
||||
curl -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
|
||||
{
|
||||
"query": {
|
||||
"bool": {
|
||||
@@ -64,7 +64,7 @@ List all the github repositories including kustomization files and kustomize res
|
||||
and how many kustomization files and kustomize resource files each github repository includes
|
||||
(the github repository including the most kustomization files is listed first):
|
||||
```
|
||||
curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
|
||||
curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
|
||||
{
|
||||
"aggs" : {
|
||||
"repositoryUrl" : {
|
||||
@@ -80,7 +80,7 @@ curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?size=0&pretty" -H 'Conte
|
||||
|
||||
List the top 20 Github repositories including the most amount of kustomization files:
|
||||
```
|
||||
curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
|
||||
curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
|
||||
{
|
||||
"query": {
|
||||
"bool": {
|
||||
@@ -103,7 +103,7 @@ curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?size=0&pretty" -H 'Conte
|
||||
|
||||
List the top 20 Github repositories including the most amount of kustomize resource files:
|
||||
```
|
||||
curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
|
||||
curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
|
||||
{
|
||||
"query": {
|
||||
"bool": {
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
Search for all the kustomize resource files including a Deployment object:
|
||||
```
|
||||
curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type: application/json' -d'
|
||||
curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
|
||||
{
|
||||
"query": {
|
||||
"match" : {
|
||||
@@ -16,7 +16,7 @@ curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type
|
||||
Search for all the kustomize resource files including a Deployment object, but only
|
||||
including the `kinds` field in the result:
|
||||
```
|
||||
curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type: application/json' -d'
|
||||
curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
|
||||
{
|
||||
"_source": {
|
||||
"includes": ["kinds"]
|
||||
@@ -35,7 +35,7 @@ curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type
|
||||
Search for all the kustomize resource files including both a Deployment object and
|
||||
a Service object:
|
||||
```
|
||||
curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type: application/json' -d'
|
||||
curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
|
||||
{
|
||||
"query": {
|
||||
"match" : {
|
||||
@@ -52,7 +52,7 @@ curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type
|
||||
Count the number of documents including Deployment and the number of documents
|
||||
including Service:
|
||||
```
|
||||
curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type: application/json' -d'
|
||||
curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
|
||||
{
|
||||
"size": 0,
|
||||
"aggs" : {
|
||||
@@ -71,7 +71,7 @@ curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type
|
||||
|
||||
Search for all the kustomization files involving CRDs:
|
||||
```
|
||||
curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type: application/json' -d'
|
||||
curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
|
||||
{
|
||||
"size": 10000,
|
||||
"query": {
|
||||
@@ -87,7 +87,7 @@ curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type
|
||||
|
||||
Search for all the kustomization files defining configMapGenerator:
|
||||
```
|
||||
curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type: application/json' -d'
|
||||
curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
|
||||
{
|
||||
"size": 10000,
|
||||
"query": {
|
||||
@@ -103,7 +103,7 @@ curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type
|
||||
|
||||
Search for all the documents having a `kind` field:
|
||||
```
|
||||
curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type: application/json' -d'
|
||||
curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
|
||||
{
|
||||
"query": {
|
||||
"bool": {
|
||||
@@ -118,7 +118,7 @@ curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type
|
||||
|
||||
Search for all the kuostmization files having a `kind` field:
|
||||
```
|
||||
curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type: application/json' -d'
|
||||
curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
|
||||
{
|
||||
"query": {
|
||||
"bool": {
|
||||
@@ -134,7 +134,7 @@ curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type
|
||||
|
||||
Search for all the kustomization files defining the `generatorOptions:disableNameSuffixHash` feature:
|
||||
```
|
||||
curl -X GET "${ElasticSearchURL}:9200/kustomize/_search?pretty" -H 'Content-Type: application/json' -d'
|
||||
curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d'
|
||||
{
|
||||
"query": {
|
||||
"match" : {
|
||||
|
||||
16
api/internal/crawl/utils/utils.go
Normal file
16
api/internal/crawl/utils/utils.go
Normal file
@@ -0,0 +1,16 @@
|
||||
package utils
|
||||
|
||||
type SeenMap map[string]struct{}
|
||||
|
||||
func (seen SeenMap) Seen(item string) bool {
|
||||
_, ok := seen[item]
|
||||
return ok
|
||||
}
|
||||
|
||||
func (seen SeenMap) Add(item string) {
|
||||
seen[item] = struct{}{}
|
||||
}
|
||||
|
||||
func NewSeenMap() SeenMap {
|
||||
return make(map[string]struct{})
|
||||
}
|
||||
@@ -45,6 +45,7 @@ Advanced Documentation Topics:
|
||||
|
||||
// Export commands publicly for composition
|
||||
var (
|
||||
Annotate = commands.AnnotateCommand
|
||||
Cat = commands.CatCommand
|
||||
Count = commands.CountCommand
|
||||
CreateSetter = commands.CreateSetterCommand
|
||||
@@ -58,6 +59,8 @@ var (
|
||||
Sink = commands.SinkCommand
|
||||
Source = commands.SourceCommand
|
||||
Tree = commands.TreeCommand
|
||||
Wrap = commands.WrapCommand
|
||||
XArgs = commands.XArgsCommand
|
||||
|
||||
StackOnError = &commands.StackOnError
|
||||
ExitOnError = &commands.ExitOnError
|
||||
@@ -89,6 +92,7 @@ func NewConfigCommand(name string) *cobra.Command {
|
||||
|
||||
name = strings.TrimSpace(name + " config")
|
||||
commands.ExitOnError = true
|
||||
root.AddCommand(commands.AnnotateCommand(name))
|
||||
root.AddCommand(commands.GrepCommand(name))
|
||||
root.AddCommand(commands.TreeCommand(name))
|
||||
root.AddCommand(commands.CatCommand(name))
|
||||
|
||||
18
cmd/config/docs/commands/annotate.md
Normal file
18
cmd/config/docs/commands/annotate.md
Normal file
@@ -0,0 +1,18 @@
|
||||
## annotate
|
||||
|
||||
[Alpha] Set an annotation on Resources.
|
||||
|
||||
### Synopsis
|
||||
|
||||
[Alpha] Set an annotation on Resources.
|
||||
|
||||
DIR:
|
||||
Path to local directory.
|
||||
|
||||
### Examples
|
||||
|
||||
kustomize config annotate my-dir/ --kv foo=bar
|
||||
|
||||
kustomize config annotate my-dir/ --kv foo=bar --kv a=b
|
||||
|
||||
kustomize config annotate my-dir/ --kv foo=bar --kind Deployment --name foo
|
||||
103
cmd/config/internal/commands/annotate.go
Normal file
103
cmd/config/internal/commands/annotate.go
Normal file
@@ -0,0 +1,103 @@
|
||||
// Copyright 2019 The Kubernetes Authors.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package commands
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
"sigs.k8s.io/kustomize/cmd/config/internal/generateddocs/commands"
|
||||
"sigs.k8s.io/kustomize/kyaml/errors"
|
||||
"sigs.k8s.io/kustomize/kyaml/kio"
|
||||
"sigs.k8s.io/kustomize/kyaml/yaml"
|
||||
)
|
||||
|
||||
// NewAnnotateRunner returns a command runner.
|
||||
func NewAnnotateRunner(parent string) *AnnotateRunner {
|
||||
r := &AnnotateRunner{}
|
||||
c := &cobra.Command{
|
||||
Use: "annotate [DIR]",
|
||||
Args: cobra.MaximumNArgs(1),
|
||||
Short: commands.AnnotateShort,
|
||||
Long: commands.AnnotateLong,
|
||||
Example: commands.AnnotateExamples,
|
||||
RunE: r.runE,
|
||||
}
|
||||
fixDocs(parent, c)
|
||||
r.Command = c
|
||||
c.Flags().StringVar(&r.Kind, "kind", "", "Resource kind to annotate")
|
||||
c.Flags().StringVar(&r.ApiVersion, "apiVersion", "", "Resource apiVersion to annotate")
|
||||
c.Flags().StringVar(&r.Name, "name", "", "Resource name to annotate")
|
||||
c.Flags().StringVar(&r.Namespace, "namespace", "", "Resource namespace to annotate")
|
||||
c.Flags().StringSliceVar(&r.Values, "kv", []string{}, "annotation as KEY=VALUE")
|
||||
return r
|
||||
}
|
||||
|
||||
func AnnotateCommand(parent string) *cobra.Command {
|
||||
return NewAnnotateRunner(parent).Command
|
||||
}
|
||||
|
||||
type AnnotateRunner struct {
|
||||
Command *cobra.Command
|
||||
Values []string
|
||||
Kind string
|
||||
Name string
|
||||
ApiVersion string
|
||||
Namespace string
|
||||
Path string
|
||||
}
|
||||
|
||||
func (r *AnnotateRunner) runE(c *cobra.Command, args []string) error {
|
||||
var input []kio.Reader
|
||||
var output []kio.Writer
|
||||
if len(args) == 0 {
|
||||
rw := &kio.ByteReadWriter{Reader: c.InOrStdin(), Writer: c.OutOrStdout()}
|
||||
input = []kio.Reader{rw}
|
||||
output = []kio.Writer{rw}
|
||||
} else {
|
||||
rw := &kio.LocalPackageReadWriter{PackagePath: args[0], NoDeleteFiles: true}
|
||||
input = []kio.Reader{rw}
|
||||
output = []kio.Writer{rw}
|
||||
}
|
||||
return handleError(c, kio.Pipeline{
|
||||
Inputs: input,
|
||||
Filters: []kio.Filter{r},
|
||||
Outputs: output,
|
||||
}.Execute())
|
||||
}
|
||||
|
||||
func (r *AnnotateRunner) Filter(nodes []*yaml.RNode) ([]*yaml.RNode, error) {
|
||||
for i := range nodes {
|
||||
n := nodes[i]
|
||||
m, err := n.GetMeta()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if r.Kind != "" && r.Kind != m.Kind {
|
||||
continue
|
||||
}
|
||||
if r.ApiVersion != "" && r.ApiVersion != m.APIVersion {
|
||||
continue
|
||||
}
|
||||
if r.Namespace != "" && r.Namespace != m.Namespace {
|
||||
continue
|
||||
}
|
||||
if r.Name != "" && r.Name != m.Name {
|
||||
continue
|
||||
}
|
||||
|
||||
for i := range r.Values {
|
||||
// split key, value pairs
|
||||
kv := strings.SplitN(r.Values[i], "=", 2)
|
||||
if len(kv) != 2 {
|
||||
return nil, errors.Errorf("must specify --kv as KEY=VALUE: %s", r.Values[i])
|
||||
}
|
||||
if err := n.PipeE(yaml.SetAnnotation(kv[0], kv[1])); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
return nodes, nil
|
||||
}
|
||||
488
cmd/config/internal/commands/annotate_test.go
Normal file
488
cmd/config/internal/commands/annotate_test.go
Normal file
@@ -0,0 +1,488 @@
|
||||
// Copyright 2019 The Kubernetes Authors.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package commands
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"sigs.k8s.io/kustomize/kyaml/kio"
|
||||
)
|
||||
|
||||
func TestAnnotateCommand(t *testing.T) {
|
||||
var tests = []struct {
|
||||
name string
|
||||
args []string
|
||||
expected string
|
||||
}{
|
||||
{
|
||||
name: "single value",
|
||||
args: []string{"--kv", "a=b"},
|
||||
expected: expectedSingleValue,
|
||||
},
|
||||
{
|
||||
name: "multi value",
|
||||
args: []string{"--kv", "a=b", "--kv", "c=d"},
|
||||
expected: expectedMultiValue,
|
||||
},
|
||||
{
|
||||
name: "filter kind",
|
||||
args: []string{"--kv", "a=b", "--kind", "Service"},
|
||||
expected: expectedFilterKindService,
|
||||
},
|
||||
{
|
||||
name: "filter apiVersion",
|
||||
args: []string{"--kv", "a=b", "--apiVersion", "v1"},
|
||||
expected: expectedFilterApiVersionV1,
|
||||
},
|
||||
{
|
||||
name: "filter name",
|
||||
args: []string{"--kv", "a=b", "--name", "bar"},
|
||||
expected: expectedFilterNameBar,
|
||||
},
|
||||
{
|
||||
name: "filter namespace",
|
||||
args: []string{"--kv", "a=b", "--namespace", "bar"},
|
||||
expected: expectedFilterNamespaceBar,
|
||||
},
|
||||
}
|
||||
for i := range tests {
|
||||
tt := tests[i]
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
d := initTestDir(t)
|
||||
defer os.RemoveAll(d)
|
||||
|
||||
a := NewAnnotateRunner("")
|
||||
a.Command.SetArgs(append([]string{d}, tt.args...))
|
||||
a.Command.SilenceUsage = true
|
||||
a.Command.SilenceErrors = true
|
||||
|
||||
err := a.Command.Execute()
|
||||
if !assert.NoError(t, err) {
|
||||
t.FailNow()
|
||||
}
|
||||
|
||||
actual := &bytes.Buffer{}
|
||||
err = kio.Pipeline{
|
||||
Inputs: []kio.Reader{kio.LocalPackageReader{PackagePath: d}},
|
||||
Outputs: []kio.Writer{kio.ByteWriter{Writer: actual, KeepReaderAnnotations: true}},
|
||||
}.Execute()
|
||||
if !assert.NoError(t, err) {
|
||||
t.FailNow()
|
||||
}
|
||||
if !assert.Equal(t,
|
||||
strings.TrimSpace(tt.expected),
|
||||
strings.TrimSpace(actual.String())) {
|
||||
t.FailNow()
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func initTestDir(t *testing.T) string {
|
||||
d, err := ioutil.TempDir("", "kustomize-annotate-test")
|
||||
if !assert.NoError(t, err) {
|
||||
t.FailNow()
|
||||
}
|
||||
err = ioutil.WriteFile(filepath.Join(d, "f1.yaml"), []byte(f1Input), 0600)
|
||||
if !assert.NoError(t, err) {
|
||||
defer os.RemoveAll(d)
|
||||
t.FailNow()
|
||||
}
|
||||
err = ioutil.WriteFile(filepath.Join(d, "f2.yaml"), []byte(f2Input), 0600)
|
||||
if !assert.NoError(t, err) {
|
||||
defer os.RemoveAll(d)
|
||||
t.FailNow()
|
||||
}
|
||||
return d
|
||||
}
|
||||
|
||||
var (
|
||||
f1Input = `
|
||||
kind: Deployment
|
||||
metadata:
|
||||
labels:
|
||||
app: nginx2
|
||||
name: foo
|
||||
annotations:
|
||||
app: nginx2
|
||||
spec:
|
||||
replicas: 1
|
||||
---
|
||||
kind: Service
|
||||
metadata:
|
||||
name: foo
|
||||
annotations:
|
||||
app: nginx
|
||||
spec:
|
||||
selector:
|
||||
app: nginx
|
||||
`
|
||||
|
||||
f2Input = `
|
||||
apiVersion: v1
|
||||
kind: Abstraction
|
||||
metadata:
|
||||
name: foo
|
||||
configFn:
|
||||
container:
|
||||
image: gcr.io/example/reconciler:v1
|
||||
annotations:
|
||||
config.kubernetes.io/local-config: "true"
|
||||
namespace: bar
|
||||
spec:
|
||||
replicas: 3
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
labels:
|
||||
app: nginx
|
||||
name: bar
|
||||
annotations:
|
||||
app: nginx
|
||||
namespace: foo
|
||||
spec:
|
||||
replicas: 3
|
||||
`
|
||||
|
||||
expectedSingleValue = `kind: Deployment
|
||||
metadata:
|
||||
labels:
|
||||
app: nginx2
|
||||
name: foo
|
||||
annotations:
|
||||
app: nginx2
|
||||
a: 'b'
|
||||
config.kubernetes.io/index: '0'
|
||||
config.kubernetes.io/path: 'f1.yaml'
|
||||
spec:
|
||||
replicas: 1
|
||||
---
|
||||
kind: Service
|
||||
metadata:
|
||||
name: foo
|
||||
annotations:
|
||||
app: nginx
|
||||
a: 'b'
|
||||
config.kubernetes.io/index: '1'
|
||||
config.kubernetes.io/path: 'f1.yaml'
|
||||
spec:
|
||||
selector:
|
||||
app: nginx
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Abstraction
|
||||
metadata:
|
||||
name: foo
|
||||
configFn:
|
||||
container:
|
||||
image: gcr.io/example/reconciler:v1
|
||||
annotations:
|
||||
config.kubernetes.io/local-config: "true"
|
||||
a: 'b'
|
||||
config.kubernetes.io/index: '0'
|
||||
config.kubernetes.io/path: 'f2.yaml'
|
||||
namespace: bar
|
||||
spec:
|
||||
replicas: 3
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
labels:
|
||||
app: nginx
|
||||
name: bar
|
||||
annotations:
|
||||
app: nginx
|
||||
a: 'b'
|
||||
config.kubernetes.io/index: '1'
|
||||
config.kubernetes.io/path: 'f2.yaml'
|
||||
namespace: foo
|
||||
spec:
|
||||
replicas: 3
|
||||
`
|
||||
|
||||
expectedMultiValue = `kind: Deployment
|
||||
metadata:
|
||||
labels:
|
||||
app: nginx2
|
||||
name: foo
|
||||
annotations:
|
||||
app: nginx2
|
||||
a: 'b'
|
||||
c: 'd'
|
||||
config.kubernetes.io/index: '0'
|
||||
config.kubernetes.io/path: 'f1.yaml'
|
||||
spec:
|
||||
replicas: 1
|
||||
---
|
||||
kind: Service
|
||||
metadata:
|
||||
name: foo
|
||||
annotations:
|
||||
app: nginx
|
||||
a: 'b'
|
||||
c: 'd'
|
||||
config.kubernetes.io/index: '1'
|
||||
config.kubernetes.io/path: 'f1.yaml'
|
||||
spec:
|
||||
selector:
|
||||
app: nginx
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Abstraction
|
||||
metadata:
|
||||
name: foo
|
||||
configFn:
|
||||
container:
|
||||
image: gcr.io/example/reconciler:v1
|
||||
annotations:
|
||||
config.kubernetes.io/local-config: "true"
|
||||
a: 'b'
|
||||
c: 'd'
|
||||
config.kubernetes.io/index: '0'
|
||||
config.kubernetes.io/path: 'f2.yaml'
|
||||
namespace: bar
|
||||
spec:
|
||||
replicas: 3
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
labels:
|
||||
app: nginx
|
||||
name: bar
|
||||
annotations:
|
||||
app: nginx
|
||||
a: 'b'
|
||||
c: 'd'
|
||||
config.kubernetes.io/index: '1'
|
||||
config.kubernetes.io/path: 'f2.yaml'
|
||||
namespace: foo
|
||||
spec:
|
||||
replicas: 3
|
||||
`
|
||||
|
||||
expectedFilterKindService = `kind: Deployment
|
||||
metadata:
|
||||
labels:
|
||||
app: nginx2
|
||||
name: foo
|
||||
annotations:
|
||||
app: nginx2
|
||||
config.kubernetes.io/index: '0'
|
||||
config.kubernetes.io/path: 'f1.yaml'
|
||||
spec:
|
||||
replicas: 1
|
||||
---
|
||||
kind: Service
|
||||
metadata:
|
||||
name: foo
|
||||
annotations:
|
||||
app: nginx
|
||||
a: 'b'
|
||||
config.kubernetes.io/index: '1'
|
||||
config.kubernetes.io/path: 'f1.yaml'
|
||||
spec:
|
||||
selector:
|
||||
app: nginx
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Abstraction
|
||||
metadata:
|
||||
name: foo
|
||||
configFn:
|
||||
container:
|
||||
image: gcr.io/example/reconciler:v1
|
||||
annotations:
|
||||
config.kubernetes.io/local-config: "true"
|
||||
config.kubernetes.io/index: '0'
|
||||
config.kubernetes.io/path: 'f2.yaml'
|
||||
namespace: bar
|
||||
spec:
|
||||
replicas: 3
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
labels:
|
||||
app: nginx
|
||||
name: bar
|
||||
annotations:
|
||||
app: nginx
|
||||
config.kubernetes.io/index: '1'
|
||||
config.kubernetes.io/path: 'f2.yaml'
|
||||
namespace: foo
|
||||
spec:
|
||||
replicas: 3
|
||||
`
|
||||
|
||||
expectedFilterApiVersionV1 = `kind: Deployment
|
||||
metadata:
|
||||
labels:
|
||||
app: nginx2
|
||||
name: foo
|
||||
annotations:
|
||||
app: nginx2
|
||||
config.kubernetes.io/index: '0'
|
||||
config.kubernetes.io/path: 'f1.yaml'
|
||||
spec:
|
||||
replicas: 1
|
||||
---
|
||||
kind: Service
|
||||
metadata:
|
||||
name: foo
|
||||
annotations:
|
||||
app: nginx
|
||||
config.kubernetes.io/index: '1'
|
||||
config.kubernetes.io/path: 'f1.yaml'
|
||||
spec:
|
||||
selector:
|
||||
app: nginx
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Abstraction
|
||||
metadata:
|
||||
name: foo
|
||||
configFn:
|
||||
container:
|
||||
image: gcr.io/example/reconciler:v1
|
||||
annotations:
|
||||
config.kubernetes.io/local-config: "true"
|
||||
a: 'b'
|
||||
config.kubernetes.io/index: '0'
|
||||
config.kubernetes.io/path: 'f2.yaml'
|
||||
namespace: bar
|
||||
spec:
|
||||
replicas: 3
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
labels:
|
||||
app: nginx
|
||||
name: bar
|
||||
annotations:
|
||||
app: nginx
|
||||
config.kubernetes.io/index: '1'
|
||||
config.kubernetes.io/path: 'f2.yaml'
|
||||
namespace: foo
|
||||
spec:
|
||||
replicas: 3
|
||||
`
|
||||
|
||||
expectedFilterNameBar = `kind: Deployment
|
||||
metadata:
|
||||
labels:
|
||||
app: nginx2
|
||||
name: foo
|
||||
annotations:
|
||||
app: nginx2
|
||||
config.kubernetes.io/index: '0'
|
||||
config.kubernetes.io/path: 'f1.yaml'
|
||||
spec:
|
||||
replicas: 1
|
||||
---
|
||||
kind: Service
|
||||
metadata:
|
||||
name: foo
|
||||
annotations:
|
||||
app: nginx
|
||||
config.kubernetes.io/index: '1'
|
||||
config.kubernetes.io/path: 'f1.yaml'
|
||||
spec:
|
||||
selector:
|
||||
app: nginx
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Abstraction
|
||||
metadata:
|
||||
name: foo
|
||||
configFn:
|
||||
container:
|
||||
image: gcr.io/example/reconciler:v1
|
||||
annotations:
|
||||
config.kubernetes.io/local-config: "true"
|
||||
config.kubernetes.io/index: '0'
|
||||
config.kubernetes.io/path: 'f2.yaml'
|
||||
namespace: bar
|
||||
spec:
|
||||
replicas: 3
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
labels:
|
||||
app: nginx
|
||||
name: bar
|
||||
annotations:
|
||||
app: nginx
|
||||
a: 'b'
|
||||
config.kubernetes.io/index: '1'
|
||||
config.kubernetes.io/path: 'f2.yaml'
|
||||
namespace: foo
|
||||
spec:
|
||||
replicas: 3
|
||||
`
|
||||
|
||||
expectedFilterNamespaceBar = `kind: Deployment
|
||||
metadata:
|
||||
labels:
|
||||
app: nginx2
|
||||
name: foo
|
||||
annotations:
|
||||
app: nginx2
|
||||
config.kubernetes.io/index: '0'
|
||||
config.kubernetes.io/path: 'f1.yaml'
|
||||
spec:
|
||||
replicas: 1
|
||||
---
|
||||
kind: Service
|
||||
metadata:
|
||||
name: foo
|
||||
annotations:
|
||||
app: nginx
|
||||
config.kubernetes.io/index: '1'
|
||||
config.kubernetes.io/path: 'f1.yaml'
|
||||
spec:
|
||||
selector:
|
||||
app: nginx
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Abstraction
|
||||
metadata:
|
||||
name: foo
|
||||
configFn:
|
||||
container:
|
||||
image: gcr.io/example/reconciler:v1
|
||||
annotations:
|
||||
config.kubernetes.io/local-config: "true"
|
||||
a: 'b'
|
||||
config.kubernetes.io/index: '0'
|
||||
config.kubernetes.io/path: 'f2.yaml'
|
||||
namespace: bar
|
||||
spec:
|
||||
replicas: 3
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
labels:
|
||||
app: nginx
|
||||
name: bar
|
||||
annotations:
|
||||
app: nginx
|
||||
config.kubernetes.io/index: '1'
|
||||
config.kubernetes.io/path: 'f2.yaml'
|
||||
namespace: foo
|
||||
spec:
|
||||
replicas: 3
|
||||
`
|
||||
)
|
||||
@@ -4,6 +4,20 @@
|
||||
// Code generated by "mdtogo"; DO NOT EDIT.
|
||||
package commands
|
||||
|
||||
var AnnotateShort = `[Alpha] Set an annotation on Resources.`
|
||||
var AnnotateLong = `
|
||||
[Alpha] Set an annotation on Resources.
|
||||
|
||||
DIR:
|
||||
Path to local directory.
|
||||
`
|
||||
var AnnotateExamples = `
|
||||
kustomize config annotate my-dir/ --kv foo=bar
|
||||
|
||||
kustomize config annotate my-dir/ --kv foo=bar --kv a=b
|
||||
|
||||
kustomize config annotate my-dir/ --kv foo=bar --kind Deployment --name foo`
|
||||
|
||||
var CatShort = `[Alpha] Print Resource Config from a local directory.`
|
||||
var CatLong = `
|
||||
[Alpha] Print Resource Config from a local directory.
|
||||
|
||||
@@ -11,11 +11,11 @@ require (
|
||||
k8s.io/component-base v0.17.0 // indirect
|
||||
k8s.io/kubectl v0.0.0-20191219154910-1528d4eea6dd
|
||||
sigs.k8s.io/controller-runtime v0.4.0
|
||||
sigs.k8s.io/kustomize/kstatus v0.0.0-20200109211150-9555095de939
|
||||
sigs.k8s.io/kustomize/kstatus v0.0.1
|
||||
sigs.k8s.io/kustomize/kyaml v0.0.2
|
||||
)
|
||||
|
||||
replace (
|
||||
sigs.k8s.io/kustomize/kstatus v0.0.0-20200109211150-9555095de939 => ../../kstatus
|
||||
sigs.k8s.io/kustomize/kstatus v0.0.1 => ../../kstatus
|
||||
sigs.k8s.io/kustomize/kyaml v0.0.0 => ../../kyaml
|
||||
)
|
||||
|
||||
@@ -6,6 +6,7 @@ package kubectlcobra
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"k8s.io/apimachinery/pkg/runtime/schema"
|
||||
@@ -81,3 +82,99 @@ func (i *Inventory) String() string {
|
||||
i.GroupKind.Group, fieldSeparator,
|
||||
i.GroupKind.Kind)
|
||||
}
|
||||
|
||||
// InventorySet encapsulates a grouping of unique Inventory
|
||||
// structs. Organizes the Inventory structs with a map,
|
||||
// which ensures there are no duplicates. Allows set
|
||||
// operations such as merging sets and subtracting sets.
|
||||
type InventorySet struct {
|
||||
set map[string]*Inventory
|
||||
}
|
||||
|
||||
// NewInventorySet returns a pointer to an InventorySet
|
||||
// struct grouping the passed Inventory items.
|
||||
func NewInventorySet(items []*Inventory) *InventorySet {
|
||||
invSet := InventorySet{set: map[string]*Inventory{}}
|
||||
invSet.AddItems(items)
|
||||
return &invSet
|
||||
}
|
||||
|
||||
// GetItems returns the set of pointers to Inventory
|
||||
// structs.
|
||||
func (is *InventorySet) GetItems() []*Inventory {
|
||||
items := []*Inventory{}
|
||||
for _, item := range is.set {
|
||||
items = append(items, item)
|
||||
}
|
||||
return items
|
||||
}
|
||||
|
||||
// AddItems adds Inventory structs to the set which
|
||||
// are not already in the set.
|
||||
func (is *InventorySet) AddItems(items []*Inventory) {
|
||||
for _, item := range items {
|
||||
if item != nil {
|
||||
is.set[item.String()] = item
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// DeleteItem removes an Inventory struct from the
|
||||
// set if it exists in the set. Returns true if the
|
||||
// Inventory item was deleted, false if it did not exist
|
||||
// in the set.
|
||||
func (is *InventorySet) DeleteItem(item *Inventory) bool {
|
||||
if item == nil {
|
||||
return false
|
||||
}
|
||||
if _, ok := is.set[item.String()]; ok {
|
||||
delete(is.set, item.String())
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// Merge combines the unique set of Inventory items from the
|
||||
// current set with the passed "other" set, returning a new
|
||||
// set or error. Returns an error if the passed set to merge
|
||||
// is nil.
|
||||
func (is *InventorySet) Merge(other *InventorySet) (*InventorySet, error) {
|
||||
if other == nil {
|
||||
return nil, fmt.Errorf("InventorySet to merge is nil.")
|
||||
}
|
||||
// Copy the current InventorySet into result
|
||||
result := NewInventorySet(is.GetItems())
|
||||
result.AddItems(other.GetItems())
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// Subtract removes the Inventory items in the "other" set from the
|
||||
// current set, returning a new set. This does not modify the current
|
||||
// set. Returns an error if the passed set to subtract is nil.
|
||||
func (is *InventorySet) Subtract(other *InventorySet) (*InventorySet, error) {
|
||||
if other == nil {
|
||||
return nil, fmt.Errorf("InventorySet to subtract is nil.")
|
||||
}
|
||||
// Copy the current InventorySet into result
|
||||
result := NewInventorySet(is.GetItems())
|
||||
// Remove each item in "other" which exists in "result"
|
||||
for _, item := range other.GetItems() {
|
||||
result.DeleteItem(item)
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// String returns a string describing set of Inventory structs.
|
||||
func (is *InventorySet) String() string {
|
||||
strs := []string{}
|
||||
for _, item := range is.GetItems() {
|
||||
strs = append(strs, item.String())
|
||||
}
|
||||
sort.Strings(strs)
|
||||
return strings.Join(strs, ", ")
|
||||
}
|
||||
|
||||
// Size returns the number of Inventory structs in the set.
|
||||
func (is *InventorySet) Size() int {
|
||||
return len(is.set)
|
||||
}
|
||||
|
||||
@@ -216,3 +216,274 @@ func TestParseInventory(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var inventory1 = Inventory{
|
||||
Namespace: "test-namespace",
|
||||
Name: "test-inv-1",
|
||||
GroupKind: schema.GroupKind{
|
||||
Group: "apps",
|
||||
Kind: "Deployment",
|
||||
},
|
||||
}
|
||||
|
||||
var inventory2 = Inventory{
|
||||
Namespace: "test-namespace",
|
||||
Name: "test-inv-2",
|
||||
GroupKind: schema.GroupKind{
|
||||
Group: "",
|
||||
Kind: "Pod",
|
||||
},
|
||||
}
|
||||
|
||||
var inventory3 = Inventory{
|
||||
Namespace: "test-namespace",
|
||||
Name: "test-inv-3",
|
||||
GroupKind: schema.GroupKind{
|
||||
Group: "",
|
||||
Kind: "Service",
|
||||
},
|
||||
}
|
||||
|
||||
var inventory4 = Inventory{
|
||||
Namespace: "test-namespace",
|
||||
Name: "test-inv-4",
|
||||
GroupKind: schema.GroupKind{
|
||||
Group: "apps",
|
||||
Kind: "DaemonSet",
|
||||
},
|
||||
}
|
||||
|
||||
func TestNewInventorySet(t *testing.T) {
|
||||
tests := []struct {
|
||||
items []*Inventory
|
||||
expectedStr string
|
||||
expectedSize int
|
||||
}{
|
||||
{
|
||||
items: []*Inventory{},
|
||||
expectedStr: "",
|
||||
expectedSize: 0,
|
||||
},
|
||||
{
|
||||
items: []*Inventory{&inventory1},
|
||||
expectedStr: "test-namespace_test-inv-1_apps_Deployment",
|
||||
expectedSize: 1,
|
||||
},
|
||||
{
|
||||
items: []*Inventory{&inventory1, &inventory2},
|
||||
expectedStr: "test-namespace_test-inv-1_apps_Deployment, test-namespace_test-inv-2__Pod",
|
||||
expectedSize: 2,
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
invSet := NewInventorySet(test.items)
|
||||
actualStr := invSet.String()
|
||||
actualSize := invSet.Size()
|
||||
if test.expectedStr != actualStr {
|
||||
t.Errorf("Expected InventorySet (%s), got (%s)\n", test.expectedStr, actualStr)
|
||||
}
|
||||
if test.expectedSize != actualSize {
|
||||
t.Errorf("Expected InventorySet size (%d), got (%d)\n", test.expectedSize, actualSize)
|
||||
}
|
||||
actualItems := invSet.GetItems()
|
||||
if len(test.items) != len(actualItems) {
|
||||
t.Errorf("Expected num inventory items (%d), got (%d)\n", len(test.items), len(actualItems))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestInventorySetAddItems(t *testing.T) {
|
||||
tests := []struct {
|
||||
initialItems []*Inventory
|
||||
addItems []*Inventory
|
||||
expectedItems []*Inventory
|
||||
}{
|
||||
// Adding no items to empty inventory set.
|
||||
{
|
||||
initialItems: []*Inventory{},
|
||||
addItems: []*Inventory{},
|
||||
expectedItems: []*Inventory{},
|
||||
},
|
||||
// Adding item to empty inventory set.
|
||||
{
|
||||
initialItems: []*Inventory{},
|
||||
addItems: []*Inventory{&inventory1},
|
||||
expectedItems: []*Inventory{&inventory1},
|
||||
},
|
||||
// Adding no items does not change the inventory set
|
||||
{
|
||||
initialItems: []*Inventory{&inventory1},
|
||||
addItems: []*Inventory{},
|
||||
expectedItems: []*Inventory{&inventory1},
|
||||
},
|
||||
// Adding an item which alread exists does not increase size.
|
||||
{
|
||||
initialItems: []*Inventory{&inventory1, &inventory2},
|
||||
addItems: []*Inventory{&inventory1},
|
||||
expectedItems: []*Inventory{&inventory1, &inventory2},
|
||||
},
|
||||
{
|
||||
initialItems: []*Inventory{&inventory1, &inventory2},
|
||||
addItems: []*Inventory{&inventory3, &inventory4},
|
||||
expectedItems: []*Inventory{&inventory1, &inventory2, &inventory3, &inventory4},
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
invSet := NewInventorySet(test.initialItems)
|
||||
invSet.AddItems(test.addItems)
|
||||
if len(test.expectedItems) != invSet.Size() {
|
||||
t.Errorf("Expected num inventory items (%d), got (%d)\n", len(test.expectedItems), invSet.Size())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestInventorySetDeleteItem(t *testing.T) {
|
||||
tests := []struct {
|
||||
initialItems []*Inventory
|
||||
deleteItem *Inventory
|
||||
expected bool
|
||||
expectedItems []*Inventory
|
||||
}{
|
||||
{
|
||||
initialItems: []*Inventory{},
|
||||
deleteItem: nil,
|
||||
expected: false,
|
||||
expectedItems: []*Inventory{},
|
||||
},
|
||||
{
|
||||
initialItems: []*Inventory{},
|
||||
deleteItem: &inventory1,
|
||||
expected: false,
|
||||
expectedItems: []*Inventory{},
|
||||
},
|
||||
{
|
||||
initialItems: []*Inventory{&inventory2},
|
||||
deleteItem: &inventory1,
|
||||
expected: false,
|
||||
expectedItems: []*Inventory{&inventory2},
|
||||
},
|
||||
{
|
||||
initialItems: []*Inventory{&inventory1},
|
||||
deleteItem: &inventory1,
|
||||
expected: true,
|
||||
expectedItems: []*Inventory{},
|
||||
},
|
||||
{
|
||||
initialItems: []*Inventory{&inventory1, &inventory2},
|
||||
deleteItem: &inventory1,
|
||||
expected: true,
|
||||
expectedItems: []*Inventory{&inventory2},
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
invSet := NewInventorySet(test.initialItems)
|
||||
actual := invSet.DeleteItem(test.deleteItem)
|
||||
if test.expected != actual {
|
||||
t.Errorf("Expected return value (%t), got (%t)\n", test.expected, actual)
|
||||
}
|
||||
if len(test.expectedItems) != invSet.Size() {
|
||||
t.Errorf("Expected num inventory items (%d), got (%d)\n", len(test.expectedItems), invSet.Size())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestInventorySetMerge(t *testing.T) {
|
||||
tests := []struct {
|
||||
set1 []*Inventory
|
||||
set2 []*Inventory
|
||||
merged []*Inventory
|
||||
}{
|
||||
{
|
||||
set1: []*Inventory{},
|
||||
set2: []*Inventory{},
|
||||
merged: []*Inventory{},
|
||||
},
|
||||
{
|
||||
set1: []*Inventory{},
|
||||
set2: []*Inventory{&inventory1},
|
||||
merged: []*Inventory{&inventory1},
|
||||
},
|
||||
{
|
||||
set1: []*Inventory{&inventory1},
|
||||
set2: []*Inventory{},
|
||||
merged: []*Inventory{&inventory1},
|
||||
},
|
||||
{
|
||||
set1: []*Inventory{&inventory1, &inventory2},
|
||||
set2: []*Inventory{&inventory1},
|
||||
merged: []*Inventory{&inventory1, &inventory2},
|
||||
},
|
||||
{
|
||||
set1: []*Inventory{&inventory1, &inventory2},
|
||||
set2: []*Inventory{&inventory1, &inventory2},
|
||||
merged: []*Inventory{&inventory1, &inventory2},
|
||||
},
|
||||
{
|
||||
set1: []*Inventory{&inventory1, &inventory2},
|
||||
set2: []*Inventory{&inventory3, &inventory4},
|
||||
merged: []*Inventory{&inventory1, &inventory2, &inventory3, &inventory4},
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
invSet1 := NewInventorySet(test.set1)
|
||||
invSet2 := NewInventorySet(test.set2)
|
||||
expected := NewInventorySet(test.merged)
|
||||
merged, _ := invSet1.Merge(invSet2)
|
||||
if expected.Size() != merged.Size() {
|
||||
t.Errorf("Expected merged inventory set size (%d), got (%d)\n", expected.Size(), merged.Size())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestInventorySetSubtract(t *testing.T) {
|
||||
tests := []struct {
|
||||
initialItems []*Inventory
|
||||
subtractItems []*Inventory
|
||||
expected []*Inventory
|
||||
}{
|
||||
{
|
||||
initialItems: []*Inventory{},
|
||||
subtractItems: []*Inventory{},
|
||||
expected: []*Inventory{},
|
||||
},
|
||||
{
|
||||
initialItems: []*Inventory{},
|
||||
subtractItems: []*Inventory{&inventory1},
|
||||
expected: []*Inventory{},
|
||||
},
|
||||
{
|
||||
initialItems: []*Inventory{&inventory1},
|
||||
subtractItems: []*Inventory{},
|
||||
expected: []*Inventory{&inventory1},
|
||||
},
|
||||
{
|
||||
initialItems: []*Inventory{&inventory1, &inventory2},
|
||||
subtractItems: []*Inventory{&inventory1},
|
||||
expected: []*Inventory{&inventory2},
|
||||
},
|
||||
{
|
||||
initialItems: []*Inventory{&inventory1, &inventory2},
|
||||
subtractItems: []*Inventory{&inventory1, &inventory2},
|
||||
expected: []*Inventory{},
|
||||
},
|
||||
{
|
||||
initialItems: []*Inventory{&inventory1, &inventory2},
|
||||
subtractItems: []*Inventory{&inventory3, &inventory4},
|
||||
expected: []*Inventory{&inventory1, &inventory2},
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
invInitialItems := NewInventorySet(test.initialItems)
|
||||
invSubtractItems := NewInventorySet(test.subtractItems)
|
||||
expected := NewInventorySet(test.expected)
|
||||
actual, _ := invInitialItems.Subtract(invSubtractItems)
|
||||
if expected.Size() != actual.Size() {
|
||||
t.Errorf("Expected subtracted inventory set size (%d), got (%d)\n", expected.Size(), actual.Size())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
113
examples/alphaTestExamples/helloworld/README.md
Normal file
113
examples/alphaTestExamples/helloworld/README.md
Normal file
@@ -0,0 +1,113 @@
|
||||
[base]: ../../docs/glossary.md#base
|
||||
[config]: https://github.com/kinflate/example-hello
|
||||
[gitops]: ../../docs/glossary.md#gitops
|
||||
[hello]: https://github.com/monopole/hello
|
||||
[kustomization]: ../../docs/glossary.md#kustomization
|
||||
[original]: https://github.com/kinflate/example-hello
|
||||
[overlay]: ../../docs/glossary.md#overlay
|
||||
[overlays]: ../../docs/glossary.md#overlay
|
||||
[patch]: ../../docs/glossary.md#patch
|
||||
[variant]: ../../docs/glossary.md#variant
|
||||
[variants]: ../../docs/glossary.md#variant
|
||||
|
||||
# Demo: hello world
|
||||
|
||||
Steps:
|
||||
|
||||
1. Clone an existing configuration as a [base].
|
||||
1. Customize it.
|
||||
|
||||
First define a place to work:
|
||||
|
||||
<!-- @makeWorkplace @testE2EAgainstLatestRelease-->
|
||||
```
|
||||
DEMO_HOME=$(mktemp -d)
|
||||
```
|
||||
|
||||
Alternatively, use
|
||||
|
||||
> ```
|
||||
> DEMO_HOME=~/hello
|
||||
> ```
|
||||
|
||||
## Establish the base
|
||||
|
||||
Let's run the [hello] service.
|
||||
|
||||
To keep this document shorter, the base resources are
|
||||
off in a supplemental data directory rather than
|
||||
declared here as HERE documents. Download them:
|
||||
|
||||
<!-- @downloadBase @testE2EAgainstLatestRelease-->
|
||||
```
|
||||
BASE=$DEMO_HOME/base
|
||||
mkdir -p $BASE
|
||||
|
||||
curl -s -o "$BASE/#1.yaml" "https://raw.githubusercontent.com\
|
||||
/kubernetes-sigs/kustomize\
|
||||
/master/examples/alphaTestExamples/helloWorld\
|
||||
/{configMap,deployment,grouping,kustomization,service}.yaml"
|
||||
```
|
||||
|
||||
### The Base Kustomization
|
||||
|
||||
The `base` directory has a [kustomization] file:
|
||||
|
||||
<!-- @showKustomization @testE2EAgainstLatestRelease -->
|
||||
```
|
||||
more $BASE/kustomization.yaml
|
||||
```
|
||||
|
||||
### Customize the base
|
||||
|
||||
A first customization step could be to change the _app
|
||||
label_ applied to all resources:
|
||||
|
||||
<!-- @addLabel @testE2EAgainstLatestRelease -->
|
||||
```
|
||||
sed -i.bak 's/app: hello/app: my-hello/' \
|
||||
$BASE/kustomization.yaml
|
||||
```
|
||||
|
||||
To do end to end tests using kustomize, go through the following section. You should have GOPATH set up and "kind" installed(https://github.com/kubernetes-sigs/kind).
|
||||
|
||||
<!-- @setGoBin @testE2EAgainstLatestRelease -->
|
||||
```
|
||||
MYGOBIN=$GOPATH/bin
|
||||
```
|
||||
|
||||
Delete any existing kind cluster and create a new one. By default the name of the cluster is "kind"
|
||||
<!-- @deleteAndCreateKindCluster @testE2EAgainstLatestRelease -->
|
||||
```
|
||||
kind delete cluster;
|
||||
kind create cluster;
|
||||
```
|
||||
|
||||
Use the kustomize binary in MYGOBIN to apply a deployment, fetch the status and verify the status.
|
||||
<!-- @e2eTestUsingKustomize @testE2EAgainstLatestRelease -->
|
||||
```
|
||||
export KUSTOMIZE_ENABLE_ALPHA_COMMANDS=true
|
||||
|
||||
$MYGOBIN/kustomize resources apply $BASE --status;
|
||||
|
||||
status=$(mktemp);
|
||||
$MYGOBIN/resource status fetch $BASE > $status
|
||||
|
||||
test 1 == \
|
||||
$(grep "the-deployment" $status | grep "Deployment is available. Replicas: 3" | wc -l); \
|
||||
echo $?
|
||||
|
||||
test 1 == \
|
||||
$(grep "the-map" $status | grep "Resource is always ready" | wc -l); \
|
||||
echo $?
|
||||
|
||||
test 1 == \
|
||||
$(grep "the-service" $status | grep "Service is ready" | wc -l); \
|
||||
echo $?
|
||||
```
|
||||
|
||||
Clean-up the cluster
|
||||
<!-- @createKindCluster @testE2EAgainstLatestRelease -->
|
||||
```
|
||||
kind delete cluster;
|
||||
```
|
||||
7
examples/alphaTestExamples/helloworld/configMap.yaml
Normal file
7
examples/alphaTestExamples/helloworld/configMap.yaml
Normal file
@@ -0,0 +1,7 @@
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: the-map
|
||||
data:
|
||||
altGreeting: "Good Morning!"
|
||||
enableRisky: "false"
|
||||
30
examples/alphaTestExamples/helloworld/deployment.yaml
Normal file
30
examples/alphaTestExamples/helloworld/deployment.yaml
Normal file
@@ -0,0 +1,30 @@
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: the-deployment
|
||||
spec:
|
||||
replicas: 3
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
deployment: hello
|
||||
spec:
|
||||
containers:
|
||||
- name: the-container
|
||||
image: monopole/hello:1
|
||||
command: ["/hello",
|
||||
"--port=8080",
|
||||
"--enableRiskyFeature=$(ENABLE_RISKY)"]
|
||||
ports:
|
||||
- containerPort: 8080
|
||||
env:
|
||||
- name: ALT_GREETING
|
||||
valueFrom:
|
||||
configMapKeyRef:
|
||||
name: the-map
|
||||
key: altGreeting
|
||||
- name: ENABLE_RISKY
|
||||
valueFrom:
|
||||
configMapKeyRef:
|
||||
name: the-map
|
||||
key: enableRisky
|
||||
6
examples/alphaTestExamples/helloworld/grouping.yaml
Normal file
6
examples/alphaTestExamples/helloworld/grouping.yaml
Normal file
@@ -0,0 +1,6 @@
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: inventory-map
|
||||
labels:
|
||||
"kustomize.config.k8s.io/inventory-id": "hello-world-app"
|
||||
10
examples/alphaTestExamples/helloworld/kustomization.yaml
Normal file
10
examples/alphaTestExamples/helloworld/kustomization.yaml
Normal file
@@ -0,0 +1,10 @@
|
||||
# Example configuration for the webserver
|
||||
# at https://github.com/monopole/hello
|
||||
commonLabels:
|
||||
app: hello
|
||||
|
||||
resources:
|
||||
- deployment.yaml
|
||||
- service.yaml
|
||||
- configMap.yaml
|
||||
- grouping.yaml
|
||||
12
examples/alphaTestExamples/helloworld/service.yaml
Normal file
12
examples/alphaTestExamples/helloworld/service.yaml
Normal file
@@ -0,0 +1,12 @@
|
||||
kind: Service
|
||||
apiVersion: v1
|
||||
metadata:
|
||||
name: the-service
|
||||
spec:
|
||||
selector:
|
||||
deployment: hello
|
||||
type: LoadBalancer
|
||||
ports:
|
||||
- protocol: TCP
|
||||
port: 8666
|
||||
targetPort: 8080
|
||||
@@ -22,7 +22,7 @@ Steps:
|
||||
|
||||
First define a place to work:
|
||||
|
||||
<!-- @makeWorkplace @testAgainstLatestRelease @testE2EAgainstLatestRelease-->
|
||||
<!-- @makeWorkplace @testAgainstLatestRelease -->
|
||||
```
|
||||
DEMO_HOME=$(mktemp -d)
|
||||
```
|
||||
@@ -44,7 +44,7 @@ To keep this document shorter, the base resources are
|
||||
off in a supplemental data directory rather than
|
||||
declared here as HERE documents. Download them:
|
||||
|
||||
<!-- @downloadBase @testAgainstLatestRelease @testE2EAgainstLatestRelease-->
|
||||
<!-- @downloadBase @testAgainstLatestRelease -->
|
||||
```
|
||||
BASE=$DEMO_HOME/base
|
||||
mkdir -p $BASE
|
||||
@@ -309,31 +309,3 @@ To deploy, pipe the above commands to kubectl apply:
|
||||
> kustomize build $OVERLAYS/production |\
|
||||
> kubectl apply -f -
|
||||
> ```
|
||||
|
||||
[Alpha] To do end to end tests using kustomize, use the following commands on any folder. You should have GOPATH set up and "kind" installed(https://github.com/kubernetes-sigs/kind).
|
||||
|
||||
<!-- @e2eTest @testE2EAgainstLatestRelease -->
|
||||
```
|
||||
MYGOBIN=$GOPATH/bin
|
||||
kind delete cluster;
|
||||
kind create cluster;
|
||||
$MYGOBIN/kustomize build $BASE | kubectl apply -f -;
|
||||
status=$(mktemp);
|
||||
$MYGOBIN/resource status events $BASE #Waits for all transient events to finish
|
||||
$MYGOBIN/resource status fetch $BASE > $status
|
||||
|
||||
test 1 == \
|
||||
$(grep "apps/v1/Deployment" $status | grep "Deployment is available. Replicas: 3" | wc -l); \
|
||||
echo $?
|
||||
|
||||
test 1 == \
|
||||
$(grep "v1/ConfigMap" $status | grep "Resource is always ready" | wc -l); \
|
||||
echo $?
|
||||
|
||||
test 1 == \
|
||||
$(grep "v1/Service" $status | grep "Service is ready" | wc -l); \
|
||||
echo $?
|
||||
|
||||
$MYGOBIN/kustomize build $BASE | kubectl delete -f -;
|
||||
kind delete cluster;
|
||||
```
|
||||
@@ -8,6 +8,6 @@ set -o errexit
|
||||
set -o pipefail
|
||||
|
||||
mdrip --blockTimeOut 60m0s --mode test \
|
||||
--label testE2EAgainstLatestRelease examples
|
||||
--label testE2EAgainstLatestRelease examples/alphaTestExamples
|
||||
|
||||
echo "Example e2e tests passed against ${version}."
|
||||
echo "Example e2e tests passed against ."
|
||||
|
||||
@@ -18,3 +18,8 @@ exclude (
|
||||
github.com/russross/blackfriday v2.0.0+incompatible
|
||||
sigs.k8s.io/kustomize/api v0.2.0
|
||||
)
|
||||
|
||||
replace (
|
||||
sigs.k8s.io/kustomize/cmd/kubectl v0.0.3 => ../cmd/kubectl
|
||||
sigs.k8s.io/kustomize/kstatus v0.0.1 => ../kstatus
|
||||
)
|
||||
|
||||
@@ -24,6 +24,8 @@ type flagsAndArgs struct {
|
||||
EnvFileSource string
|
||||
// Type of secret to create
|
||||
Type string
|
||||
// Namespace of secret
|
||||
Namespace string
|
||||
}
|
||||
|
||||
// Validate validates required fields are set to support structured generation.
|
||||
|
||||
@@ -86,6 +86,11 @@ func newCmdAddSecret(
|
||||
"type",
|
||||
"Opaque",
|
||||
"Specify the secret type this can be 'Opaque' (default), or 'kubernetes.io/tls'")
|
||||
cmd.Flags().StringVar(
|
||||
&flags.Namespace,
|
||||
"namespace",
|
||||
"",
|
||||
"Specify the namespace of the secret")
|
||||
|
||||
return cmd
|
||||
}
|
||||
@@ -97,7 +102,7 @@ func addSecret(
|
||||
ldr ifc.KvLoader,
|
||||
k *types.Kustomization,
|
||||
flags flagsAndArgs, kf ifc.KunstructuredFactory) error {
|
||||
args := findOrMakeSecretArgs(k, flags.Name, flags.Type)
|
||||
args := findOrMakeSecretArgs(k, flags.Name, flags.Namespace, flags.Type)
|
||||
mergeFlagsIntoGeneratorArgs(&args.GeneratorArgs, flags)
|
||||
// Validate by trying to create corev1.secret.
|
||||
_, err := kf.MakeSecret(ldr, k.GeneratorOptions, args)
|
||||
@@ -107,16 +112,17 @@ func addSecret(
|
||||
return nil
|
||||
}
|
||||
|
||||
func findOrMakeSecretArgs(m *types.Kustomization, name, secretType string) *types.SecretArgs {
|
||||
func findOrMakeSecretArgs(m *types.Kustomization, name, namespace, secretType string) *types.SecretArgs {
|
||||
for i, v := range m.SecretGenerator {
|
||||
if name == v.Name {
|
||||
if name == v.Name && namespace == v.Namespace {
|
||||
return &m.SecretGenerator[i]
|
||||
}
|
||||
}
|
||||
// secret not found, create new one and add it to the kustomization file.
|
||||
secret := &types.SecretArgs{
|
||||
GeneratorArgs: types.GeneratorArgs{Name: name},
|
||||
Type: secretType}
|
||||
GeneratorArgs: types.GeneratorArgs{Name: name, Namespace: namespace},
|
||||
Type: secretType,
|
||||
}
|
||||
m.SecretGenerator = append(m.SecretGenerator, *secret)
|
||||
return &m.SecretGenerator[len(m.SecretGenerator)-1]
|
||||
}
|
||||
|
||||
@@ -28,6 +28,7 @@ func TestNewCmdAddSecretIsNotNil(t *testing.T) {
|
||||
|
||||
func TestMakeSecretArgs(t *testing.T) {
|
||||
secretName := "test-secret-name"
|
||||
namespace := "test-secret-namespace"
|
||||
|
||||
kustomization := &types.Kustomization{
|
||||
NamePrefix: "test-name-prefix",
|
||||
@@ -38,7 +39,7 @@ func TestMakeSecretArgs(t *testing.T) {
|
||||
if len(kustomization.SecretGenerator) != 0 {
|
||||
t.Fatal("Initial kustomization should not have any secrets")
|
||||
}
|
||||
args := findOrMakeSecretArgs(kustomization, secretName, secretType)
|
||||
args := findOrMakeSecretArgs(kustomization, secretName, namespace, secretType)
|
||||
|
||||
if args == nil {
|
||||
t.Fatalf("args should always be non-nil")
|
||||
@@ -52,7 +53,7 @@ func TestMakeSecretArgs(t *testing.T) {
|
||||
t.Fatalf("Pointer address for newly inserted secret generator should be same")
|
||||
}
|
||||
|
||||
args2 := findOrMakeSecretArgs(kustomization, secretName, secretType)
|
||||
args2 := findOrMakeSecretArgs(kustomization, secretName, namespace, secretType)
|
||||
|
||||
if args2 != args {
|
||||
t.Fatalf("should have returned an existing args with name: %v", secretName)
|
||||
@@ -65,7 +66,7 @@ func TestMakeSecretArgs(t *testing.T) {
|
||||
|
||||
func TestMergeFlagsIntoSecretArgs_LiteralSources(t *testing.T) {
|
||||
k := &types.Kustomization{}
|
||||
args := findOrMakeSecretArgs(k, "foo", "forbidden")
|
||||
args := findOrMakeSecretArgs(k, "foo", "bar", "forbidden")
|
||||
mergeFlagsIntoGeneratorArgs(
|
||||
&args.GeneratorArgs,
|
||||
flagsAndArgs{LiteralSources: []string{"k1=v1"}})
|
||||
@@ -82,7 +83,7 @@ func TestMergeFlagsIntoSecretArgs_LiteralSources(t *testing.T) {
|
||||
|
||||
func TestMergeFlagsIntoSecretArgs_FileSources(t *testing.T) {
|
||||
k := &types.Kustomization{}
|
||||
args := findOrMakeSecretArgs(k, "foo", "forbidden")
|
||||
args := findOrMakeSecretArgs(k, "foo", "bar", "forbidden")
|
||||
mergeFlagsIntoGeneratorArgs(
|
||||
&args.GeneratorArgs,
|
||||
flagsAndArgs{FileSources: []string{"file1"}})
|
||||
@@ -99,7 +100,7 @@ func TestMergeFlagsIntoSecretArgs_FileSources(t *testing.T) {
|
||||
|
||||
func TestMergeFlagsIntoSecretArgs_EnvSource(t *testing.T) {
|
||||
k := &types.Kustomization{}
|
||||
args := findOrMakeSecretArgs(k, "foo", "forbidden")
|
||||
args := findOrMakeSecretArgs(k, "foo", "bar", "forbidden")
|
||||
mergeFlagsIntoGeneratorArgs(
|
||||
&args.GeneratorArgs,
|
||||
flagsAndArgs{EnvFileSource: "env1"})
|
||||
|
||||
@@ -40,10 +40,11 @@ function parseYaml {
|
||||
local file=$1
|
||||
while read -r line
|
||||
do
|
||||
local k=${line%:*}
|
||||
local k=${line%%:*}
|
||||
local v=${line#*:}
|
||||
|
||||
[ "$k" == "chartName" ] && chartName=$v
|
||||
[ "$k" == "chartRepo" ] && chartRepo=$v
|
||||
[ "$k" == "chartHome" ] && chartHome=$v
|
||||
[ "$k" == "chartRelease" ] && chartRelease=$v
|
||||
[ "$k" == "chartVersion" ] && chartVersion=$v
|
||||
@@ -56,6 +57,7 @@ function parseYaml {
|
||||
|
||||
# Trim leading space
|
||||
chartName="${chartName#"${chartName%%[![:space:]]*}"}"
|
||||
chartRepo="${chartRepo#"${chartRepo%%[![:space:]]*}"}"
|
||||
chartHome="${chartHome#"${chartHome%%[![:space:]]*}"}"
|
||||
chartRelease="${chartRelease#"${chartRelease%%[![:space:]]*}"}"
|
||||
chartVersion="${chartVersion#"${chartVersion%%[![:space:]]*}"}"
|
||||
@@ -84,6 +86,14 @@ if [ -z "$chartRelease" ]; then
|
||||
chartRelease="stable"
|
||||
fi
|
||||
|
||||
# The repo to pull the chart from
|
||||
if [ -n "$chartRepo" ]; then
|
||||
chartRepoArg="--repo=$chartRepo"
|
||||
chartNameArg="$chartName"
|
||||
else
|
||||
chartNameArg="$chartRelease/$chartName"
|
||||
fi
|
||||
|
||||
# Set version only if specified
|
||||
if [ ! -z "$chartVersion" ]; then
|
||||
chartVersionArg="--version=$chartVersion"
|
||||
@@ -114,9 +124,10 @@ doHelm init --client-only >& /dev/null
|
||||
|
||||
if [ ! -d "$chartHome/$chartName" ]; then
|
||||
doHelm fetch $chartVersionArg \
|
||||
$chartRepoArg \
|
||||
--untar \
|
||||
--untardir $chartHome \
|
||||
${chartRelease}/$chartName
|
||||
$chartNameArg
|
||||
fi
|
||||
|
||||
doHelm template \
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
# kyaml version
|
||||
export kyaml_major=0
|
||||
export kyaml_minor=0
|
||||
export kyaml_patch=7
|
||||
export kyaml_patch=8
|
||||
|
||||
# kstatus version
|
||||
export kstatus_major=0
|
||||
@@ -21,7 +21,7 @@ export api_patch=2
|
||||
# cmd/config version
|
||||
export cmd_config_major=0
|
||||
export cmd_config_minor=0
|
||||
export cmd_config_patch=8
|
||||
export cmd_config_patch=9
|
||||
|
||||
# cmd/kubectl version
|
||||
export cmd_kubectl_major=0
|
||||
|
||||
Reference in New Issue
Block a user