diff --git a/api/internal/crawl/cmd/kustomize_stats/main.go b/api/internal/crawl/cmd/kustomize_stats/main.go index 403989d86..4f8fc195f 100644 --- a/api/internal/crawl/cmd/kustomize_stats/main.go +++ b/api/internal/crawl/cmd/kustomize_stats/main.go @@ -5,12 +5,12 @@ import ( "flag" "fmt" "log" - "path/filepath" "sort" "time" + "sigs.k8s.io/kustomize/api/internal/crawl/doc" + "sigs.k8s.io/kustomize/api/internal/crawl/index" - "sigs.k8s.io/kustomize/api/konfig" ) // iterateArr adds each item in arr into countMap. @@ -25,17 +25,6 @@ func iterateArr(arr []string, countMap map[string]int) { } -// isKustomizationFile determines whether a file path is a kustomization file -func isKustomizationFile(path string) bool { - basename := filepath.Base(path) - for _, name := range konfig.RecognizedKustomizationFileNames() { - if basename == name { - return true - } - } - return false -} - // SortMapKeyByValue takes a map as its input, sorts its keys according to their values // in the map, and outputs the sorted keys as a slice. func SortMapKeyByValue(m map[string]int) []string { @@ -44,7 +33,7 @@ func SortMapKeyByValue(m map[string]int) []string { keys = append(keys, key) } // sort keys according to their values in the map m - sort.Slice(keys, func(i, j int) bool {return m[keys[i]] > m[keys[j]]}) + sort.Slice(keys, func(i, j int) bool { return m[keys[i]] > m[keys[j]] }) return keys } @@ -101,7 +90,7 @@ If you only want to list the 10 most popular features, set the flag to 10.`) iterateArr(hit.Document.Kinds, kindsMap) iterateArr(hit.Document.Identifiers, identifiersMap) - if isKustomizationFile(hit.Document.FilePath) { + if doc.IsKustomizationFile(hit.Document.FilePath) { kustomizationFilecount++ iterateArr(hit.Document.Identifiers, kustomizeIdentifiersMap) } diff --git a/api/internal/crawl/crawler/crawler.go b/api/internal/crawl/crawler/crawler.go index b8f9d3874..b2f3ccd5f 100644 --- a/api/internal/crawl/crawler/crawler.go +++ b/api/internal/crawl/crawler/crawler.go @@ -43,7 +43,12 @@ type CrawledDocument interface { ID() string GetDocument() *doc.Document // Get all the Documents directly referred in a Document. - GetResources() ([]*doc.Document, error) + // For a Document representing a non-kustomization file, an empty slice will be returned. + // For a Document representing a kustomization file: + // the `includeResources` parameter determines whether the documents referred in the `resources` field are returned or not; + // the `includeTransformers` parameter determines whether the documents referred in the `transformers` field are returned or not; + // the `includeGenerators` parameter determines whether the documents referred in the `generators` field are returned or not. + GetResources(includeResources, includeTransformers, includeGenerators bool) ([]*doc.Document, error) WasCached() bool } @@ -95,7 +100,7 @@ func addBranches(cdoc CrawledDocument, match Crawler, indx IndexFunc, return } - deps, err := cdoc.GetResources() + deps, err := cdoc.GetResources(true, false, false) if err != nil { logger.Println(err) return diff --git a/api/internal/crawl/doc/doc.go b/api/internal/crawl/doc/doc.go index 4a709f693..87cd5a3e2 100644 --- a/api/internal/crawl/doc/doc.go +++ b/api/internal/crawl/doc/doc.go @@ -3,6 +3,7 @@ package doc import ( "fmt" "log" + "path/filepath" "sort" "strings" @@ -51,15 +52,21 @@ func (doc *KustomizationDocument) String() string { doc.IsSame, doc.Kinds, len(doc.Identifiers), len(doc.Values)) } -// Implements the CrawlerDocument interface. -func (doc *KustomizationDocument) GetResources() ([]*Document, error) { - isResource := true - for _, suffix := range konfig.RecognizedKustomizationFileNames() { - if strings.HasSuffix(doc.FilePath, "/"+suffix) { - isResource = false +// IsKustomizationFile determines whether a file path is a kustomization file +func IsKustomizationFile(path string) bool { + basename := filepath.Base(path) + for _, name := range konfig.RecognizedKustomizationFileNames() { + if basename == name { + return true } } - if isResource { + return false +} + +// Implements the CrawlerDocument interface. +func (doc *KustomizationDocument) GetResources( + includeResources, includeTransformers, includeGenerators bool) ([]*Document, error) { + if !IsKustomizationFile(doc.FilePath) { return []*Document{}, nil } @@ -77,20 +84,42 @@ func (doc *KustomizationDocument) GetResources() ([]*Document, error) { } k.FixKustomizationPostUnmarshalling() - res := make([]*Document, 0, len(k.Resources)) - for _, r := range k.Resources { + res := make([]*Document, 0) + + if includeResources { + resourceDocs := doc.CollectDocuments(k.Resources) + res = append(res, resourceDocs...) + } + + if includeGenerators { + generatorDocs := doc.CollectDocuments(k.Generators) + res = append(res, generatorDocs...) + } + + if includeTransformers { + transformerDocs := doc.CollectDocuments(k.Transformers) + res = append(res, transformerDocs...) + } + + return res, nil +} + +// CollectDocuments construct a Document for each path in paths, and return +// a slice of Document pointers. +func (doc *KustomizationDocument) CollectDocuments(paths []string) []*Document { + docs := make([]*Document, 0, len(paths)) + for _, r := range paths { if strings.TrimSpace(r) == "" { continue } next, err := doc.Document.FromRelativePath(r) if err != nil { - log.Printf("GetResources error: %v\n", err) + log.Printf("CollectDocuments error: %v\n", err) continue } - res = append(res, &next) + docs = append(docs, &next) } - - return res, nil + return docs } func (doc *KustomizationDocument) readBytes() ([]map[string]interface{}, error) { diff --git a/api/internal/crawl/doc/doc_test.go b/api/internal/crawl/doc/doc_test.go index 6fef92d97..c193809a6 100644 --- a/api/internal/crawl/doc/doc_test.go +++ b/api/internal/crawl/doc/doc_test.go @@ -189,11 +189,13 @@ metadata: } } +type TestStructForGetResources struct { + doc KustomizationDocument + resources []*Document +} + func TestGetResources(t *testing.T) { - tests := []struct { - doc KustomizationDocument - resources []*Document - }{ + tests := []TestStructForGetResources{ { doc: KustomizationDocument{ Document: Document{ @@ -248,9 +250,12 @@ resources: resources: []*Document{}, }, } + runTest(t, tests, true, false, false) +} +func runTest(t *testing.T, tests []TestStructForGetResources, includeResources, includeTransformers, includeGenerators bool) { for _, test := range tests { - res, err := test.doc.GetResources() + res, err := test.doc.GetResources(includeResources, includeTransformers, includeGenerators) if err != nil { t.Errorf("Unexpected error: %v\n", err) continue @@ -284,3 +289,73 @@ resources: } } } + +func TestGetResourcesAndGenerators(t *testing.T) { + tests := []TestStructForGetResources{ + { + doc: KustomizationDocument{ + Document: Document{ + RepositoryURL: "sigs.k8s.io/kustomize", + FilePath: "some/path/to/kdir/kustomization.yaml", + DocumentData: ` +resources: +- file.yaml + +generators: +- gen.yaml + +transformers: +- tr.yaml +`}, + }, + resources: []*Document{ + { + RepositoryURL: "sigs.k8s.io/kustomize", + FilePath: "some/path/to/kdir/gen.yaml", + }, + { + RepositoryURL: "sigs.k8s.io/kustomize", + FilePath: "some/path/to/kdir/file.yaml", + }, + }, + }, + } + runTest(t, tests, true, false, true) +} + +func TestGetResourcesAndGeneratorsAndTransformers(t *testing.T) { + tests := []TestStructForGetResources{ + { + doc: KustomizationDocument{ + Document: Document{ + RepositoryURL: "sigs.k8s.io/kustomize", + FilePath: "some/path/to/kdir/kustomization.yaml", + DocumentData: ` +resources: +- file.yaml + +generators: +- gen.yaml + +transformers: +- tr.yaml +`}, + }, + resources: []*Document{ + { + RepositoryURL: "sigs.k8s.io/kustomize", + FilePath: "some/path/to/kdir/tr.yaml", + }, + { + RepositoryURL: "sigs.k8s.io/kustomize", + FilePath: "some/path/to/kdir/gen.yaml", + }, + { + RepositoryURL: "sigs.k8s.io/kustomize", + FilePath: "some/path/to/kdir/file.yaml", + }, + }, + }, + } + runTest(t, tests, true, true, true) +}