Add support to get files referred in the generators and tranformers

fields

api/internal/crawl/cmd/kustomize_stats/main.go

@@ -5,12 +5,12 @@ import (
 	"flag"
 	"fmt"
 	"log"
-	"path/filepath"
 	"sort"
 	"time"
 
+	"sigs.k8s.io/kustomize/api/internal/crawl/doc"
+
 	"sigs.k8s.io/kustomize/api/internal/crawl/index"
-	"sigs.k8s.io/kustomize/api/konfig"
 )
 
 // iterateArr adds each item in arr into countMap.
@@ -25,17 +25,6 @@ func iterateArr(arr []string, countMap map[string]int) {
 
 }
 
-// isKustomizationFile determines whether a file path is a kustomization file
-func isKustomizationFile(path string) bool {
-	basename := filepath.Base(path)
-	for _, name := range konfig.RecognizedKustomizationFileNames() {
-		if basename == name {
-			return true
-		}
-	}
-	return false
-}
-
 // SortMapKeyByValue takes a map as its input, sorts its keys according to their values
 // in the map, and outputs the sorted keys as a slice.
 func SortMapKeyByValue(m map[string]int) []string {
@@ -44,7 +33,7 @@ func SortMapKeyByValue(m map[string]int) []string {
 		keys = append(keys, key)
 	}
 	// sort keys according to their values in the map m
-	sort.Slice(keys, func(i, j int) bool {return m[keys[i]] > m[keys[j]]})
+	sort.Slice(keys, func(i, j int) bool { return m[keys[i]] > m[keys[j]] })
 	return keys
 }
 
@@ -101,7 +90,7 @@ If you only want to list the 10 most popular features, set the flag to 10.`)
 			iterateArr(hit.Document.Kinds, kindsMap)
 			iterateArr(hit.Document.Identifiers, identifiersMap)
 
-			if isKustomizationFile(hit.Document.FilePath) {
+			if doc.IsKustomizationFile(hit.Document.FilePath) {
 				kustomizationFilecount++
 				iterateArr(hit.Document.Identifiers, kustomizeIdentifiersMap)
 			}

api/internal/crawl/crawler/crawler.go

@@ -43,7 +43,12 @@ type CrawledDocument interface {
 	ID() string
 	GetDocument() *doc.Document
 	// Get all the Documents directly referred in a Document.
-	GetResources() ([]*doc.Document, error)
+	// For a Document representing a non-kustomization file, an empty slice will be returned.
+	// For a Document representing a kustomization file:
+	// the `includeResources` parameter determines whether the documents referred in the `resources` field are returned or not;
+	// the `includeTransformers` parameter determines whether the documents referred in the `transformers` field are returned or not;
+	// the `includeGenerators` parameter determines whether the documents referred in the `generators` field are returned or not.
+	GetResources(includeResources, includeTransformers, includeGenerators bool) ([]*doc.Document, error)
 	WasCached() bool
 }
 
@@ -95,7 +100,7 @@ func addBranches(cdoc CrawledDocument, match Crawler, indx IndexFunc,
 		return
 	}
 
-	deps, err := cdoc.GetResources()
+	deps, err := cdoc.GetResources(true, false, false)
 	if err != nil {
 		logger.Println(err)
 		return

api/internal/crawl/doc/doc.go

@@ -3,6 +3,7 @@ package doc
 import (
 	"fmt"
 	"log"
+	"path/filepath"
 	"sort"
 	"strings"
 
@@ -51,15 +52,21 @@ func (doc *KustomizationDocument) String() string {
 		doc.IsSame, doc.Kinds, len(doc.Identifiers), len(doc.Values))
 }
 
-// Implements the CrawlerDocument interface.
-func (doc *KustomizationDocument) GetResources() ([]*Document, error) {
-	isResource := true
-	for _, suffix := range konfig.RecognizedKustomizationFileNames() {
-		if strings.HasSuffix(doc.FilePath, "/"+suffix) {
-			isResource = false
+// IsKustomizationFile determines whether a file path is a kustomization file
+func IsKustomizationFile(path string) bool {
+	basename := filepath.Base(path)
+	for _, name := range konfig.RecognizedKustomizationFileNames() {
+		if basename == name {
+			return true
 		}
 	}
-	if isResource {
+	return false
+}
+
+// Implements the CrawlerDocument interface.
+func (doc *KustomizationDocument) GetResources(
+	includeResources, includeTransformers, includeGenerators bool) ([]*Document, error) {
+	if !IsKustomizationFile(doc.FilePath) {
 		return []*Document{}, nil
 	}
 
@@ -77,20 +84,42 @@ func (doc *KustomizationDocument) GetResources() ([]*Document, error) {
 	}
 	k.FixKustomizationPostUnmarshalling()
 
-	res := make([]*Document, 0, len(k.Resources))
-	for _, r := range k.Resources {
+	res := make([]*Document, 0)
+
+	if includeResources {
+		resourceDocs := doc.CollectDocuments(k.Resources)
+		res = append(res, resourceDocs...)
+	}
+
+	if includeGenerators {
+		generatorDocs := doc.CollectDocuments(k.Generators)
+		res = append(res, generatorDocs...)
+	}
+
+	if includeTransformers {
+		transformerDocs := doc.CollectDocuments(k.Transformers)
+		res = append(res, transformerDocs...)
+	}
+
+	return res, nil
+}
+
+// CollectDocuments construct a Document for each path in paths, and return
+// a slice of Document pointers.
+func (doc *KustomizationDocument) CollectDocuments(paths []string) []*Document {
+	docs := make([]*Document, 0, len(paths))
+	for _, r := range paths {
 		if strings.TrimSpace(r) == "" {
 			continue
 		}
 		next, err := doc.Document.FromRelativePath(r)
 		if err != nil {
-			log.Printf("GetResources error: %v\n", err)
+			log.Printf("CollectDocuments error: %v\n", err)
 			continue
 		}
-		res = append(res, &next)
+		docs = append(docs, &next)
 	}
-
-	return res, nil
+	return docs
 }
 
 func (doc *KustomizationDocument) readBytes() ([]map[string]interface{}, error) {

api/internal/crawl/doc/doc_test.go

@@ -189,11 +189,13 @@ metadata:
 	}
 }
 
+type TestStructForGetResources struct {
+	doc       KustomizationDocument
+	resources []*Document
+}
+
 func TestGetResources(t *testing.T) {
-	tests := []struct {
-		doc       KustomizationDocument
-		resources []*Document
-	}{
+	tests := []TestStructForGetResources{
 		{
 			doc: KustomizationDocument{
 				Document: Document{
@@ -248,9 +250,12 @@ resources:
 			resources: []*Document{},
 		},
 	}
+	runTest(t, tests, true, false, false)
+}
 
+func runTest(t *testing.T, tests []TestStructForGetResources, includeResources, includeTransformers, includeGenerators bool) {
 	for _, test := range tests {
-		res, err := test.doc.GetResources()
+		res, err := test.doc.GetResources(includeResources, includeTransformers, includeGenerators)
 		if err != nil {
 			t.Errorf("Unexpected error: %v\n", err)
 			continue
@@ -284,3 +289,73 @@ resources:
 		}
 	}
 }
+
+func TestGetResourcesAndGenerators(t *testing.T) {
+	tests := []TestStructForGetResources{
+		{
+			doc: KustomizationDocument{
+				Document: Document{
+					RepositoryURL: "sigs.k8s.io/kustomize",
+					FilePath:      "some/path/to/kdir/kustomization.yaml",
+					DocumentData: `
+resources:
+- file.yaml
+
+generators:
+- gen.yaml
+
+transformers:
+- tr.yaml
+`},
+			},
+			resources: []*Document{
+				{
+					RepositoryURL: "sigs.k8s.io/kustomize",
+					FilePath:      "some/path/to/kdir/gen.yaml",
+				},
+				{
+					RepositoryURL: "sigs.k8s.io/kustomize",
+					FilePath:      "some/path/to/kdir/file.yaml",
+				},
+			},
+		},
+	}
+	runTest(t, tests, true, false, true)
+}
+
+func TestGetResourcesAndGeneratorsAndTransformers(t *testing.T) {
+	tests := []TestStructForGetResources{
+		{
+			doc: KustomizationDocument{
+				Document: Document{
+					RepositoryURL: "sigs.k8s.io/kustomize",
+					FilePath:      "some/path/to/kdir/kustomization.yaml",
+					DocumentData: `
+resources:
+- file.yaml
+
+generators:
+- gen.yaml
+
+transformers:
+- tr.yaml
+`},
+			},
+			resources: []*Document{
+				{
+					RepositoryURL: "sigs.k8s.io/kustomize",
+					FilePath:      "some/path/to/kdir/tr.yaml",
+				},
+				{
+					RepositoryURL: "sigs.k8s.io/kustomize",
+					FilePath:      "some/path/to/kdir/gen.yaml",
+				},
+				{
+					RepositoryURL: "sigs.k8s.io/kustomize",
+					FilePath:      "some/path/to/kdir/file.yaml",
+				},
+			},
+		},
+	}
+	runTest(t, tests, true, true, true)
+}