From f4636f85551079c03d584475928f3ffb4dc28219 Mon Sep 17 00:00:00 2001 From: Haiyan Meng Date: Thu, 16 Jan 2020 11:28:41 -0800 Subject: [PATCH] Add a `fileType` field into the index --- api/internal/crawl/cmd/crawler/crawler.go | 2 +- api/internal/crawl/crawler/crawler.go | 68 ++++++---- api/internal/crawl/crawler/crawler_test.go | 2 + api/internal/crawl/crawler/github/crawler.go | 27 ++-- api/internal/crawl/doc/doc.go | 10 +- api/internal/crawl/doc/doc_test.go | 9 ++ api/internal/crawl/doc/docname.go | 4 + api/internal/crawl/doc/unique_doc.go | 2 +- api/internal/crawl/index/elasticsearch.go | 3 + api/internal/crawl/search_cmds/fileType.md | 123 +++++++++++++++++++ api/internal/crawl/search_cmds/misc.md | 13 ++ api/internal/crawl/utils/utils.go | 13 +- 12 files changed, 224 insertions(+), 52 deletions(-) create mode 100644 api/internal/crawl/search_cmds/fileType.md diff --git a/api/internal/crawl/cmd/crawler/crawler.go b/api/internal/crawl/cmd/crawler/crawler.go index 2d301efe1..54bf3f432 100644 --- a/api/internal/crawl/cmd/crawler/crawler.go +++ b/api/internal/crawl/cmd/crawler/crawler.go @@ -126,7 +126,7 @@ func main() { } } - // seen tracks the IDs of all the documents in the index. + // seen tracks the IDs of all the documents in the index and their corresponding file types. // This helps avoid indexing a given document multiple times. seen := utils.NewSeenMap() diff --git a/api/internal/crawl/crawler/crawler.go b/api/internal/crawl/crawler/crawler.go index 11715616d..7861749f0 100644 --- a/api/internal/crawl/crawler/crawler.go +++ b/api/internal/crawl/crawler/crawler.go @@ -38,6 +38,8 @@ type Crawler interface { // Write to the document what the created time is. SetCreated(context.Context, *doc.Document) error + SetDefaultBranch(*doc.Document) + Match(*doc.Document) bool } @@ -78,7 +80,7 @@ func findMatch(d *doc.Document, crawlers []Crawler) Crawler { func addBranches(cdoc CrawledDocument, match Crawler, indx IndexFunc, seen utils.SeenMap, stack *CrawlSeed) { - seen.Add(cdoc.ID()) + seen.Set(cdoc.ID(), cdoc.GetDocument().FileType) // Insert into index if err := indx(cdoc, index.InsertOrUpdate); err != nil { @@ -87,14 +89,14 @@ func addBranches(cdoc CrawledDocument, match Crawler, indx IndexFunc, return } - deps, err := cdoc.GetResources(true, false, false) + deps, err := cdoc.GetResources(true, true, true) if err != nil { logger.Println(err) return } for _, dep := range deps { - if seen.Seen(dep.ID()) { + if seen.Seen(dep.ID()) && seen.Value(dep.ID()) == dep.FileType { continue } *stack = append(*stack, dep) @@ -102,7 +104,7 @@ func addBranches(cdoc CrawledDocument, match Crawler, indx IndexFunc, } func doCrawl(ctx context.Context, docsPtr *CrawlSeed, crawlers []Crawler, conv Converter, indx IndexFunc, - seen utils.SeenMap, stack *CrawlSeed) { + seen utils.SeenMap, stack *CrawlSeed, refreshDoc bool, updateFileType bool) { UpdatedDocCount := 0 seenDocCount := 0 @@ -126,9 +128,11 @@ func doCrawl(ctx context.Context, docsPtr *CrawlSeed, crawlers []Crawler, conv C logger.Printf("Crawling doc %d: %s", crawledDocCount, tail.Path()) if seen.Seen(tail.ID()) { - logger.Printf("this doc has been seen before") - seenDocCount++ - continue + if !updateFileType || seen.Value(tail.ID()) == tail.FileType { + logger.Printf("this doc has been seen before") + seenDocCount++ + continue + } } if tail.WasCached() { @@ -151,26 +155,34 @@ func doCrawl(ctx context.Context, docsPtr *CrawlSeed, crawlers []Crawler, conv C // calling FetchDocument. Otherwise, the binary may enter into an infinite loop // if a kustomization file points to its kustmozation root in its `resources` or // `bases` field. - seen.Add(tail.ID()) + seen.Set(tail.ID(), tail.FileType) - if err := match.FetchDocument(ctx, tail); err != nil { - logger.Printf("FetchDocument failed on doc(%s): %v", tail.Path(), err) - FetchDocumentErrCount++ - // delete the document from the index - cdoc := &doc.KustomizationDocument{ - Document: *tail, - } - seen.Add(cdoc.ID()) - if err := indx(cdoc, index.Delete); err != nil { - logger.Printf("Failed to delete doc(%s): %v", cdoc.Path(), err) - } - deleteDocCount++ - continue + if refreshDoc || tail.DefaultBranch == "" { + match.SetDefaultBranch(tail) } - if err := match.SetCreated(ctx, tail); err != nil { - logger.Printf("SetCreated failed on doc(%s): %v", tail.Path(), err) - SetCreatedErrCount++ + if refreshDoc || tail.DocumentData == "" { + if err := match.FetchDocument(ctx, tail); err != nil { + logger.Printf("FetchDocument failed on doc(%s): %v", tail.Path(), err) + FetchDocumentErrCount++ + // delete the document from the index + cdoc := &doc.KustomizationDocument{ + Document: *tail, + } + seen.Set(cdoc.ID(), tail.FileType) + if err := indx(cdoc, index.Delete); err != nil { + logger.Printf("Failed to delete doc(%s): %v", cdoc.Path(), err) + } + deleteDocCount++ + continue + } + } + + if refreshDoc || tail.CreationTime == nil { + if err := match.SetCreated(ctx, tail); err != nil { + logger.Printf("SetCreated failed on doc(%s): %v", tail.Path(), err) + SetCreatedErrCount++ + } } cdoc, err := conv(tail) @@ -206,14 +218,14 @@ func CrawlFromSeed(ctx context.Context, seed CrawlSeed, crawlers []Crawler, // Exploit seed to update bulk of corpus. logger.Printf("updating %d documents from seed\n", len(seed)) // each unique document in seed will be crawled once. - doCrawl(ctx, &seed, crawlers, conv, indx, seen, &stack) + doCrawl(ctx, &seed, crawlers, conv, indx, seen, &stack, true, false) // Traverse any new documents added while updating corpus. logger.Printf("crawling %d new documents found in the seed\n", len(stack)) // While crawling each document in stack, the documents directly referred in the document // will be added into stack. // After this statement is done, stack will become empty. - doCrawl(ctx, &stack, crawlers, conv, indx, seen, &stack) + doCrawl(ctx, &stack, crawlers, conv, indx, seen, &stack, false, true) } // CrawlGithubRunner is a blocking function and only returns once all of the @@ -294,6 +306,8 @@ func CrawlGithub(ctx context.Context, crawlers []Crawler, conv Converter, for cdoc := range ch { docCount++ logger.Printf("Processing doc %d found on Github", docCount) + // all the docs here are kustomization files found by querying Github, and + // their `FileType` fields all should be empty. if seen.Seen(cdoc.ID()) { logger.Printf("the doc has been seen before") continue @@ -320,5 +334,5 @@ func CrawlGithub(ctx context.Context, crawlers []Crawler, conv Converter, // Handle deps of newly discovered documents. logger.Printf("crawling the %d new documents referred by other documents", len(stack)) - doCrawl(ctx, &stack, crawlers, conv, indx, seen, &stack) + doCrawl(ctx, &stack, crawlers, conv, indx, seen, &stack, false, true) } diff --git a/api/internal/crawl/crawler/crawler_test.go b/api/internal/crawl/crawler/crawler_test.go index d18a4afce..5e93d5bcb 100644 --- a/api/internal/crawl/crawler/crawler_test.go +++ b/api/internal/crawl/crawler/crawler_test.go @@ -37,6 +37,8 @@ func (c testCrawler) Match(d *doc.Document) bool { return d != nil } +func (c testCrawler) SetDefaultBranch(d *doc.Document) {} + func (c testCrawler) FetchDocument(_ context.Context, d *doc.Document) error { if i, ok := c.lukp[d.ID()]; ok { d.DocumentData = c.docs[i].DocumentData diff --git a/api/internal/crawl/crawler/github/crawler.go b/api/internal/crawl/crawler/github/crawler.go index e1aeff401..70c1a23ca 100644 --- a/api/internal/crawl/crawler/github/crawler.go +++ b/api/internal/crawl/crawler/github/crawler.go @@ -60,8 +60,16 @@ func NewCrawler(accessToken string, retryCount uint64, client *http.Client, } } -func (gc githubCrawler) SetDefaultBranch(repo, branch string) { - gc.branchMap[repo] = branch +func (gc githubCrawler) SetDefaultBranch(d *doc.Document) { + url := gc.client.ReposRequest(d.RepositoryFullName()) + defaultBranch, err := gc.client.GetDefaultBranch(url, d.RepositoryURL, gc.branchMap) + if err != nil { + logger.Printf( + "(error: %v) setting default_branch to master\n", err) + defaultBranch = "master" + } + d.DefaultBranch = defaultBranch + gc.branchMap[d.RepositoryURL] = d.DefaultBranch } func (gc githubCrawler) DefaultBranch(repo string) string { @@ -114,19 +122,6 @@ func (gc githubCrawler) Crawl(ctx context.Context, // it will try to add each string in konfig.RecognizedKustomizationFileNames() to // d.FilePath, and try to fetch the document again. func (gc githubCrawler) FetchDocument(_ context.Context, d *doc.Document) error { - // set the default branch if it is empty - if d.DefaultBranch == "" { - url := gc.client.ReposRequest(d.RepositoryFullName()) - defaultBranch, err := gc.client.GetDefaultBranch(url, d.RepositoryURL, gc.branchMap) - if err != nil { - logger.Printf( - "(error: %v) setting default_branch to master\n", err) - defaultBranch = "master" - } - d.DefaultBranch = defaultBranch - } - gc.SetDefaultBranch(d.RepositoryURL, d.DefaultBranch) - repoURL := d.RepositoryURL + "/" + d.FilePath + "?ref=" + d.DefaultBranch repoSpec, err := git.NewRepoSpecFromUrl(repoURL) if err != nil { @@ -283,6 +278,8 @@ func kustomizationResultAdapter(gcl GhClient, k GhFileSpec, seen utils.SeenMap, defaultBranch = "master" } + // document here is a kustomization file found by querying Github, whose + // `FileType` field should be empty. document := doc.Document{ FilePath: k.Path, DefaultBranch: defaultBranch, diff --git a/api/internal/crawl/doc/doc.go b/api/internal/crawl/doc/doc.go index 87cd5a3e2..0e5965ac1 100644 --- a/api/internal/crawl/doc/doc.go +++ b/api/internal/crawl/doc/doc.go @@ -87,17 +87,17 @@ func (doc *KustomizationDocument) GetResources( res := make([]*Document, 0) if includeResources { - resourceDocs := doc.CollectDocuments(k.Resources) + resourceDocs := doc.CollectDocuments(k.Resources, "resource") res = append(res, resourceDocs...) } if includeGenerators { - generatorDocs := doc.CollectDocuments(k.Generators) + generatorDocs := doc.CollectDocuments(k.Generators, "generator") res = append(res, generatorDocs...) } if includeTransformers { - transformerDocs := doc.CollectDocuments(k.Transformers) + transformerDocs := doc.CollectDocuments(k.Transformers, "transformer") res = append(res, transformerDocs...) } @@ -106,7 +106,8 @@ func (doc *KustomizationDocument) GetResources( // CollectDocuments construct a Document for each path in paths, and return // a slice of Document pointers. -func (doc *KustomizationDocument) CollectDocuments(paths []string) []*Document { +func (doc *KustomizationDocument) CollectDocuments( + paths []string, fileType string) []*Document { docs := make([]*Document, 0, len(paths)) for _, r := range paths { if strings.TrimSpace(r) == "" { @@ -117,6 +118,7 @@ func (doc *KustomizationDocument) CollectDocuments(paths []string) []*Document { log.Printf("CollectDocuments error: %v\n", err) continue } + next.FileType = fileType docs = append(docs, &next) } return docs diff --git a/api/internal/crawl/doc/doc_test.go b/api/internal/crawl/doc/doc_test.go index c193809a6..e66241287 100644 --- a/api/internal/crawl/doc/doc_test.go +++ b/api/internal/crawl/doc/doc_test.go @@ -215,19 +215,23 @@ resources: { RepositoryURL: "sigs.k8s.io/kustomize", FilePath: "some/path/to/base", + FileType: "resource", }, { RepositoryURL: "sigs.k8s.io/kustomize", FilePath: "some/path/to/otherbase", + FileType: "resource", }, { RepositoryURL: "sigs.k8s.io/kustomize", FilePath: "some/path/to/kdir/file.yaml", + FileType: "resource", }, { RepositoryURL: "https://github.com/kubernetes-sigs/kustomize", FilePath: "examples/helloWorld", DefaultBranch: "v3.1.0", + FileType: "resource", }, }, }, @@ -312,10 +316,12 @@ transformers: { RepositoryURL: "sigs.k8s.io/kustomize", FilePath: "some/path/to/kdir/gen.yaml", + FileType: "generator", }, { RepositoryURL: "sigs.k8s.io/kustomize", FilePath: "some/path/to/kdir/file.yaml", + FileType: "resource", }, }, }, @@ -345,14 +351,17 @@ transformers: { RepositoryURL: "sigs.k8s.io/kustomize", FilePath: "some/path/to/kdir/tr.yaml", + FileType: "transformer", }, { RepositoryURL: "sigs.k8s.io/kustomize", FilePath: "some/path/to/kdir/gen.yaml", + FileType: "generator", }, { RepositoryURL: "sigs.k8s.io/kustomize", FilePath: "some/path/to/kdir/file.yaml", + FileType: "resource", }, }, }, diff --git a/api/internal/crawl/doc/docname.go b/api/internal/crawl/doc/docname.go index e295e4620..a3cab942a 100644 --- a/api/internal/crawl/doc/docname.go +++ b/api/internal/crawl/doc/docname.go @@ -17,6 +17,9 @@ type Document struct { DocumentData string `json:"document,omitempty"` CreationTime *time.Time `json:"creationTime,omitempty"` IsSame bool `json:"-"` + // FileType can be one of the following: + // "generator", "transformer", "resource", "". + FileType string `json:"fileType,omitempty"` } // Implements the CrawlerDocument interface. @@ -32,6 +35,7 @@ func (doc *Document) Copy() *Document { DocumentData: doc.DocumentData, CreationTime: doc.CreationTime, IsSame: doc.IsSame, + FileType: doc.FileType, } } diff --git a/api/internal/crawl/doc/unique_doc.go b/api/internal/crawl/doc/unique_doc.go index 026b345a5..da5294b1f 100644 --- a/api/internal/crawl/doc/unique_doc.go +++ b/api/internal/crawl/doc/unique_doc.go @@ -22,7 +22,7 @@ func (uds *UniqueDocuments) Add(d *Document) { return } uds.docs = append(uds.docs, d) - uds.docIDs.Add(d.ID()) + uds.docIDs.Set(d.ID(), "") } func (uds *UniqueDocuments) AddDocuments(docs []*Document) { diff --git a/api/internal/crawl/index/elasticsearch.go b/api/internal/crawl/index/elasticsearch.go index 7d7ce2b9e..ebd6b70c0 100644 --- a/api/internal/crawl/index/elasticsearch.go +++ b/api/internal/crawl/index/elasticsearch.go @@ -26,6 +26,9 @@ const IndexConfig = ` "defaultBranch": { "type": "keyword" }, + "fileType": { + "type": "keyword" + }, "document": { "type": "text" }, diff --git a/api/internal/crawl/search_cmds/fileType.md b/api/internal/crawl/search_cmds/fileType.md new file mode 100644 index 000000000..ced952f3e --- /dev/null +++ b/api/internal/crawl/search_cmds/fileType.md @@ -0,0 +1,123 @@ +Find all the documents having the `fileType` field set: +``` +curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d' +{ + "query": { + "exists": { + "field": "fileType" + } + } +} +' +``` + +Find all the documents whose `fileType` field is not set: +``` +curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d' +{ + "size": 10000, + "query": { + "bool": { + "must_not": { + "exists": { + "field": "fileType" + } + } + } + } +} +' +``` + +Search for all the documents whose `fileType` field is `resource`: +``` +curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d' +{ + "query": { + "bool": { + "filter": [ + { "regexp": { "fileType": "resource" }} + ] + } + } +} +' +``` + +Count distinct values of the `fileType` field: +``` +curl -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' +{ + "aggs" : { + "fileType_count" : { + "cardinality" : { + "field" : "fileType", + "precision_threshold": 40000 + } + } + } +} +' +``` + +List all the values of the `fileType` field and the frequency of each value: +``` +curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' +{ + "aggs" : { + "fileType" : { + "terms" : { + "field" : "fileType" + } + } + } +} +' +``` + + +For all the kustomization files in the index, list all the values of the +`fileType` field and the frequency of each value: +``` +curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' +{ + "query": { + "bool": { + "filter": [ + { "regexp": { "filePath": ".*/kustomization((.yaml)?|(.yml)?)/*" }} + ] + } + }, + "aggs" : { + "fileType" : { + "terms" : { + "field" : "fileType" + } + } + } +} +' +``` + +For all the non-kustomization files in the index, list all the values of the +`fileType` field and the frequency of each value: +``` +curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' +{ + "query": { + "bool": { + "must_not": { + "regexp": { "filePath": ".*/kustomization((.yaml)?|(.yml)?)/*" } + } + } + }, + "aggs" : { + "fileType" : { + "terms" : { + "field" : "fileType" + } + } + } +} +' +``` diff --git a/api/internal/crawl/search_cmds/misc.md b/api/internal/crawl/search_cmds/misc.md index 303ae5d3b..d68736ea2 100644 --- a/api/internal/crawl/search_cmds/misc.md +++ b/api/internal/crawl/search_cmds/misc.md @@ -16,4 +16,17 @@ curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_mapping?pretty" Delete the kustomize index from the ElasticSearch cluster (**Use this command with caution**): ``` curl -X DELETE "${ElasticSearchURL}:9200/${INDEXNAME}?pretty" +``` + +Add a new field into an existing index. +``` +curl -X PUT "${ElasticSearchURL}:9200/${INDEXNAME}/_mapping/_doc?pretty" -H 'Content-Type: application/json' -d' +{ + "properties": { + "fileType": { + "type": "keyword" + } + } +} +' ``` \ No newline at end of file diff --git a/api/internal/crawl/utils/utils.go b/api/internal/crawl/utils/utils.go index a397b2d52..d6b6fab68 100644 --- a/api/internal/crawl/utils/utils.go +++ b/api/internal/crawl/utils/utils.go @@ -1,16 +1,21 @@ package utils -type SeenMap map[string]struct{} +type SeenMap map[string]string func (seen SeenMap) Seen(item string) bool { _, ok := seen[item] return ok } -func (seen SeenMap) Add(item string) { - seen[item] = struct{}{} +func (seen SeenMap) Set(k, v string) { + seen[k] = v +} + +// The caller should make sure that key is in the map. +func (seen SeenMap) Value(k string) string { + return seen[k] } func NewSeenMap() SeenMap { - return make(map[string]struct{}) + return make(map[string]string) }