kubernetes-sigs/kustomize
1
0
Fork 0
mirror of https://github.com/kubernetes-sigs/kustomize.git synced 2026-06-13 01:50:55 +00:00
Code Issues Packages Projects Releases 1 Wiki Activity

Retry FindRangesForRepoSearch

Browse Source
This commit is contained in:
Haiyan Meng
2020-01-22 10:13:57 -08:00
parent 1120c6bc7a
commit 0820865e1d
1 changed files with 11 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
api/internal/crawl/crawler/github/crawler.go
View File

@@ -87,10 +87,20 @@ func (gc githubCrawler) Crawl(ctx context.Context,
accessToken: gc.client.accessToken,
}
var ranges []string
var err error
// Since Github returns a max of 1000 results per query, we can use
// multiple queries that split the search space into chunks of at most
// 1000 files to get all of the data.
ranges, err := FindRangesForRepoSearch(newCache(noETagClient, gc.query))
for i := 0; i < 5; i++ {
ranges, err = FindRangesForRepoSearch(newCache(noETagClient, gc.query))
if err == nil {
logger.Printf("FindRangesForRepoSearch succeeded after %d retries", i)
break
} else {
time.Sleep(time.Minute)
}
}
if err != nil {
return fmt.Errorf("could not split %v into ranges, %v\n",
gc.query, err)