From 0820865e1d6e756ed452e1af12abe3881529b023 Mon Sep 17 00:00:00 2001 From: Haiyan Meng Date: Wed, 22 Jan 2020 10:13:57 -0800 Subject: [PATCH] Retry FindRangesForRepoSearch --- api/internal/crawl/crawler/github/crawler.go | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/api/internal/crawl/crawler/github/crawler.go b/api/internal/crawl/crawler/github/crawler.go index f1b723c5b..21cfd6f81 100644 --- a/api/internal/crawl/crawler/github/crawler.go +++ b/api/internal/crawl/crawler/github/crawler.go @@ -87,10 +87,20 @@ func (gc githubCrawler) Crawl(ctx context.Context, accessToken: gc.client.accessToken, } + var ranges []string + var err error // Since Github returns a max of 1000 results per query, we can use // multiple queries that split the search space into chunks of at most // 1000 files to get all of the data. - ranges, err := FindRangesForRepoSearch(newCache(noETagClient, gc.query)) + for i := 0; i < 5; i++ { + ranges, err = FindRangesForRepoSearch(newCache(noETagClient, gc.query)) + if err == nil { + logger.Printf("FindRangesForRepoSearch succeeded after %d retries", i) + break + } else { + time.Sleep(time.Minute) + } + } if err != nil { return fmt.Errorf("could not split %v into ranges, %v\n", gc.query, err)