kubernetes-sigs/kustomize
1
0
Fork 0
mirror of https://github.com/kubernetes-sigs/kustomize.git synced 2026-06-29 17:41:13 +00:00
Code Issues Packages Projects Releases 1 Wiki Activity

Check the incomplete_results field of github query responses

Browse Source 
Currently, we don't check the `incomplete_results` field of a github
query response, which is problematic when incomplete query results are
used to split the query ranges: the splitted query ranges will
be very wild.
This commit is contained in:
Haiyan Meng
2020-01-30 11:12:21 -08:00
parent 7a87c84403
commit f5419e9f72
2 changed files with 19 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
api/internal/crawl/crawler/github/split_search_ranges.go
View File

@@ -172,7 +172,7 @@ func (c githubCachedSearch) CountResults(lowerBound, upperBound uint64) (uint64,
sizeRange := RangeWithin{lowerBound, upperBound}
rangeRequest := c.RequestString(sizeRange)
result := c.gcl.parseGithubResponse(rangeRequest)
result := c.gcl.parseGithubResponseWithRetry(rangeRequest)
if result.Error != nil {
return count, result.Error
}
@@ -206,7 +206,7 @@ func (c githubCachedSearch) CountResults(lowerBound, upperBound uint64) (uint64,
"Retrying query... current lower bound: %d, got: %d\n",
c.cache[prev], result.Parsed.TotalCount)
result = c.gcl.parseGithubResponse(rangeRequest)
result = c.gcl.parseGithubResponseWithRetry(rangeRequest)
if result.Error != nil {
return count, result.Error
}
@@ -221,8 +221,8 @@ func (c githubCachedSearch) CountResults(lowerBound, upperBound uint64) (uint64,
}
count = result.Parsed.TotalCount
logger.Printf("Caching new query %s, with count %d\n",
sizeRange.RangeString(), count)
logger.Printf("Caching new query %s, with count %d (incomplete_results: %v)\n",
sizeRange.RangeString(), count, result.Parsed.IncompleteResults)
c.cache[upperBound] = count
return count, nil
}