kubernetes-sigs/kustomize
1
0
Fork 0
mirror of https://github.com/kubernetes-sigs/kustomize.git synced 2026-06-11 17:12:51 +00:00
Code Issues Packages Projects Releases 1 Wiki Activity

Merge pull request #2051 from haiyanmeng/nil

Browse Source 
Two fixes of the crawler
This commit is contained in:
Jeff Regan
2020-01-08 18:39:26 -08:00
committed by GitHub
parent 1404d2749d b154af8be4
commit a46046dac5
1 changed files with 21 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

29
api/internal/crawl/crawler/github/crawler.go
View File

@@ -118,9 +118,13 @@ func (gc githubCrawler) FetchDocument(_ context.Context, d *doc.Document) error
"/" + repoSpec.Ref + "/" + repoSpec.Path "/" + repoSpec.Ref + "/" + repoSpec.Path
handle := func(resp *http.Response, err error, path string) error { handle := func(resp *http.Response, err error, path string) error {
if resp == nil {
return fmt.Errorf("empty http response (url: %s; path: %s), error: %v",
url, path, err)
}
if err == nil && resp.StatusCode == http.StatusOK { if err == nil && resp.StatusCode == http.StatusOK {
d.IsSame = httpclient.FromCache(resp.Header) d.IsSame = httpclient.FromCache(resp.Header)
defer resp.Body.Close() defer CloseResponseBody(resp)
data, err := ioutil.ReadAll(resp.Body) data, err := ioutil.ReadAll(resp.Body)
if err != nil { if err != nil {
return err return err
@@ -307,7 +311,10 @@ func (gcl GhClient) GetFileData(k GhFileSpec) ([]byte, error) {
return nil, fmt.Errorf("%+v: could not read '%s' metadata: %v", return nil, fmt.Errorf("%+v: could not read '%s' metadata: %v",
k, url, err) k, url, err)
} }
resp.Body.Close()
if err := resp.Body.Close(); err != nil {
return nil, err
}
type githubContentRawURL struct { type githubContentRawURL struct {
DownloadURL string `json:"download_url,omitempty"` DownloadURL string `json:"download_url,omitempty"`
@@ -326,18 +333,24 @@ func (gcl GhClient) GetFileData(k GhFileSpec) ([]byte, error) {
k, rawURL.DownloadURL, err) k, rawURL.DownloadURL, err)
} }
defer resp.Body.Close() defer CloseResponseBody(resp)
data, err = ioutil.ReadAll(resp.Body) data, err = ioutil.ReadAll(resp.Body)
return data, err return data, err
} }
func CloseResponseBody(resp *http.Response) {
if err := resp.Body.Close(); err != nil {
log.Printf("failed to close response body: %v", err)
}
}
func (gcl GhClient) GetDefaultBranch(url string) (string, error) { func (gcl GhClient) GetDefaultBranch(url string) (string, error) {
resp, err := gcl.GetReposData(url) resp, err := gcl.GetReposData(url)
if err != nil { if err != nil {
return "", fmt.Errorf( return "", fmt.Errorf(
"'%s' could not get default_branch: %v", url, err) "'%s' could not get default_branch: %v", url, err)
} }
defer resp.Body.Close() defer CloseResponseBody(resp)
data, err := ioutil.ReadAll(resp.Body) data, err := ioutil.ReadAll(resp.Body)
if err != nil { if err != nil {
return "", fmt.Errorf( return "", fmt.Errorf(
@@ -389,7 +402,7 @@ func (gcl GhClient) GetFileCreationTime(
} }
} }
defer resp.Body.Close() defer CloseResponseBody(resp)
data, err := ioutil.ReadAll(resp.Body) data, err := ioutil.ReadAll(resp.Body)
if err != nil { if err != nil {
return defaultTime, fmt.Errorf( return defaultTime, fmt.Errorf(
@@ -512,7 +525,7 @@ func (gcl GhClient) parseGithubResponse(getRequest string) GhResponseInfo {
} }
var data []byte var data []byte
defer resp.Body.Close() defer CloseResponseBody(resp)
data, requestInfo.Error = ioutil.ReadAll(resp.Body) data, requestInfo.Error = ioutil.ReadAll(resp.Body)
if requestInfo.Error != nil { if requestInfo.Error != nil {
return requestInfo return requestInfo
@@ -577,7 +590,7 @@ func (gcl GhClient) Do(query string) (*http.Response, error) {
// gcl.client.Do: a non-2xx status code doesn't cause an error. // gcl.client.Do: a non-2xx status code doesn't cause an error.
// See https://golang.org/pkg/net/http/#Client.Do for more info. // See https://golang.org/pkg/net/http/#Client.Do for more info.
resp, err := gcl.client.Do(req) resp, err := gcl.client.Do(req)
if resp.StatusCode != http.StatusOK { if resp != nil && resp.StatusCode != http.StatusOK {
err = fmt.Errorf("GhClient.Do(%s) failed with response code: %d", err = fmt.Errorf("GhClient.Do(%s) failed with response code: %d",
query, resp.StatusCode) query, resp.StatusCode)
} }
@@ -591,7 +604,7 @@ func (gcl GhClient) getWithRetry(
retryCount := gcl.retryCount retryCount := gcl.retryCount
for resp.StatusCode == http.StatusForbidden && retryCount > 0 { for resp != nil && resp.StatusCode == http.StatusForbidden && retryCount > 0 {
retryTime := resp.Header.Get("Retry-After") retryTime := resp.Header.Get("Retry-After")
i, errAtoi := strconv.Atoi(retryTime) i, errAtoi := strconv.Atoi(retryTime)
if errAtoi != nil { if errAtoi != nil {