diff --git a/api/internal/crawl/crawler/github/crawler.go b/api/internal/crawl/crawler/github/crawler.go index 1696a296a..0ad438169 100644 --- a/api/internal/crawl/crawler/github/crawler.go +++ b/api/internal/crawl/crawler/github/crawler.go @@ -118,9 +118,13 @@ func (gc githubCrawler) FetchDocument(_ context.Context, d *doc.Document) error "/" + repoSpec.Ref + "/" + repoSpec.Path handle := func(resp *http.Response, err error, path string) error { + if resp == nil { + return fmt.Errorf("empty http response (url: %s; path: %s), error: %v", + url, path, err) + } if err == nil && resp.StatusCode == http.StatusOK { d.IsSame = httpclient.FromCache(resp.Header) - defer resp.Body.Close() + defer CloseResponseBody(resp) data, err := ioutil.ReadAll(resp.Body) if err != nil { return err @@ -307,7 +311,10 @@ func (gcl GhClient) GetFileData(k GhFileSpec) ([]byte, error) { return nil, fmt.Errorf("%+v: could not read '%s' metadata: %v", k, url, err) } - resp.Body.Close() + + if err := resp.Body.Close(); err != nil { + return nil, err + } type githubContentRawURL struct { DownloadURL string `json:"download_url,omitempty"` @@ -326,18 +333,24 @@ func (gcl GhClient) GetFileData(k GhFileSpec) ([]byte, error) { k, rawURL.DownloadURL, err) } - defer resp.Body.Close() + defer CloseResponseBody(resp) data, err = ioutil.ReadAll(resp.Body) return data, err } +func CloseResponseBody(resp *http.Response) { + if err := resp.Body.Close(); err != nil { + log.Printf("failed to close response body: %v", err) + } +} + func (gcl GhClient) GetDefaultBranch(url string) (string, error) { resp, err := gcl.GetReposData(url) if err != nil { return "", fmt.Errorf( "'%s' could not get default_branch: %v", url, err) } - defer resp.Body.Close() + defer CloseResponseBody(resp) data, err := ioutil.ReadAll(resp.Body) if err != nil { return "", fmt.Errorf( @@ -389,7 +402,7 @@ func (gcl GhClient) GetFileCreationTime( } } - defer resp.Body.Close() + defer CloseResponseBody(resp) data, err := ioutil.ReadAll(resp.Body) if err != nil { return defaultTime, fmt.Errorf( @@ -512,7 +525,7 @@ func (gcl GhClient) parseGithubResponse(getRequest string) GhResponseInfo { } var data []byte - defer resp.Body.Close() + defer CloseResponseBody(resp) data, requestInfo.Error = ioutil.ReadAll(resp.Body) if requestInfo.Error != nil { return requestInfo @@ -577,7 +590,7 @@ func (gcl GhClient) Do(query string) (*http.Response, error) { // gcl.client.Do: a non-2xx status code doesn't cause an error. // See https://golang.org/pkg/net/http/#Client.Do for more info. resp, err := gcl.client.Do(req) - if resp.StatusCode != http.StatusOK { + if resp != nil && resp.StatusCode != http.StatusOK { err = fmt.Errorf("GhClient.Do(%s) failed with response code: %d", query, resp.StatusCode) } @@ -591,7 +604,7 @@ func (gcl GhClient) getWithRetry( retryCount := gcl.retryCount - for resp.StatusCode == http.StatusForbidden && retryCount > 0 { + for resp != nil && resp.StatusCode == http.StatusForbidden && retryCount > 0 { retryTime := resp.Header.Get("Retry-After") i, errAtoi := strconv.Atoi(retryTime) if errAtoi != nil {