From ccd129f7a54896ccea1fb904f125f9756be9e154 Mon Sep 17 00:00:00 2001
From: Haiyan Meng <haiyanmeng@google.com>
Date: Wed, 8 Jan 2020 10:24:00 -0800
Subject: [PATCH 1/2] Check empty http response before accessing it

---
 api/internal/crawl/crawler/github/crawler.go | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/api/internal/crawl/crawler/github/crawler.go b/api/internal/crawl/crawler/github/crawler.go
index 1696a296a..6467294da 100644
--- a/api/internal/crawl/crawler/github/crawler.go
+++ b/api/internal/crawl/crawler/github/crawler.go
@@ -118,6 +118,10 @@ func (gc githubCrawler) FetchDocument(_ context.Context, d *doc.Document) error
 		"/" + repoSpec.Ref + "/" + repoSpec.Path
 
 	handle := func(resp *http.Response, err error, path string) error {
+		if resp == nil {
+			return fmt.Errorf("empty http response (url: %s; path: %s), error: %v",
+				url, path, err)
+		}
 		if err == nil && resp.StatusCode == http.StatusOK {
 			d.IsSame = httpclient.FromCache(resp.Header)
 			defer resp.Body.Close()
@@ -577,7 +581,7 @@ func (gcl GhClient) Do(query string) (*http.Response, error) {
 	// gcl.client.Do: a non-2xx status code doesn't cause an error.
 	// See https://golang.org/pkg/net/http/#Client.Do for more info.
 	resp, err :=  gcl.client.Do(req)
-	if resp.StatusCode != http.StatusOK {
+	if resp != nil && resp.StatusCode != http.StatusOK {
 		err = fmt.Errorf("GhClient.Do(%s) failed with response code: %d",
 			query, resp.StatusCode)
 	}
@@ -591,7 +595,7 @@ func (gcl GhClient) getWithRetry(
 
 	retryCount := gcl.retryCount
 
-	for resp.StatusCode == http.StatusForbidden && retryCount > 0 {
+	for resp != nil && resp.StatusCode == http.StatusForbidden && retryCount > 0 {
 		retryTime := resp.Header.Get("Retry-After")
 		i, errAtoi := strconv.Atoi(retryTime)
 		if errAtoi != nil {

From b154af8be4caa53b104fbac3ec6d9a2b53ee6af2 Mon Sep 17 00:00:00 2001
From: Haiyan Meng <haiyanmeng@google.com>
Date: Wed, 8 Jan 2020 10:32:12 -0800
Subject: [PATCH 2/2] Check the error of closing response body

---
 api/internal/crawl/crawler/github/crawler.go | 21 ++++++++++++++------
 1 file changed, 15 insertions(+), 6 deletions(-)

diff --git a/api/internal/crawl/crawler/github/crawler.go b/api/internal/crawl/crawler/github/crawler.go
index 6467294da..0ad438169 100644
--- a/api/internal/crawl/crawler/github/crawler.go
+++ b/api/internal/crawl/crawler/github/crawler.go
@@ -124,7 +124,7 @@ func (gc githubCrawler) FetchDocument(_ context.Context, d *doc.Document) error
 		}
 		if err == nil && resp.StatusCode == http.StatusOK {
 			d.IsSame = httpclient.FromCache(resp.Header)
-			defer resp.Body.Close()
+			defer CloseResponseBody(resp)
 			data, err := ioutil.ReadAll(resp.Body)
 			if err != nil {
 				return err
@@ -311,7 +311,10 @@ func (gcl GhClient) GetFileData(k GhFileSpec) ([]byte, error) {
 		return nil, fmt.Errorf("%+v: could not read '%s' metadata: %v",
 			k, url, err)
 	}
-	resp.Body.Close()
+
+	if err := resp.Body.Close(); err != nil {
+		return nil, err
+	}
 
 	type githubContentRawURL struct {
 		DownloadURL string `json:"download_url,omitempty"`
@@ -330,18 +333,24 @@ func (gcl GhClient) GetFileData(k GhFileSpec) ([]byte, error) {
 			k, rawURL.DownloadURL, err)
 	}
 
-	defer resp.Body.Close()
+	defer CloseResponseBody(resp)
 	data, err = ioutil.ReadAll(resp.Body)
 	return data, err
 }
 
+func CloseResponseBody(resp *http.Response) {
+	if err := resp.Body.Close(); err != nil {
+		log.Printf("failed to close response body: %v", err)
+	}
+}
+
 func (gcl GhClient) GetDefaultBranch(url string) (string, error) {
 	resp, err := gcl.GetReposData(url)
 	if err != nil {
 		return "", fmt.Errorf(
 			"'%s' could not get default_branch: %v", url, err)
 	}
-	defer resp.Body.Close()
+	defer CloseResponseBody(resp)
 	data, err := ioutil.ReadAll(resp.Body)
 	if err != nil {
 		return "", fmt.Errorf(
@@ -393,7 +402,7 @@ func (gcl GhClient) GetFileCreationTime(
 		}
 	}
 
-	defer resp.Body.Close()
+	defer CloseResponseBody(resp)
 	data, err := ioutil.ReadAll(resp.Body)
 	if err != nil {
 		return defaultTime, fmt.Errorf(
@@ -516,7 +525,7 @@ func (gcl GhClient) parseGithubResponse(getRequest string) GhResponseInfo {
 	}
 
 	var data []byte
-	defer resp.Body.Close()
+	defer CloseResponseBody(resp)
 	data, requestInfo.Error = ioutil.ReadAll(resp.Body)
 	if requestInfo.Error != nil {
 		return requestInfo