Merge pull request #2001 from haiyanmeng/expose-es

Several fixes of Crawler

api/internal/crawl/cmd/crawler/crawler.go

@@ -136,6 +136,7 @@ func main() {
 				github.QueryWith(
 					github.Filename("kustomization.yaml"),
 					github.Filename("kustomization.yml"),
+					github.Filename("kustomization"),
 					github.User(user)),
 			)
 		} else if repo != "" {
@@ -143,13 +144,15 @@ func main() {
 				github.QueryWith(
 					github.Filename("kustomization.yaml"),
 					github.Filename("kustomization.yml"),
+					github.Filename("kustomization"),
 					github.Repo(repo)),
 			)
 		} else {
 			return github.NewCrawler(githubToken, retryCount, clientCache,
 				github.QueryWith(
 					github.Filename("kustomization.yaml"),
-					github.Filename("kustomization.yml")),
+					github.Filename("kustomization.yml"),
+					github.Filename("kustomization")),
 			)
 		}
 	}

api/internal/crawl/crawler/github/crawler.go

@@ -97,6 +97,17 @@ func (gc githubCrawler) Crawl(
 // it will try to add each string in konfig.RecognizedKustomizationFileNames() to
 // d.FilePath, and try to fetch the document again.
 func (gc githubCrawler) FetchDocument(_ context.Context, d *doc.Document) error {
+	// set the default branch if it is empty
+	if d.DefaultBranch == "" {
+		url := gc.client.ReposRequest(d.RepositoryFullName())
+		defaultBranch, err := gc.client.GetDefaultBranch(url)
+		if err != nil {
+			logger.Printf(
+				"(error: %v) setting default_branch to master\n", err)
+			defaultBranch = "master"
+		}
+		d.DefaultBranch = defaultBranch
+	}
 	repoURL := d.RepositoryURL + "/" + d.FilePath + "?ref=" + d.DefaultBranch
 	repoSpec, err := git.NewRepoSpecFromUrl(repoURL)
 	if err != nil {

api/internal/crawl/crawler/github/queries.go

@@ -116,9 +116,11 @@ type RequestConfig struct {
 // the URL method to get the string value of the URL. See request.CopyWith, to
 // understand why the request object is useful.
 func (rc RequestConfig) CodeSearchRequestWith(query Query) request {
-	req := rc.makeRequest("search/code", query)
-	req.vals.Set("sort", "indexed")
-	req.vals.Set("order", "desc")
+	vals := url.Values{
+		"sort": []string{"indexed"},
+		"order": []string{"desc"},
+	}
+	req := rc.makeRequest("search/code", query, vals)
 	return req
 }
 
@@ -126,27 +128,25 @@ func (rc RequestConfig) CodeSearchRequestWith(query Query) request {
 // query for the Github API to find the dowload information of this filepath.
 func (rc RequestConfig) ContentsRequest(fullRepoName, path string) string {
 	uri := fmt.Sprintf("repos/%s/contents/%s", fullRepoName, path)
-	return rc.makeRequest(uri, Query{}).URL()
+	return rc.makeRequest(uri, Query{}, url.Values{}).URL()
 }
 
 func (rc RequestConfig) ReposRequest(fullRepoName string) string {
 	uri := fmt.Sprintf("repos/%s", fullRepoName)
-	return rc.makeRequest(uri, Query{}).URL()
-}
-
-func escapeSpace(s string) string {
-	return strings.Replace(s, " ", "%20", -1)
+	return rc.makeRequest(uri, Query{}, url.Values{}).URL()
 }
 
 // CommitsRequest given the repo name, and a filepath returns a formatted query
 // for the Github API to find the commits that affect this file.
 func (rc RequestConfig) CommitsRequest(fullRepoName, path string) string {
 	uri := fmt.Sprintf("repos/%s/commits", fullRepoName)
-	return rc.makeRequest(uri, Query{Path(escapeSpace(path))}).URL()
+	vals := url.Values{
+		"path": []string{path},
+	}
+	return rc.makeRequest(uri, Query{}, vals).URL()
 }
 
-func (rc RequestConfig) makeRequest(path string, query Query) request {
-	vals := url.Values{}
+func (rc RequestConfig) makeRequest(path string, query Query, vals url.Values) request {
 	vals.Set(perPageArg, fmt.Sprint(rc.perPage))
 
 	return request{

api/internal/crawl/crawler/github/queries_test.go

@@ -101,7 +101,7 @@ func TestGithubSearchQuery(t *testing.T) {
 				"examples/helloWorld/kustomization.yaml?per_page=100",
 
 			expectedCommitsQuery: "https://api.github.com/repos/kubernetes-sigs/kustomize/commits?" +
-				"q=path:examples/helloWorld/kustomization.yaml&per_page=100",
+				"path=examples%2FhelloWorld%2Fkustomization.yaml&per_page=100",
 		},
 		{
 			rc: RequestConfig{
@@ -121,7 +121,7 @@ func TestGithubSearchQuery(t *testing.T) {
 				"examples%201/helloWorld/kustomization.yaml?per_page=100",
 
 			expectedCommitsQuery: "https://api.github.com/repos/kubernetes-sigs/kustomize/commits?" +
-				"q=path:examples%201/helloWorld/kustomization.yaml&per_page=100",
+				"path=examples+1%2FhelloWorld%2Fkustomization.yaml&per_page=100",
 		},
 	}
 

api/internal/crawl/doc/doc.go

@@ -78,6 +78,9 @@ func (doc *KustomizationDocument) GetResources() ([]*Document, error) {
 
 	res := make([]*Document, 0, len(k.Resources))
 	for _, r := range k.Resources {
+		if strings.TrimSpace(r) == "" {
+			continue
+		}
 		next, err := doc.Document.FromRelativePath(r)
 		if err != nil {
 			fmt.Printf("GetResources error: %v\n", err)

api/internal/crawl/index/elasticsearch.go

@@ -309,9 +309,9 @@ func (idx *index) Exists(id string) (bool, error) {
 		op.WithPretty(),
 	)
 
-	if !res.IsError() {
+	if res != nil && !res.IsError() {
 		return true, nil
-	} else if res.StatusCode == 404 {
+	} else if res != nil && res.StatusCode == 404 {
 		return false, nil
 	} else {
 		return false, idx.responseErrorOrNil(