Mulitple improvements of the crawler

1) Set document IDs to avoid duplicating documents;
2) Set the `creationTime` field of each document in the index;
3) set the `values`, `kinds` and `identifiers` fields for all documents;
4) Add a `Copy` method into the `Document` struct: this fixes the issue
where all the documents existing in the index point to the same Document
object;
5) Avoid using keystore redis;
6) Set imagePullPolicy to `Always` for crawler jobs.

api/internal/crawl/crawler/crawler.go

@@ -102,11 +102,9 @@ func CrawlFromSeed(ctx context.Context, seed CrawlSeed,
 	}
 
 	doCrawl := func(docsPtr *CrawlSeed) {
-		for len(*docsPtr) > 0 {
-			back := len(*docsPtr) - 1
-			next := (*docsPtr)[back]
-			*docsPtr = (*docsPtr)[:back]
-
+		n := len(*docsPtr)
+		for i := 0; i < n; i++ {
+			next := (*docsPtr)[i]
 			match := findMatch(next)
 			if match == nil {
 				logIfErr(fmt.Errorf(
@@ -114,24 +112,28 @@ func CrawlFromSeed(ctx context.Context, seed CrawlSeed,
 				continue
 			}
 
+			logger.Println("Crawling ", next.RepositoryURL, next.FilePath)
 			err := match.FetchDocument(ctx, next)
 			logIfErr(err)
 			// If there was no change or there is an error, we don't have
 			// to branch out, since the dependencies are already in the
 			// index, or we cannot find the document.
 			if err != nil || next.WasCached() {
+				if next.WasCached() {
+					logger.Println(next.RepositoryURL, next.FilePath, "is cached already")
+				}
 				continue
 			}
 
+			logIfErr(match.SetCreated(ctx, next))
+
 			cdoc, err := conv(next)
 			logIfErr(err)
-			if err != nil {
-				continue
-			}
 
 			addBranches(cdoc, match)
 		}
 	}
+
 	// Exploit seed to update bulk of corpus.
 	logger.Printf("updating %d documents from seed\n", len(seed))
 	doCrawl(&seed)

api/internal/crawl/crawler/crawler_test.go

@@ -29,7 +29,7 @@ type testCrawler struct {
 }
 
 func (c testCrawler) Match(d *doc.Document) bool {
-	return d != nil && strings.HasPrefix(d.ID(), c.matchPrefix)
+	return d != nil
 }
 
 func (c testCrawler) FetchDocument(_ context.Context, d *doc.Document) error {

api/internal/crawl/crawler/github/crawler.go

@@ -133,8 +133,12 @@ func (gc githubCrawler) FetchDocument(_ context.Context, d *doc.Document) error
 }
 
 func (gc githubCrawler) SetCreated(_ context.Context, d *doc.Document) error {
-	fs := GhFileSpec{}
-	fs.Repository.FullName = d.RepositoryURL + "/" + d.FilePath
+	fs := GhFileSpec{
+		Path: d.FilePath,
+		Repository: GitRepository{
+			FullName: d.RepositoryFullName(),
+		},
+	}
 	creationTime, err := gc.client.GetFileCreationTime(fs)
 	if err != nil {
 		return err
@@ -185,9 +189,9 @@ func processQuery(ctx context.Context, gcl GhClient, query string,
 		for _, file := range page.Parsed.Items {
 			k, err := kustomizationResultAdapter(gcl, file)
 			if err != nil {
+				logger.Printf("kustomizationResultAdapter failed: %v", err)
 				errs = append(errs, err)
 				errorCnt++
-				continue
 			}
 			output <- k
 			totalCnt++
@@ -224,6 +228,18 @@ func kustomizationResultAdapter(gcl GhClient, k GhFileSpec) (
 			RepositoryURL: k.Repository.URL,
 		},
 	}
+	logger.Printf("Set the creationTime field")
+	creationTime, err := gcl.GetFileCreationTime(k)
+	if err != nil {
+		logger.Printf("GetFileCreationTime failed: %v", err)
+		return &d, err
+	}
+	d.CreationTime = &creationTime
+
+	if err := d.ParseYAML(); err != nil {
+		logger.Printf("ParseYAML failed: %v", err)
+		return &d, err
+	}
 
 	return &d, nil
 }
@@ -410,13 +426,15 @@ func (e multiError) Error() string {
 	return strings.Join(strs, "\n")
 }
 
+type GitRepository struct {
+	API      string `json:"url,omitempty"`
+	URL      string `json:"html_url,omitempty"`
+	FullName string `json:"full_name,omitempty"`
+}
+
 type GhFileSpec struct {
-	Path       string `json:"path,omitempty"`
-	Repository struct {
-		API      string `json:"url,omitempty"`
-		URL      string `json:"html_url,omitempty"`
-		FullName string `json:"full_name,omitempty"`
-	} `json:"repository,omitempty"`
+	Path       string        `json:"path,omitempty"`
+	Repository GitRepository `json:"repository,omitempty"`
 }
 
 type githubResponse struct {

api/internal/crawl/crawler/github/queries.go

@@ -7,7 +7,7 @@ import (
 )
 
 const (
-	perPageArg     = "per_page"
+	perPageArg = "per_page"
 )
 
 const githubMaxPageSize = 100

api/internal/crawl/crawler/github/queries_test.go

@@ -68,7 +68,7 @@ func TestQueryType(t *testing.T) {
 
 func TestGithubSearchQuery(t *testing.T) {
 	const (
-		perPage     = 100
+		perPage = 100
 	)
 
 	testCases := []struct {
@@ -82,7 +82,7 @@ func TestGithubSearchQuery(t *testing.T) {
 	}{
 		{
 			rc: RequestConfig{
-				perPage:     perPage,
+				perPage: perPage,
 			},
 			codeQuery: Query{
 				Filename("kustomization.yaml"),