Mulitple improvements of the crawler

1) Set document IDs to avoid duplicating documents;
2) Set the `creationTime` field of each document in the index;
3) set the `values`, `kinds` and `identifiers` fields for all documents;
4) Add a `Copy` method into the `Document` struct: this fixes the issue
where all the documents existing in the index point to the same Document
object;
5) Avoid using keystore redis;
6) Set imagePullPolicy to `Always` for crawler jobs.

api/internal/crawl/cmd/crawler/crawler.go

@@ -39,16 +39,16 @@ func main() {
 	}
 
 	cacheURL := os.Getenv(redisCacheURL)
-	keystoreURL := os.Getenv(redisKeyURL)
 
 	query := []byte(`{ "query":{ "match_all":{} } }`)
 	it := idx.IterateQuery(query, 10000, 60*time.Second)
 	docs := make(crawler.CrawlSeed, 0)
 	for it.Next() {
 		for _, hit := range it.Value().Hits.Hits {
-			docs = append(docs, hit.Document.GetDocument())
+			docs = append(docs, hit.Document.Copy())
 		}
 	}
+
 	if err := it.Err(); err != nil {
 		fmt.Printf("Error iterating: %v\n", err)
 	}
@@ -61,12 +61,6 @@ func main() {
 		clientCache = httpclient.NewClient(cache)
 	}
 
-	_, err = redis.DialURL(keystoreURL)
-	if err != nil {
-		fmt.Printf("Error: redis could not make a connection: %v\n", err)
-		os.Exit(1)
-	}
-
 	ghCrawler := github.NewCrawler(githubToken, retryCount, clientCache,
 		github.QueryWith(
 			github.Filename("kustomization.yaml"),
@@ -88,8 +82,8 @@ func main() {
 		func(cdoc crawler.CrawledDocument, crwlr crawler.Crawler) error {
 			switch d := cdoc.(type) {
 			case *doc.KustomizationDocument:
-				fmt.Println("Inserting: ", d)
-				_, err := idx.Put("", d)
+				fmt.Println("Inserting: ", d.ID(), d)
+				_, err := idx.Put(d.ID(), d)
 				return err
 			default:
 				return fmt.Errorf("type %T not supported", d)