Mulitple improvements of the crawler

1) Set document IDs to avoid duplicating documents;
2) Set the `creationTime` field of each document in the index;
3) set the `values`, `kinds` and `identifiers` fields for all documents;
4) Add a `Copy` method into the `Document` struct: this fixes the issue
where all the documents existing in the index point to the same Document
object;
5) Avoid using keystore redis;
6) Set imagePullPolicy to `Always` for crawler jobs.

api/internal/crawl/doc/doc.go

@@ -44,9 +44,9 @@ type KustomizationDocument struct {
 type set map[string]struct{}
 
 func (doc *KustomizationDocument) String() string {
-	return fmt.Sprintf("%s %s %s %v %v %v %v %v", doc.RepositoryURL, doc.FilePath,
-		doc.DefaultBranch, doc.CreationTime, doc.IsSame,
-		doc.Kinds, doc.Identifiers, doc.Values)
+	return fmt.Sprintf("%s %s %s %v %v %v len(identifiers):%v  len(values):%v",
+		doc.RepositoryURL, doc.FilePath, doc.DefaultBranch, doc.CreationTime,
+		doc.IsSame,	doc.Kinds, len(doc.Identifiers), len(doc.Values))
 }
 
 // Implements the CrawlerDocument interface.

api/internal/crawl/doc/docname.go

@@ -1,7 +1,10 @@
 package doc
 
 import (
+	"crypto/sha256"
+	"fmt"
 	"path"
+	"strings"
 	"time"
 
 	"sigs.k8s.io/kustomize/api/internal/git"
@@ -21,6 +24,17 @@ func (doc *Document) GetDocument() *Document {
 	return doc
 }
 
+func (doc *Document) Copy() *Document {
+	return &Document{
+		RepositoryURL: doc.RepositoryURL,
+		FilePath:      doc.FilePath,
+		DefaultBranch: doc.DefaultBranch,
+		DocumentData:  doc.DocumentData,
+		CreationTime:  doc.CreationTime,
+		IsSame:        doc.IsSame,
+	}
+}
+
 // Implements the CrawlerDocument interface.
 func (doc *Document) WasCached() bool {
 	return doc.IsSame
@@ -53,6 +67,22 @@ func (doc *Document) FromRelativePath(newFile string) (Document, error) {
 }
 
 func (doc *Document) ID() string {
-	return doc.RepositoryURL + "/" +
-		doc.DefaultBranch + "/" + doc.FilePath
+	sum := sha256.Sum256([]byte(strings.Join(
+		[]string{
+			doc.RepositoryURL,
+			doc.DefaultBranch,
+			doc.FilePath,
+		},
+		"---|---")))
+	return fmt.Sprintf("%x", sum)
+}
+
+func (doc *Document) RepositoryFullName() string {
+	doc.RepositoryURL = strings.TrimRight(doc.RepositoryURL, "/")
+	sections := strings.Split(doc.RepositoryURL, "/")
+	l := len(sections)
+	if l < 2 {
+		return doc.RepositoryURL
+	}
+	return path.Join(sections[l-2], sections[l-1])
 }

api/internal/crawl/doc/docname_test.go

@@ -62,3 +62,44 @@ func TestFromRelativePath(t *testing.T) {
 		}
 	}
 }
+
+func TestDocument_RepositoryFullName(t *testing.T) {
+	testCases := []struct {
+		doc Document
+		expectedRepositoryFullName string
+	}{
+		{
+			doc: Document{
+				RepositoryURL: "https://github.com/user/repo",
+			},
+			expectedRepositoryFullName: "user/repo",
+		},
+		{
+			doc: Document{
+				RepositoryURL: "https://github.com//user/repo////",
+			},
+			expectedRepositoryFullName: "user/repo",
+		},
+		{
+			doc: Document{
+				RepositoryURL: "repo/",
+			},
+			expectedRepositoryFullName: "repo",
+		},
+		{
+			doc: Document{
+				RepositoryURL: "",
+			},
+			expectedRepositoryFullName: "",
+		},
+	}
+
+	for _, tc := range testCases {
+		returnedRepositoryFullName := tc.doc.RepositoryFullName()
+		if returnedRepositoryFullName != tc.expectedRepositoryFullName {
+			t.Errorf("RepositoryFullName expected %s, got %s",
+				tc.expectedRepositoryFullName,
+				returnedRepositoryFullName)
+		}
+	}
+}