Merge branch 'master' into fix-process-args-fields

api/internal/accumulator/loadconfigfromcrds_test.go

@@ -42,7 +42,7 @@ More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#ty
 					"type": "string"
 				},
 				"metadata": {
-					"$ref": "sigs.k8s.io/kustomize/pseudo/k8s/apimachinery/pkg/apis/meta/v1.ObjectMeta"
+					"$ref": "k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta"
 				},
 				"spec": {
 					"$ref": "github.com/example/pkg/apis/jingfang/v1beta1.BeeSpec"
@@ -55,7 +55,7 @@ More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#ty
 		"Dependencies": [
 			"github.com/example/pkg/apis/jingfang/v1beta1.BeeSpec",
 			"github.com/example/pkg/apis/jingfang/v1beta1.BeeStatus",
-			"sigs.k8s.io/kustomize/pseudo/k8s/apimachinery/pkg/apis/meta/v1.ObjectMeta"
+			"k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta"
 		]
 	},
 	"github.com/example/pkg/apis/jingfang/v1beta1.BeeSpec": {
@@ -87,7 +87,7 @@ In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-con
 					"type": "string"
 				},
 				"metadata": {
-					"$ref": "sigs.k8s.io/kustomize/pseudo/k8s/apimachinery/pkg/apis/meta/v1.ObjectMeta"
+					"$ref": "k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta"
 				},
 				"spec": {
 					"$ref": "github.com/example/pkg/apis/jingfang/v1beta1.MyKindSpec"
@@ -100,7 +100,7 @@ In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-con
 		"Dependencies": [
 			"github.com/example/pkg/apis/jingfang/v1beta1.MyKindSpec",
 			"github.com/example/pkg/apis/jingfang/v1beta1.MyKindStatus",
-			"sigs.k8s.io/kustomize/pseudo/k8s/apimachinery/pkg/apis/meta/v1.ObjectMeta"
+			"k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta"
 		]
 	},
 	"github.com/example/pkg/apis/jingfang/v1beta1.MyKindSpec": {
@@ -117,13 +117,13 @@ In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-con
 If it is not set we generate a secret dynamically",
 					"x-kubernetes-object-ref-api-version": "v1",
 					"x-kubernetes-object-ref-kind": "Secret",
-					"$ref": "sigs.k8s.io/kustomize/pseudo/k8s/api/core/v1.LocalObjectReference"
+					"$ref": "k8s.io/api/core/v1.LocalObjectReference"
 				}
 			}
 		},
 		"Dependencies": [
 			"github.com/example/pkg/apis/jingfang/v1beta1.Bee",
-			"sigs.k8s.io/kustomize/pseudo/k8s/api/core/v1.LocalObjectReference"
+			"k8s.io/api/core/v1.LocalObjectReference"
 		]
 	},
 	"github.com/example/pkg/apis/jingfang/v1beta1.MyKindStatus": {

api/internal/crawl/cmd/crawler/crawler.go

@@ -39,16 +39,16 @@ func main() {
 	}
 
 	cacheURL := os.Getenv(redisCacheURL)
-	keystoreURL := os.Getenv(redisKeyURL)
 
 	query := []byte(`{ "query":{ "match_all":{} } }`)
 	it := idx.IterateQuery(query, 10000, 60*time.Second)
 	docs := make(crawler.CrawlSeed, 0)
 	for it.Next() {
 		for _, hit := range it.Value().Hits.Hits {
-			docs = append(docs, hit.Document.GetDocument())
+			docs = append(docs, hit.Document.Copy())
 		}
 	}
+
 	if err := it.Err(); err != nil {
 		fmt.Printf("Error iterating: %v\n", err)
 	}
@@ -61,12 +61,6 @@ func main() {
 		clientCache = httpclient.NewClient(cache)
 	}
 
-	_, err = redis.DialURL(keystoreURL)
-	if err != nil {
-		fmt.Printf("Error: redis could not make a connection: %v\n", err)
-		os.Exit(1)
-	}
-
 	ghCrawler := github.NewCrawler(githubToken, retryCount, clientCache,
 		github.QueryWith(
 			github.Filename("kustomization.yaml"),
@@ -88,8 +82,8 @@ func main() {
 		func(cdoc crawler.CrawledDocument, crwlr crawler.Crawler) error {
 			switch d := cdoc.(type) {
 			case *doc.KustomizationDocument:
-				fmt.Println("Inserting: ", d)
-				_, err := idx.Put("", d)
+				fmt.Println("Inserting: ", d.ID(), d)
+				_, err := idx.Put(d.ID(), d)
 				return err
 			default:
 				return fmt.Errorf("type %T not supported", d)

api/internal/crawl/config/crawler/cronjob/cronjob.yaml

@@ -12,6 +12,7 @@ spec:
           containers:
           - name: crawler
             image: gcr.io/kustomize-search/crawler:latest
+            imagePullPolicy: Always
             env:
             - name: GITHUB_ACCESS_TOKEN
               valueFrom:

api/internal/crawl/config/crawler/job/job.yaml

@@ -9,6 +9,7 @@ spec:
       containers:
       - name: crawler
         image: gcr.io/kustomize-search/crawler:latest
+        imagePullPolicy: Always
         env:
         - name: GITHUB_ACCESS_TOKEN
           valueFrom:

api/internal/crawl/config/elastic/service.yaml

@@ -0,0 +1,16 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: elasticsearch
+spec:
+  selector:
+    custom-resource: v1alpha1.ESCluster
+    custom-resource-name: esbasic
+    custom-resource-namespace: default
+    es/data: "true"
+    using: escluster.Cluster
+  ports:
+  - protocol: "TCP"
+    port: 9200
+  type: LoadBalancer
+  loadBalancerIP: ""

api/internal/crawl/config/webapp/backend/deployment.yaml

@@ -17,6 +17,7 @@ spec:
       containers:
       - name: kustomize-search
         image: gcr.io/kustomize-search/backend:latest
+        imagePullPolicy: Always
         livenessProbe:
           httpGet:
             path: /liveness

api/internal/crawl/config/webapp/frontend/deployment.yaml

@@ -17,6 +17,7 @@ spec:
       containers:
       - name: frontend
         image: gcr.io/kustomize-search/frontend:latest
+        imagePullPolicy: Always
         ports:
         - name: frontend-port
           containerPort: 80

api/internal/crawl/crawler/crawler.go

@@ -102,11 +102,9 @@ func CrawlFromSeed(ctx context.Context, seed CrawlSeed,
 	}
 
 	doCrawl := func(docsPtr *CrawlSeed) {
-		for len(*docsPtr) > 0 {
-			back := len(*docsPtr) - 1
-			next := (*docsPtr)[back]
-			*docsPtr = (*docsPtr)[:back]
-
+		n := len(*docsPtr)
+		for i := 0; i < n; i++ {
+			next := (*docsPtr)[i]
 			match := findMatch(next)
 			if match == nil {
 				logIfErr(fmt.Errorf(
@@ -114,24 +112,28 @@ func CrawlFromSeed(ctx context.Context, seed CrawlSeed,
 				continue
 			}
 
+			logger.Println("Crawling ", next.RepositoryURL, next.FilePath)
 			err := match.FetchDocument(ctx, next)
 			logIfErr(err)
 			// If there was no change or there is an error, we don't have
 			// to branch out, since the dependencies are already in the
 			// index, or we cannot find the document.
 			if err != nil || next.WasCached() {
+				if next.WasCached() {
+					logger.Println(next.RepositoryURL, next.FilePath, "is cached already")
+				}
 				continue
 			}
 
+			logIfErr(match.SetCreated(ctx, next))
+
 			cdoc, err := conv(next)
 			logIfErr(err)
-			if err != nil {
-				continue
-			}
 
 			addBranches(cdoc, match)
 		}
 	}
+
 	// Exploit seed to update bulk of corpus.
 	logger.Printf("updating %d documents from seed\n", len(seed))
 	doCrawl(&seed)

api/internal/crawl/crawler/crawler_test.go

@@ -29,7 +29,7 @@ type testCrawler struct {
 }
 
 func (c testCrawler) Match(d *doc.Document) bool {
-	return d != nil && strings.HasPrefix(d.ID(), c.matchPrefix)
+	return d != nil
 }
 
 func (c testCrawler) FetchDocument(_ context.Context, d *doc.Document) error {

api/internal/crawl/crawler/github/crawler.go

@@ -133,8 +133,12 @@ func (gc githubCrawler) FetchDocument(_ context.Context, d *doc.Document) error
 }
 
 func (gc githubCrawler) SetCreated(_ context.Context, d *doc.Document) error {
-	fs := GhFileSpec{}
-	fs.Repository.FullName = d.RepositoryURL + "/" + d.FilePath
+	fs := GhFileSpec{
+		Path: d.FilePath,
+		Repository: GitRepository{
+			FullName: d.RepositoryFullName(),
+		},
+	}
 	creationTime, err := gc.client.GetFileCreationTime(fs)
 	if err != nil {
 		return err
@@ -185,11 +189,13 @@ func processQuery(ctx context.Context, gcl GhClient, query string,
 		for _, file := range page.Parsed.Items {
 			k, err := kustomizationResultAdapter(gcl, file)
 			if err != nil {
+				logger.Printf("kustomizationResultAdapter failed: %v", err)
 				errs = append(errs, err)
 				errorCnt++
-				continue
 			}
-			output <- k
+			if k != nil {
+				output <- k
+			}
 			totalCnt++
 		}
 
@@ -224,6 +230,18 @@ func kustomizationResultAdapter(gcl GhClient, k GhFileSpec) (
 			RepositoryURL: k.Repository.URL,
 		},
 	}
+	logger.Printf("Set the creationTime field")
+	creationTime, err := gcl.GetFileCreationTime(k)
+	if err != nil {
+		logger.Printf("GetFileCreationTime failed: %v", err)
+		return &d, err
+	}
+	d.CreationTime = &creationTime
+
+	if err := d.ParseYAML(); err != nil {
+		logger.Printf("ParseYAML failed: %v", err)
+		return &d, err
+	}
 
 	return &d, nil
 }
@@ -410,13 +428,15 @@ func (e multiError) Error() string {
 	return strings.Join(strs, "\n")
 }
 
+type GitRepository struct {
+	API      string `json:"url,omitempty"`
+	URL      string `json:"html_url,omitempty"`
+	FullName string `json:"full_name,omitempty"`
+}
+
 type GhFileSpec struct {
-	Path       string `json:"path,omitempty"`
-	Repository struct {
-		API      string `json:"url,omitempty"`
-		URL      string `json:"html_url,omitempty"`
-		FullName string `json:"full_name,omitempty"`
-	} `json:"repository,omitempty"`
+	Path       string        `json:"path,omitempty"`
+	Repository GitRepository `json:"repository,omitempty"`
 }
 
 type githubResponse struct {

api/internal/crawl/crawler/github/queries.go

@@ -7,7 +7,7 @@ import (
 )
 
 const (
-	perPageArg     = "per_page"
+	perPageArg = "per_page"
 )
 
 const githubMaxPageSize = 100

api/internal/crawl/crawler/github/queries_test.go

@@ -68,7 +68,7 @@ func TestQueryType(t *testing.T) {
 
 func TestGithubSearchQuery(t *testing.T) {
 	const (
-		perPage     = 100
+		perPage = 100
 	)
 
 	testCases := []struct {
@@ -82,7 +82,7 @@ func TestGithubSearchQuery(t *testing.T) {
 	}{
 		{
 			rc: RequestConfig{
-				perPage:     perPage,
+				perPage: perPage,
 			},
 			codeQuery: Query{
 				Filename("kustomization.yaml"),

api/internal/crawl/doc/doc.go

@@ -44,9 +44,9 @@ type KustomizationDocument struct {
 type set map[string]struct{}
 
 func (doc *KustomizationDocument) String() string {
-	return fmt.Sprintf("%s %s %s %v %v %v %v %v", doc.RepositoryURL, doc.FilePath,
-		doc.DefaultBranch, doc.CreationTime, doc.IsSame,
-		doc.Kinds, doc.Identifiers, doc.Values)
+	return fmt.Sprintf("%s %s %s %v %v %v len(identifiers):%v  len(values):%v",
+		doc.RepositoryURL, doc.FilePath, doc.DefaultBranch, doc.CreationTime,
+		doc.IsSame,	doc.Kinds, len(doc.Identifiers), len(doc.Values))
 }
 
 // Implements the CrawlerDocument interface.

api/internal/crawl/doc/docname.go

@@ -1,7 +1,10 @@
 package doc
 
 import (
+	"crypto/sha256"
+	"fmt"
 	"path"
+	"strings"
 	"time"
 
 	"sigs.k8s.io/kustomize/api/internal/git"
@@ -21,6 +24,17 @@ func (doc *Document) GetDocument() *Document {
 	return doc
 }
 
+func (doc *Document) Copy() *Document {
+	return &Document{
+		RepositoryURL: doc.RepositoryURL,
+		FilePath:      doc.FilePath,
+		DefaultBranch: doc.DefaultBranch,
+		DocumentData:  doc.DocumentData,
+		CreationTime:  doc.CreationTime,
+		IsSame:        doc.IsSame,
+	}
+}
+
 // Implements the CrawlerDocument interface.
 func (doc *Document) WasCached() bool {
 	return doc.IsSame
@@ -53,6 +67,22 @@ func (doc *Document) FromRelativePath(newFile string) (Document, error) {
 }
 
 func (doc *Document) ID() string {
-	return doc.RepositoryURL + "/" +
-		doc.DefaultBranch + "/" + doc.FilePath
+	sum := sha256.Sum256([]byte(strings.Join(
+		[]string{
+			doc.RepositoryURL,
+			doc.DefaultBranch,
+			doc.FilePath,
+		},
+		"---|---")))
+	return fmt.Sprintf("%x", sum)
+}
+
+func (doc *Document) RepositoryFullName() string {
+	doc.RepositoryURL = strings.TrimRight(doc.RepositoryURL, "/")
+	sections := strings.Split(doc.RepositoryURL, "/")
+	l := len(sections)
+	if l < 2 {
+		return doc.RepositoryURL
+	}
+	return path.Join(sections[l-2], sections[l-1])
 }

api/internal/crawl/doc/docname_test.go

@@ -62,3 +62,44 @@ func TestFromRelativePath(t *testing.T) {
 		}
 	}
 }
+
+func TestDocument_RepositoryFullName(t *testing.T) {
+	testCases := []struct {
+		doc Document
+		expectedRepositoryFullName string
+	}{
+		{
+			doc: Document{
+				RepositoryURL: "https://github.com/user/repo",
+			},
+			expectedRepositoryFullName: "user/repo",
+		},
+		{
+			doc: Document{
+				RepositoryURL: "https://github.com//user/repo////",
+			},
+			expectedRepositoryFullName: "user/repo",
+		},
+		{
+			doc: Document{
+				RepositoryURL: "repo/",
+			},
+			expectedRepositoryFullName: "repo",
+		},
+		{
+			doc: Document{
+				RepositoryURL: "",
+			},
+			expectedRepositoryFullName: "",
+		},
+	}
+
+	for _, tc := range testCases {
+		returnedRepositoryFullName := tc.doc.RepositoryFullName()
+		if returnedRepositoryFullName != tc.expectedRepositoryFullName {
+			t.Errorf("RepositoryFullName expected %s, got %s",
+				tc.expectedRepositoryFullName,
+				returnedRepositoryFullName)
+		}
+	}
+}

api/internal/crawl/ui/package.json

@@ -11,38 +11,39 @@
   },
   "private": true,
   "dependencies": {
-    "@angular/animations": "~8.1.0",
+    "@angular/animations": "^8.2.14",
     "@angular/cdk": "~8.0.2",
-    "@angular/common": "~8.1.0",
-    "@angular/compiler": "~8.1.0",
-    "@angular/core": "~8.1.0",
-    "@angular/forms": "~8.1.0",
+    "@angular/common": "^8.2.14",
+    "@angular/compiler": "^8.2.14",
+    "@angular/core": "^8.2.14",
+    "@angular/forms": "^8.2.14",
     "@angular/http": "^7.2.15",
-    "@angular/material": "^8.0.2",
-    "@angular/platform-browser": "~8.1.0",
-    "@angular/platform-browser-dynamic": "~8.1.0",
-    "@angular/router": "~8.1.0",
+    "@angular/material": "^8.2.3",
+    "@angular/platform-browser": "^8.2.14",
+    "@angular/platform-browser-dynamic": "^8.2.14",
+    "@angular/router": "^8.2.14",
     "angular-google-charts": "^0.1.6",
-    "chart.js": "^2.8.0",
+    "chart.js": "^2.9.3",
+    "core-js": "^3.5.0",
     "hammerjs": "^2.0.8",
     "rxjs": "~6.4.0",
     "tslib": "^1.9.0",
     "zone.js": "~0.9.1"
   },
   "devDependencies": {
-    "@angular-devkit/build-angular": "~0.801.0",
-    "@angular/cli": "^8.1.2",
-    "@angular/compiler-cli": "~8.1.0",
-    "@angular/language-service": "~8.1.0",
-    "@types/jasmine": "~3.3.8",
-    "@types/jasminewd2": "~2.0.3",
+    "@angular-devkit/build-angular": "^0.801.3",
+    "@angular/cli": "^8.3.20",
+    "@angular/compiler-cli": "^8.2.14",
+    "@angular/language-service": "^8.2.14",
+    "@types/jasmine": "^3.5.0",
+    "@types/jasminewd2": "^2.0.8",
     "@types/node": "~8.9.4",
-    "codelyzer": "^5.0.0",
+    "codelyzer": "^5.2.0",
     "jasmine-core": "~3.4.0",
     "jasmine-spec-reporter": "~4.2.1",
     "karma": "~4.1.0",
     "karma-chrome-launcher": "~2.2.0",
-    "karma-coverage-istanbul-reporter": "~2.0.1",
+    "karma-coverage-istanbul-reporter": "^2.1.1",
     "karma-jasmine": "~2.0.1",
     "karma-jasmine-html-reporter": "^1.4.0",
     "protractor": "~5.4.0",

api/internal/k8sdeps/configmapandsecret/configmapfactory.go

@@ -9,8 +9,8 @@ import (
 	"unicode/utf8"
 
 	"github.com/pkg/errors"
+	corev1 "k8s.io/api/core/v1"
 	"sigs.k8s.io/kustomize/api/types"
-	corev1 "sigs.k8s.io/kustomize/pseudo/k8s/api/core/v1"
 )
 
 func makeFreshConfigMap(

api/internal/k8sdeps/configmapandsecret/configmapfactory_test.go

@@ -8,13 +8,13 @@ import (
 	"reflect"
 	"testing"
 
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"sigs.k8s.io/kustomize/api/filesys"
 	"sigs.k8s.io/kustomize/api/kv"
 	"sigs.k8s.io/kustomize/api/loader"
 	valtest_test "sigs.k8s.io/kustomize/api/testutils/valtest"
 	"sigs.k8s.io/kustomize/api/types"
-	corev1 "sigs.k8s.io/kustomize/pseudo/k8s/api/core/v1"
-	metav1 "sigs.k8s.io/kustomize/pseudo/k8s/apimachinery/pkg/apis/meta/v1"
 )
 
 func makeEnvConfigMap(name string) *corev1.ConfigMap {

api/internal/k8sdeps/configmapandsecret/secretfactory.go

@@ -6,8 +6,8 @@ package configmapandsecret
 import (
 	"fmt"
 
+	corev1 "k8s.io/api/core/v1"
 	"sigs.k8s.io/kustomize/api/types"
-	corev1 "sigs.k8s.io/kustomize/pseudo/k8s/api/core/v1"
 )
 
 func makeFreshSecret(

api/internal/k8sdeps/configmapandsecret/secretfactory_test.go

@@ -7,13 +7,13 @@ import (
 	"reflect"
 	"testing"
 
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"sigs.k8s.io/kustomize/api/filesys"
 	"sigs.k8s.io/kustomize/api/kv"
 	"sigs.k8s.io/kustomize/api/loader"
 	valtest_test "sigs.k8s.io/kustomize/api/testutils/valtest"
 	"sigs.k8s.io/kustomize/api/types"
-	corev1 "sigs.k8s.io/kustomize/pseudo/k8s/api/core/v1"
-	metav1 "sigs.k8s.io/kustomize/pseudo/k8s/apimachinery/pkg/apis/meta/v1"
 )
 
 func makeEnvSecret(name string) *corev1.Secret {

api/internal/k8sdeps/transformer/patch/conflictdetector.go

@@ -8,14 +8,14 @@ import (
 	"fmt"
 
 	jsonpatch "github.com/evanphx/json-patch"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/util/mergepatch"
+	"k8s.io/apimachinery/pkg/util/strategicpatch"
+	"k8s.io/client-go/kubernetes/scheme"
 	"sigs.k8s.io/kustomize/api/resid"
 	"sigs.k8s.io/kustomize/api/resmap"
 	"sigs.k8s.io/kustomize/api/resource"
-	"sigs.k8s.io/kustomize/pseudo/k8s/apimachinery/pkg/runtime"
-	"sigs.k8s.io/kustomize/pseudo/k8s/apimachinery/pkg/runtime/schema"
-	"sigs.k8s.io/kustomize/pseudo/k8s/apimachinery/pkg/util/mergepatch"
-	"sigs.k8s.io/kustomize/pseudo/k8s/apimachinery/pkg/util/strategicpatch"
-	"sigs.k8s.io/kustomize/pseudo/k8s/client-go/kubernetes/scheme"
 )
 
 type conflictDetector interface {

api/internal/plugins/compiler/compiler.go

@@ -37,6 +37,16 @@ func DeterminePluginSrcRoot(fSys filesys.FileSystem) (string, error) {
 		"source directory", fSys, []konfig.NotedFunc{
 			{
 				Note: "relative to unit test",
+				F: func() string {
+					return filepath.Clean(
+						filepath.Join(
+							os.Getenv("PWD"),
+							"..", "..",
+							konfig.RelPluginHome))
+				},
+			},
+			{
+				Note: "relative to unit test (internal pkg)",
 				F: func() string {
 					return filepath.Clean(
 						filepath.Join(