Modify document for elasticsearch migration.

internal/search/doc/doc.go

@@ -6,117 +6,53 @@ import (
 	"time"
 
 	"sigs.k8s.io/yaml"
-
-	"google.golang.org/appengine/search"
 )
 
-const (
+// This document is meant to be used at the elasticsearch document type.
-	identifierStr   = "identifier"
+// Fields are serialized as-is to elasticsearch, where indices are built
-	documentStr     = "document"
+// to facilitate text search queries. Identifiers, Values, FilePath,
-	repoURLStr      = "repo_url"
+// RepositoryURL and DocumentData are meant to be searched for text queries
-	filePathStr     = "file_path"
+// directly, while the other fields can either be used as a filter, or as
-	creationTimeStr = "creation_time"
+// additional metadata displayed in the UI.
-)
+//
-
+// The fields of the document and their purpose are listed below:
-// Represents an unbreakable character stream.
+// - DocumentData contains the contents of the kustomization file.
-type Atom = search.Atom
+// - Kinds Represents the kubernetes Kinds that are in this file.
-
+// - Identifiers are a list of (partial and full) identifier paths that can be
-// Implements search.FieldLoadSaver in order to index this representation of a kustomization.yaml
+//   found by users. Each part of a path is delimited by ":" e.g. spec:replicas.
-// file.
+// - Values are a list of identifier paths and their values that can be found by
+//   search queries. The path is delimited by ":" and the value follows the "="
+//   symbol e.g. spec:replicas=4.
+// - FilePath is the path of the file.
+// - RepositoryURL is the URL of the source repository.
+// - CreationTime is the time at which the file was created.
+//
+// Representing each Identifier and Value as a flat string representation
+// facilitates the use of complex text search features from elasticsearch such
+// as fuzzy searching, regex, wildcards, etc.
 type KustomizationDocument struct {
-	identifiers   []Atom
+	DocumentData  string    `json:"document,omitempty"`
-	FilePath      Atom
+	Kinds         []string  `json:"kinds,omitempty"`
-	RepositoryURL Atom
+	Identifiers   []string  `json:"identifiers,omitempty"`
-	DocumentData  string
+	Values        []string  `json:"values,omitempty"`
-	CreationTime  time.Time
+	FilePath      string    `json:"filePath,omitempty"`
+	RepositoryURL string    `json:"repositoryUrl,omitempty"`
+	CreationTime  time.Time `json:"creationTime,omitempty"`
 }
 
-// Partially implements search.FieldLoadSaver.
+func (doc *KustomizationDocument) ParseYAML() error {
-func (k *KustomizationDocument) Load(fields []search.Field, metadata *search.DocumentMetadata) error {
+	doc.Identifiers = make([]string, 0)
-	k.identifiers = make([]search.Atom, 0)
+	doc.Values = make([]string, 0)
-	wrongTypeError := func(name string, expected interface{}, actual interface{}) error {
-		return fmt.Errorf("%s expects type %T, found %#v", name, expected, actual)
-	}
-
-	for _, f := range fields {
-		switch f.Name {
-		case identifierStr:
-			identifier, ok := f.Value.(search.Atom)
-			if !ok {
-				return wrongTypeError(f.Name, identifier, f.Value)
-			}
-			k.identifiers = append(k.identifiers, identifier)
-
-		case documentStr:
-			document, ok := f.Value.(string)
-			if !ok {
-				return wrongTypeError(f.Name, document, f.Value)
-			}
-			k.DocumentData = document
-
-		case filePathStr:
-			fp, ok := f.Value.(search.Atom)
-			if !ok {
-				return wrongTypeError(f.Name, fp, f.Value)
-			}
-			k.FilePath = fp
-
-		case repoURLStr:
-			url, ok := f.Value.(search.Atom)
-			if !ok {
-				return wrongTypeError(f.Name, url, f.Value)
-			}
-			k.RepositoryURL = url
-
-		case creationTimeStr:
-			time, ok := f.Value.(time.Time)
-			if !ok {
-				return wrongTypeError(f.Name, time, f.Value)
-			}
-			k.CreationTime = time
-		default:
-			return fmt.Errorf("KustomizationDocument field %s not recognized", f.Name)
-		}
-	}
-
-	return nil
-}
-
-// Partially implements search.FieldLoadSaver.
-func (k *KustomizationDocument) Save() ([]search.Field, *search.DocumentMetadata, error) {
-	err := k.ParseYAML()
-	if err != nil {
-		return nil, nil, err
-	}
-
-	extraFields := []search.Field{
-		{Name: documentStr, Value: k.DocumentData},
-		{Name: filePathStr, Value: k.FilePath},
-		{Name: repoURLStr, Value: k.RepositoryURL},
-		{Name: creationTimeStr, Value: k.CreationTime},
-	}
-
-	fields := make([]search.Field, 0, len(k.identifiers)+len(extraFields))
-	for _, identifier := range k.identifiers {
-		fields = append(fields, search.Field{Name: identifierStr, Value: identifier})
-	}
-	fields = append(fields, extraFields...)
-
-	return fields, nil, nil
-}
-
-func (k *KustomizationDocument) ParseYAML() error {
-	k.identifiers = make([]Atom, 0)
 
 	var kustomization map[string]interface{}
-	err := yaml.Unmarshal([]byte(k.DocumentData), &kustomization)
+	err := yaml.Unmarshal([]byte(doc.DocumentData), &kustomization)
 	if err != nil {
 		return fmt.Errorf("unable to parse kustomization file: %s", err)
 	}
 
 	type Map struct {
 		data   map[string]interface{}
-		prefix Atom
+		prefix string
 	}
 
 	toVisit := []Map{
@@ -126,43 +62,53 @@ func (k *KustomizationDocument) ParseYAML() error {
 		},
 	}
 
-	atomJoin := func(vals ...interface{}) Atom {
+	identifierSet := make(map[string]struct{})
-		strs := make([]string, 0, len(vals))
+	valueSet := make(map[string]struct{})
-		for _, val := range vals {
-			strs = append(strs, fmt.Sprint(val))
-		}
-		return Atom(strings.Trim(strings.Join(strs, " "), " "))
-	}
-
-	set := make(map[Atom]struct{})
-
 	for i := 0; i < len(toVisit); i++ {
 		visiting := toVisit[i]
 		for k, v := range visiting.data {
-			set[atomJoin(visiting.prefix, k)] = struct{}{}
+			identifier := fmt.Sprintf("%s:%s", visiting.prefix,
-			switch value := v.(type) {
+				strings.Replace(k, ":", "%3A", -1))
+			// noop after the first iteration.
+			identifier = strings.TrimLeft(identifier, ":")
+
+			// Recursive function traverses structure to find
+			// identifiers and values. These later get formatted
+			// into doc.Identifiers and doc.Values respectively.
+			var traverseStructure func(interface{})
+			traverseStructure = func(arg interface{}) {
+				switch value := arg.(type) {
 				case map[string]interface{}:
 					toVisit = append(toVisit, Map{
 						data:   value,
-					prefix: atomJoin(visiting.prefix, fmt.Sprint(k)),
+						prefix: identifier,
 					})
 				case []interface{}:
 					for _, val := range value {
-					submap, ok := val.(map[string]interface{})
+						traverseStructure(val)
-					if !ok {
-						continue
 					}
-					toVisit = append(toVisit, Map{
+				case interface{}:
-						data:   submap,
+					esc := strings.Replace(fmt.Sprintf("%v",
-						prefix: atomJoin(visiting.prefix, fmt.Sprint(k)),
+						value), ":", "%3A", -1)
-					})
+
+					valuePath := fmt.Sprintf("%s=%v",
+						identifier, esc)
+					valueSet[valuePath] = struct{}{}
 				}
 			}
+			traverseStructure(v)
+
+			identifierSet[identifier] = struct{}{}
+
 		}
 	}
 
-	for key := range set {
+	for val := range valueSet {
-		k.identifiers = append(k.identifiers, key)
+		doc.Values = append(doc.Values, val)
+	}
+
+	for key := range identifierSet {
+		doc.Identifiers = append(doc.Identifiers, key)
 	}
 
 	return nil

internal/search/doc/doc_test.go

@@ -1,82 +1,30 @@
 package doc
 
 import (
-	"fmt"
 	"reflect"
 	"sort"
 	"strings"
 	"testing"
-	"time"
-
-	"google.golang.org/appengine/search"
 )
 
-func TestLoadFailures(t *testing.T) {
-	type sentinelType struct{}
-	sentinel := sentinelType{}
-
-	testCases := [][]search.Field{
-		{{Name: identifierStr, Value: sentinel}},
-		{{Name: documentStr, Value: sentinel}},
-		{{Name: repoURLStr, Value: sentinel}},
-		{{Name: filePathStr, Value: sentinel}},
-		{{Name: creationTimeStr, Value: sentinel}},
-	}
-
-	for _, test := range testCases {
-		var k KustomizationDocument
-		err := k.Load(test, nil)
-		if err == nil {
-			t.Errorf("Type missmatch %#v should not be loadable", test)
-		}
-	}
-}
-
-func TestFieldLoadSaver(t *testing.T) {
-
-	commonTestCases := []KustomizationDocument{
-		{
-			identifiers:   []Atom{"namePrefix", "metadata.name", "kind"},
-			FilePath:      "some/path/kustomization.yaml",
-			RepositoryURL: "https://example.com/kustomize",
-			CreationTime:  time.Now(),
-			DocumentData: `
-namePrefix: dev-
-metadata:
-  name: app
-kind: Deployment
-`,
-		},
-	}
-
-	for _, test := range commonTestCases {
-		fields, metadata, err := test.Save()
-		if err != nil {
-			t.Errorf("Error calling Save(): %s\n", err)
-		}
-		doc := KustomizationDocument{}
-		err = doc.Load(fields, metadata)
-		if err != nil {
-			t.Errorf("Doc failed to load: %s\n", err)
-		}
-		if !reflect.DeepEqual(test, doc) {
-			t.Errorf("Expected loaded document (%+v) to be equal to (%+v)\n", doc, test)
-		}
-	}
-}
-
 func TestParseYAML(t *testing.T) {
 	testCases := []struct {
-		identifiers []Atom
+		identifiers []string
+		values      []string
 		yaml        string
 	}{
 		{
-			identifiers: []Atom{
+			identifiers: []string{
 				"namePrefix",
 				"metadata",
-				"metadata name",
+				"metadata:name",
 				"kind",
 			},
+			values: []string{
+				"namePrefix=dev-",
+				"metadata:name=app",
+				"kind=Deployment",
+			},
 			yaml: `
 namePrefix: dev-
 metadata:
@@ -85,18 +33,29 @@ kind: Deployment
 `,
 		},
 		{
-			identifiers: []Atom{
+			identifiers: []string{
 				"namePrefix",
 				"metadata",
-				"metadata name",
+				"metadata:name",
-				"metadata spec",
+				"metadata:spec",
-				"metadata spec replicas",
+				"metadata:spec:replicas",
 				"kind",
 				"replicas",
-				"replicas name",
+				"replicas:name",
-				"replicas count",
+				"replicas:count",
 				"resource",
 			},
+			values: []string{
+				"namePrefix=dev-",
+				"metadata:name=n1",
+				"metadata:spec:replicas=3",
+				"kind=Deployment",
+				"replicas:name=n1",
+				"replicas:name=n2",
+				"replicas:count=3",
+				"resource=file1.yaml",
+				"resource=file2.yaml",
+			},
 			yaml: `
 namePrefix: dev-
 # map of map
@@ -121,14 +80,6 @@ resource:
 		},
 	}
 
-	atomStrs := func(atoms []Atom) []string {
-		strs := make([]string, 0, len(atoms))
-		for _, val := range atoms {
-			strs = append(strs, fmt.Sprintf("%v", val))
-		}
-		return strs
-	}
-
 	for _, test := range testCases {
 		doc := KustomizationDocument{
 			DocumentData: test.yaml,
@@ -140,14 +91,20 @@ resource:
 			t.Errorf("Document error error: %s", err)
 		}
 
-		docIDs := atomStrs(doc.identifiers)
+		cmpStrings := func(got, expected []string, label string) {
-		expectedIDs := atomStrs(test.identifiers)
+			sort.Strings(got)
-		sort.Strings(docIDs)
+			sort.Strings(expected)
-		sort.Strings(expectedIDs)
 
-		if !reflect.DeepEqual(docIDs, expectedIDs) {
+			if !reflect.DeepEqual(got, expected) {
-			t.Errorf("Expected loaded document (%v) to be equal to (%v)\n",
+				t.Errorf("Expected %s (%v) to be equal to (%v)\n",
-				strings.Join(docIDs, ","), strings.Join(expectedIDs, ","))
+					label,
+					strings.Join(got, ","),
+					strings.Join(expected, ","))
 			}
+
+		}
+
+		cmpStrings(doc.Identifiers, test.identifiers, "identifiers")
+		cmpStrings(doc.Values, test.values, "values")
 	}
 }

internal/search/go.mod

@@ -3,7 +3,6 @@ module sigs.k8s.io/kustomize/internal/search
 go 1.12
 
 require (
-	google.golang.org/appengine v1.6.1
 	gopkg.in/yaml.v2 v2.2.2 // indirect
 	sigs.k8s.io/yaml v1.1.0
 )

internal/search/go.sum

@@ -1,21 +1,3 @@
-github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg=
-github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190603091049-60506f45cf65 h1:+rhAzEzT3f4JtomfC371qB+0Ola2caSKcY69NUBZrRQ=
-golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
-golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
-golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
-google.golang.org/appengine v1.6.1 h1:QzqyMA1tlu6CgqCDUtU9V+ZKhLFT2dkJuANu5QaxI3I=
-google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=