kubernetes-sigs/kustomize
1
0
Fork 0
mirror of https://github.com/kubernetes-sigs/kustomize.git synced 2026-06-10 08:20:59 +00:00
Code Issues Packages Projects Releases 1 Wiki Activity

Modify document for elasticsearch migration.

Browse Source
This commit is contained in:
Damien Robichaud
2019-07-29 20:25:45 -07:00
parent e0d388c6f7
commit df779fd720
4 changed files with 110 additions and 226 deletions
Download Patch File Download Diff File Expand all files Collapse all files

196
internal/search/doc/doc.go
View File

@@ -6,117 +6,53 @@ import (
"time"
"sigs.k8s.io/yaml"
"google.golang.org/appengine/search"
)
const (
identifierStr = "identifier"
documentStr = "document"
repoURLStr = "repo_url"
filePathStr = "file_path"
creationTimeStr = "creation_time"
)
// Represents an unbreakable character stream.
type Atom = search.Atom
// Implements search.FieldLoadSaver in order to index this representation of a kustomization.yaml
// file.
// This document is meant to be used at the elasticsearch document type.
// Fields are serialized as-is to elasticsearch, where indices are built
// to facilitate text search queries. Identifiers, Values, FilePath,
// RepositoryURL and DocumentData are meant to be searched for text queries
// directly, while the other fields can either be used as a filter, or as
// additional metadata displayed in the UI.
//
// The fields of the document and their purpose are listed below:
// - DocumentData contains the contents of the kustomization file.
// - Kinds Represents the kubernetes Kinds that are in this file.
// - Identifiers are a list of (partial and full) identifier paths that can be
// found by users. Each part of a path is delimited by ":" e.g. spec:replicas.
// - Values are a list of identifier paths and their values that can be found by
// search queries. The path is delimited by ":" and the value follows the "="
// symbol e.g. spec:replicas=4.
// - FilePath is the path of the file.
// - RepositoryURL is the URL of the source repository.
// - CreationTime is the time at which the file was created.
//
// Representing each Identifier and Value as a flat string representation
// facilitates the use of complex text search features from elasticsearch such
// as fuzzy searching, regex, wildcards, etc.
type KustomizationDocument struct {
identifiers []Atom
FilePath Atom
RepositoryURL Atom
DocumentData string
CreationTime time.Time
DocumentData string `json:"document,omitempty"`
Kinds []string `json:"kinds,omitempty"`
Identifiers []string `json:"identifiers,omitempty"`
Values []string `json:"values,omitempty"`
FilePath string `json:"filePath,omitempty"`
RepositoryURL string `json:"repositoryUrl,omitempty"`
CreationTime time.Time `json:"creationTime,omitempty"`
}
// Partially implements search.FieldLoadSaver.
func (k *KustomizationDocument) Load(fields []search.Field, metadata *search.DocumentMetadata) error {
k.identifiers = make([]search.Atom, 0)
wrongTypeError := func(name string, expected interface{}, actual interface{}) error {
return fmt.Errorf("%s expects type %T, found %#v", name, expected, actual)
}
for _, f := range fields {
switch f.Name {
case identifierStr:
identifier, ok := f.Value.(search.Atom)
if !ok {
return wrongTypeError(f.Name, identifier, f.Value)
}
k.identifiers = append(k.identifiers, identifier)
case documentStr:
document, ok := f.Value.(string)
if !ok {
return wrongTypeError(f.Name, document, f.Value)
}
k.DocumentData = document
case filePathStr:
fp, ok := f.Value.(search.Atom)
if !ok {
return wrongTypeError(f.Name, fp, f.Value)
}
k.FilePath = fp
case repoURLStr:
url, ok := f.Value.(search.Atom)
if !ok {
return wrongTypeError(f.Name, url, f.Value)
}
k.RepositoryURL = url
case creationTimeStr:
time, ok := f.Value.(time.Time)
if !ok {
return wrongTypeError(f.Name, time, f.Value)
}
k.CreationTime = time
default:
return fmt.Errorf("KustomizationDocument field %s not recognized", f.Name)
}
}
return nil
}
// Partially implements search.FieldLoadSaver.
func (k *KustomizationDocument) Save() ([]search.Field, *search.DocumentMetadata, error) {
err := k.ParseYAML()
if err != nil {
return nil, nil, err
}
extraFields := []search.Field{
{Name: documentStr, Value: k.DocumentData},
{Name: filePathStr, Value: k.FilePath},
{Name: repoURLStr, Value: k.RepositoryURL},
{Name: creationTimeStr, Value: k.CreationTime},
}
fields := make([]search.Field, 0, len(k.identifiers)+len(extraFields))
for _, identifier := range k.identifiers {
fields = append(fields, search.Field{Name: identifierStr, Value: identifier})
}
fields = append(fields, extraFields...)
return fields, nil, nil
}
func (k *KustomizationDocument) ParseYAML() error {
k.identifiers = make([]Atom, 0)
func (doc *KustomizationDocument) ParseYAML() error {
doc.Identifiers = make([]string, 0)
doc.Values = make([]string, 0)
var kustomization map[string]interface{}
err := yaml.Unmarshal([]byte(k.DocumentData), &kustomization)
err := yaml.Unmarshal([]byte(doc.DocumentData), &kustomization)
if err != nil {
return fmt.Errorf("unable to parse kustomization file: %s", err)
}
type Map struct {
data map[string]interface{}
prefix Atom
prefix string
}
toVisit := []Map{
@@ -126,43 +62,53 @@ func (k *KustomizationDocument) ParseYAML() error {
},
}
atomJoin := func(vals ...interface{}) Atom {
strs := make([]string, 0, len(vals))
for _, val := range vals {
strs = append(strs, fmt.Sprint(val))
}
return Atom(strings.Trim(strings.Join(strs, " "), " "))
}
set := make(map[Atom]struct{})
identifierSet := make(map[string]struct{})
valueSet := make(map[string]struct{})
for i := 0; i < len(toVisit); i++ {
visiting := toVisit[i]
for k, v := range visiting.data {
set[atomJoin(visiting.prefix, k)] = struct{}{}
switch value := v.(type) {
case map[string]interface{}:
toVisit = append(toVisit, Map{
data: value,
prefix: atomJoin(visiting.prefix, fmt.Sprint(k)),
})
case []interface{}:
for _, val := range value {
submap, ok := val.(map[string]interface{})
if !ok {
continue
}
identifier := fmt.Sprintf("%s:%s", visiting.prefix,
strings.Replace(k, ":", "%3A", -1))
// noop after the first iteration.
identifier = strings.TrimLeft(identifier, ":")
// Recursive function traverses structure to find
// identifiers and values. These later get formatted
// into doc.Identifiers and doc.Values respectively.
var traverseStructure func(interface{})
traverseStructure = func(arg interface{}) {
switch value := arg.(type) {
case map[string]interface{}:
toVisit = append(toVisit, Map{
data: submap,
prefix: atomJoin(visiting.prefix, fmt.Sprint(k)),
data: value,
prefix: identifier,
})
case []interface{}:
for _, val := range value {
traverseStructure(val)
}
case interface{}:
esc := strings.Replace(fmt.Sprintf("%v",
value), ":", "%3A", -1)
valuePath := fmt.Sprintf("%s=%v",
identifier, esc)
valueSet[valuePath] = struct{}{}
}
}
traverseStructure(v)
identifierSet[identifier] = struct{}{}
}
}
for key := range set {
k.identifiers = append(k.identifiers, key)
for val := range valueSet {
doc.Values = append(doc.Values, val)
}
for key := range identifierSet {
doc.Identifiers = append(doc.Identifiers, key)
}
return nil

121
internal/search/doc/doc_test.go
View File

@@ -1,82 +1,30 @@
package doc
import (
"fmt"
"reflect"
"sort"
"strings"
"testing"
"time"
"google.golang.org/appengine/search"
)
func TestLoadFailures(t *testing.T) {
type sentinelType struct{}
sentinel := sentinelType{}
testCases := [][]search.Field{
{{Name: identifierStr, Value: sentinel}},
{{Name: documentStr, Value: sentinel}},
{{Name: repoURLStr, Value: sentinel}},
{{Name: filePathStr, Value: sentinel}},
{{Name: creationTimeStr, Value: sentinel}},
}
for _, test := range testCases {
var k KustomizationDocument
err := k.Load(test, nil)
if err == nil {
t.Errorf("Type missmatch %#v should not be loadable", test)
}
}
}
func TestFieldLoadSaver(t *testing.T) {
commonTestCases := []KustomizationDocument{
{
identifiers: []Atom{"namePrefix", "metadata.name", "kind"},
FilePath: "some/path/kustomization.yaml",
RepositoryURL: "https://example.com/kustomize",
CreationTime: time.Now(),
DocumentData: `
namePrefix: dev-
metadata:
name: app
kind: Deployment
`,
},
}
for _, test := range commonTestCases {
fields, metadata, err := test.Save()
if err != nil {
t.Errorf("Error calling Save(): %s\n", err)
}
doc := KustomizationDocument{}
err = doc.Load(fields, metadata)
if err != nil {
t.Errorf("Doc failed to load: %s\n", err)
}
if !reflect.DeepEqual(test, doc) {
t.Errorf("Expected loaded document (%+v) to be equal to (%+v)\n", doc, test)
}
}
}
func TestParseYAML(t *testing.T) {
testCases := []struct {
identifiers []Atom
identifiers []string
values []string
yaml string
}{
{
identifiers: []Atom{
identifiers: []string{
"namePrefix",
"metadata",
"metadata name",
"metadata:name",
"kind",
},
values: []string{
"namePrefix=dev-",
"metadata:name=app",
"kind=Deployment",
},
yaml: `
namePrefix: dev-
metadata:
@@ -85,18 +33,29 @@ kind: Deployment
`,
},
{
identifiers: []Atom{
identifiers: []string{
"namePrefix",
"metadata",
"metadata name",
"metadata spec",
"metadata spec replicas",
"metadata:name",
"metadata:spec",
"metadata:spec:replicas",
"kind",
"replicas",
"replicas name",
"replicas count",
"replicas:name",
"replicas:count",
"resource",
},
values: []string{
"namePrefix=dev-",
"metadata:name=n1",
"metadata:spec:replicas=3",
"kind=Deployment",
"replicas:name=n1",
"replicas:name=n2",
"replicas:count=3",
"resource=file1.yaml",
"resource=file2.yaml",
},
yaml: `
namePrefix: dev-
# map of map
@@ -121,14 +80,6 @@ resource:
},
}
atomStrs := func(atoms []Atom) []string {
strs := make([]string, 0, len(atoms))
for _, val := range atoms {
strs = append(strs, fmt.Sprintf("%v", val))
}
return strs
}
for _, test := range testCases {
doc := KustomizationDocument{
DocumentData: test.yaml,
@@ -140,14 +91,20 @@ resource:
t.Errorf("Document error error: %s", err)
}
docIDs := atomStrs(doc.identifiers)
expectedIDs := atomStrs(test.identifiers)
sort.Strings(docIDs)
sort.Strings(expectedIDs)
cmpStrings := func(got, expected []string, label string) {
sort.Strings(got)
sort.Strings(expected)
if !reflect.DeepEqual(got, expected) {
t.Errorf("Expected %s (%v) to be equal to (%v)\n",
label,
strings.Join(got, ","),
strings.Join(expected, ","))
}
if !reflect.DeepEqual(docIDs, expectedIDs) {
t.Errorf("Expected loaded document (%v) to be equal to (%v)\n",
strings.Join(docIDs, ","), strings.Join(expectedIDs, ","))
}
cmpStrings(doc.Identifiers, test.identifiers, "identifiers")
cmpStrings(doc.Values, test.values, "values")
}
}

1
internal/search/go.mod
View File

@@ -3,7 +3,6 @@ module sigs.k8s.io/kustomize/internal/search
go 1.12
require (
google.golang.org/appengine v1.6.1
gopkg.in/yaml.v2 v2.2.2 // indirect
sigs.k8s.io/yaml v1.1.0
)

18
internal/search/go.sum
View File

@@ -1,21 +1,3 @@
github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg=
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190603091049-60506f45cf65 h1:+rhAzEzT3f4JtomfC371qB+0Ola2caSKcY69NUBZrRQ=
golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
google.golang.org/appengine v1.6.1 h1:QzqyMA1tlu6CgqCDUtU9V+ZKhLFT2dkJuANu5QaxI3I=
google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=