From c02b4f3a119e1934622773ef25f80dae6249ae60 Mon Sep 17 00:00:00 2001 From: Damien Robichaud Date: Tue, 9 Jul 2019 13:04:10 -0700 Subject: [PATCH] Initial (temporary) implementation of search doc. Document describing how to convert a kustomization file into a searchable document on appengine (will be changed to elasticsearch) soon. --- internal/search/doc/doc.go | 169 ++++++++++++++++++++++++++++++++ internal/search/doc/doc_test.go | 153 +++++++++++++++++++++++++++++ internal/search/go.mod | 9 ++ internal/search/go.sum | 24 +++++ travis/pre-commit.sh | 1 + 5 files changed, 356 insertions(+) create mode 100644 internal/search/doc/doc.go create mode 100644 internal/search/doc/doc_test.go create mode 100644 internal/search/go.mod create mode 100644 internal/search/go.sum diff --git a/internal/search/doc/doc.go b/internal/search/doc/doc.go new file mode 100644 index 000000000..5a87c184d --- /dev/null +++ b/internal/search/doc/doc.go @@ -0,0 +1,169 @@ +package doc + +import ( + "fmt" + "strings" + "time" + + "sigs.k8s.io/yaml" + + "google.golang.org/appengine/search" +) + +const ( + identifierStr = "identifier" + documentStr = "document" + repoURLStr = "repo_url" + filePathStr = "file_path" + creationTimeStr = "creation_time" +) + +// Represents an unbreakable character stream. +type Atom = search.Atom + +// Implements search.FieldLoadSaver in order to index this representation of a kustomization.yaml +// file. +type KustomizationDocument struct { + identifiers []Atom + FilePath Atom + RepositoryURL Atom + DocumentData string + CreationTime time.Time +} + +// Partially implements search.FieldLoadSaver. +func (k *KustomizationDocument) Load(fields []search.Field, metadata *search.DocumentMetadata) error { + k.identifiers = make([]search.Atom, 0) + wrongTypeError := func(name string, expected interface{}, actual interface{}) error { + return fmt.Errorf("%s expects type %T, found %#v", name, expected, actual) + } + + for _, f := range fields { + switch f.Name { + case identifierStr: + identifier, ok := f.Value.(search.Atom) + if !ok { + return wrongTypeError(f.Name, identifier, f.Value) + } + k.identifiers = append(k.identifiers, identifier) + + case documentStr: + document, ok := f.Value.(string) + if !ok { + return wrongTypeError(f.Name, document, f.Value) + } + k.DocumentData = document + + case filePathStr: + fp, ok := f.Value.(search.Atom) + if !ok { + return wrongTypeError(f.Name, fp, f.Value) + } + k.FilePath = fp + + case repoURLStr: + url, ok := f.Value.(search.Atom) + if !ok { + return wrongTypeError(f.Name, url, f.Value) + } + k.RepositoryURL = url + + case creationTimeStr: + time, ok := f.Value.(time.Time) + if !ok { + return wrongTypeError(f.Name, time, f.Value) + } + k.CreationTime = time + default: + return fmt.Errorf("KustomizationDocument field %s not recognized", f.Name) + } + } + + return nil +} + +// Partially implements search.FieldLoadSaver. +func (k *KustomizationDocument) Save() ([]search.Field, *search.DocumentMetadata, error) { + err := k.ParseYAML() + if err != nil { + return nil, nil, err + } + + extraFields := []search.Field{ + {Name: documentStr, Value: k.DocumentData}, + {Name: filePathStr, Value: k.FilePath}, + {Name: repoURLStr, Value: k.RepositoryURL}, + {Name: creationTimeStr, Value: k.CreationTime}, + } + + fields := make([]search.Field, 0, len(k.identifiers)+len(extraFields)) + for _, identifier := range k.identifiers { + fields = append(fields, search.Field{Name: identifierStr, Value: identifier}) + } + fields = append(fields, extraFields...) + + return fields, nil, nil +} + +func (k *KustomizationDocument) ParseYAML() error { + k.identifiers = make([]Atom, 0) + + var kustomization map[string]interface{} + err := yaml.Unmarshal([]byte(k.DocumentData), &kustomization) + if err != nil { + return fmt.Errorf("unable to parse kustomization file: %s", err) + } + + type Map struct { + data map[string]interface{} + prefix Atom + } + + toVisit := []Map{ + { + data: kustomization, + prefix: "", + }, + } + + atomJoin := func(vals ...interface{}) Atom { + strs := make([]string, 0, len(vals)) + for _, val := range vals { + strs = append(strs, fmt.Sprint(val)) + } + return Atom(strings.Trim(strings.Join(strs, " "), " ")) + } + + set := make(map[Atom]struct{}) + + for i := 0; i < len(toVisit); i++ { + visiting := toVisit[i] + for k, v := range visiting.data { + set[atomJoin(visiting.prefix, k)] = struct{}{} + switch value := v.(type) { + case map[string]interface{}: + toVisit = append(toVisit, Map{ + data: value, + prefix: atomJoin(visiting.prefix, fmt.Sprint(k)), + }) + case []interface{}: + for _, val := range value { + submap, ok := val.(map[string]interface{}) + if !ok { + continue + } + toVisit = append(toVisit, Map{ + data: submap, + prefix: atomJoin(visiting.prefix, fmt.Sprint(k)), + }) + } + } + } + } + + for key := range set { + k.identifiers = append(k.identifiers, key) + } + + return nil +} diff --git a/internal/search/doc/doc_test.go b/internal/search/doc/doc_test.go new file mode 100644 index 000000000..a28ef6814 --- /dev/null +++ b/internal/search/doc/doc_test.go @@ -0,0 +1,153 @@ +package doc + +import ( + "fmt" + "reflect" + "sort" + "strings" + "testing" + "time" + + "google.golang.org/appengine/search" +) + +func TestLoadFailures(t *testing.T) { + type sentinelType struct{} + sentinel := sentinelType{} + + testCases := [][]search.Field{ + {{Name: identifierStr, Value: sentinel}}, + {{Name: documentStr, Value: sentinel}}, + {{Name: repoURLStr, Value: sentinel}}, + {{Name: filePathStr, Value: sentinel}}, + {{Name: creationTimeStr, Value: sentinel}}, + } + + for _, test := range testCases { + var k KustomizationDocument + err := k.Load(test, nil) + if err == nil { + t.Errorf("Type missmatch %#v should not be loadable", test) + } + } +} + +func TestFieldLoadSaver(t *testing.T) { + + commonTestCases := []KustomizationDocument{ + { + identifiers: []Atom{"namePrefix", "metadata.name", "kind"}, + FilePath: "some/path/kustomization.yaml", + RepositoryURL: "https://example.com/kustomize", + CreationTime: time.Now(), + DocumentData: ` +namePrefix: dev- +metadata: + name: app +kind: Deployment +`, + }, + } + + for _, test := range commonTestCases { + fields, metadata, err := test.Save() + if err != nil { + t.Errorf("Error calling Save(): %s\n", err) + } + doc := KustomizationDocument{} + err = doc.Load(fields, metadata) + if err != nil { + t.Errorf("Doc failed to load: %s\n", err) + } + if !reflect.DeepEqual(test, doc) { + t.Errorf("Expected loaded document (%+v) to be equal to (%+v)\n", doc, test) + } + } +} + +func TestParseYAML(t *testing.T) { + testCases := []struct { + identifiers []Atom + yaml string + }{ + { + identifiers: []Atom{ + "namePrefix", + "metadata", + "metadata name", + "kind", + }, + yaml: ` +namePrefix: dev- +metadata: + name: app +kind: Deployment +`, + }, + { + identifiers: []Atom{ + "namePrefix", + "metadata", + "metadata name", + "metadata spec", + "metadata spec replicas", + "kind", + "replicas", + "replicas name", + "replicas count", + "resource", + }, + yaml: ` +namePrefix: dev- +# map of map +metadata: + name: n1 + spec: + replicas: 3 +kind: Deployment + +#list of map +replicas: +- name: n1 + count: 3 +- name: n2 + count: 3 + +# list +resource: +- file1.yaml +- file2.yaml +`, + }, + } + + atomStrs := func(atoms []Atom) []string { + strs := make([]string, 0, len(atoms)) + for _, val := range atoms { + strs = append(strs, fmt.Sprintf("%v", val)) + } + return strs + } + + for _, test := range testCases { + doc := KustomizationDocument{ + DocumentData: test.yaml, + FilePath: "example/path/kustomization.yaml", + } + + err := doc.ParseYAML() + if err != nil { + t.Errorf("Document error error: %s", err) + } + + docIDs := atomStrs(doc.identifiers) + expectedIDs := atomStrs(test.identifiers) + sort.Strings(docIDs) + sort.Strings(expectedIDs) + + if !reflect.DeepEqual(docIDs, expectedIDs) { + t.Errorf("Expected loaded document (%v) to be equal to (%v)\n", + strings.Join(docIDs, ","), strings.Join(expectedIDs, ",")) + } + } +} diff --git a/internal/search/go.mod b/internal/search/go.mod new file mode 100644 index 000000000..7bde01290 --- /dev/null +++ b/internal/search/go.mod @@ -0,0 +1,9 @@ +module sigs.k8s.io/kustomize/internal/search + +go 1.12 + +require ( + google.golang.org/appengine v1.6.1 + gopkg.in/yaml.v2 v2.2.2 // indirect + sigs.k8s.io/yaml v1.1.0 +) diff --git a/internal/search/go.sum b/internal/search/go.sum new file mode 100644 index 000000000..a93f1159e --- /dev/null +++ b/internal/search/go.sum @@ -0,0 +1,24 @@ +github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65 h1:+rhAzEzT3f4JtomfC371qB+0Ola2caSKcY69NUBZrRQ= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +google.golang.org/appengine v1.6.1 h1:QzqyMA1tlu6CgqCDUtU9V+ZKhLFT2dkJuANu5QaxI3I= +google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs= +sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= diff --git a/travis/pre-commit.sh b/travis/pre-commit.sh index ba34d5723..8d55f2b2b 100755 --- a/travis/pre-commit.sh +++ b/travis/pre-commit.sh @@ -30,6 +30,7 @@ function testGoLangCILint { function testGoTest { go test -v ./... + (cd ./internal/search; go test -v ./...) } # These tests require the helm program, and at the moment