Introduce dummy program to help with API releases.

internal/crawl/doc/doc.go

@@ -0,0 +1,202 @@
+package doc
+
+import (
+	"fmt"
+	"strings"
+
+	"sigs.k8s.io/kustomize/v3/k8sdeps/kunstruct"
+	"sigs.k8s.io/kustomize/v3/pkg/ifc"
+	"sigs.k8s.io/kustomize/v3/pkg/pgmconfig"
+	"sigs.k8s.io/kustomize/v3/pkg/types"
+	"sigs.k8s.io/yaml"
+)
+
+var fileReader ifc.KunstructuredFactory = kunstruct.NewKunstructuredFactoryImpl()
+
+// This document is meant to be used at the elasticsearch document type.
+// Fields are serialized as-is to elasticsearch, where indices are built
+// to facilitate text search queries. Identifiers, Values, FilePath,
+// RepositoryURL and DocumentData are meant to be searched for text queries
+// directly, while the other fields can either be used as a filter, or as
+// additional metadata displayed in the UI.
+//
+// The fields of the document and their purpose are listed below:
+// - DocumentData contains the contents of the kustomization file.
+// - Kinds Represents the kubernetes Kinds that are in this file.
+// - Identifiers are a list of (partial and full) identifier paths that can be
+//   found by users. Each part of a path is delimited by ":" e.g. spec:replicas.
+// - Values are a list of identifier paths and their values that can be found by
+//   search queries. The path is delimited by ":" and the value follows the "="
+//   symbol e.g. spec:replicas=4.
+// - FilePath is the path of the file.
+// - RepositoryURL is the URL of the source repository.
+// - CreationTime is the time at which the file was created.
+//
+// Representing each Identifier and Value as a flat string representation
+// facilitates the use of complex text search features from elasticsearch such
+// as fuzzy searching, regex, wildcards, etc.
+type KustomizationDocument struct {
+	Document
+	Kinds       []string `json:"kinds,omitempty"`
+	Identifiers []string `json:"identifiers,omitempty"`
+	Values      []string `json:"values,omitempty"`
+}
+
+type set map[string]struct{}
+
+// Implements the CrawlerDocument interface.
+func (doc *KustomizationDocument) GetResources() ([]*Document, error) {
+	isResource := true
+	for _, suffix := range pgmconfig.KustomizationFileNames {
+		if strings.HasSuffix(doc.FilePath, "/"+suffix) {
+			isResource = false
+		}
+	}
+	if isResource {
+		return []*Document{}, nil
+	}
+
+	content := []byte(doc.DocumentData)
+	content, err := FixKustomizationPreUnmarshallingNonFatal(content)
+	if err != nil {
+		return nil, fmt.Errorf("could not fix kustomize file: %v", err)
+	}
+
+	var k types.Kustomization
+	err = yaml.Unmarshal(content, &k)
+	if err != nil {
+		return nil, fmt.Errorf(
+			"could not parse kustomization: %v", err)
+	}
+	k.FixKustomizationPostUnmarshalling()
+
+	res := make([]*Document, 0, len(k.Resources))
+	for _, r := range k.Resources {
+		next, err := doc.Document.FromRelativePath(r)
+		if err != nil {
+			fmt.Printf("GetResources error: %v\n", err)
+			continue
+		}
+		res = append(res, &next)
+	}
+
+	return res, nil
+}
+
+func (doc *KustomizationDocument) readBytes() ([]map[string]interface{}, error) {
+	data := []byte(doc.DocumentData)
+
+	for _, suffix := range pgmconfig.KustomizationFileNames {
+		if !strings.HasSuffix(doc.FilePath, "/"+suffix) {
+			continue
+		}
+		var config map[string]interface{}
+		err := yaml.Unmarshal(data, &config)
+		if err != nil {
+			return nil, fmt.Errorf(
+				"unable to parse kustomization: %v", err)
+		}
+		return []map[string]interface{}{config}, nil
+	}
+
+	configs := make([]map[string]interface{}, 0)
+	ks, err := fileReader.SliceFromBytes(data)
+	if err != nil {
+		return nil, fmt.Errorf("unable to parse resource: %v", err)
+	}
+	for _, k := range ks {
+		configs = append(configs, k.Map())
+	}
+
+	return configs, nil
+}
+
+func (doc *KustomizationDocument) ParseYAML() error {
+	doc.Identifiers = make([]string, 0)
+	doc.Values = make([]string, 0)
+	doc.Kinds = make([]string, 0, 1)
+
+	identifierSet := make(set)
+	valueSet := make(set)
+	getKind := func(m map[string]interface{}) string {
+		const defaultStr = "Kustomization"
+		kind, ok := m["kind"]
+		if !ok {
+			return defaultStr
+		}
+		if str, ok := kind.(string); ok && str != "" {
+			return str
+		}
+		return defaultStr
+	}
+
+	ks, err := doc.readBytes()
+	if err != nil {
+		return err
+	}
+
+	for _, contents := range ks {
+		doc.Kinds = append(doc.Kinds, getKind(contents))
+		createFlatStructure(identifierSet, valueSet, contents)
+	}
+
+	for val := range valueSet {
+		doc.Values = append(doc.Values, val)
+	}
+
+	for key := range identifierSet {
+		doc.Identifiers = append(doc.Identifiers, key)
+	}
+
+	return nil
+}
+
+func createFlatStructure(identifierSet set, valueSet set, contents map[string]interface{}) {
+	type Map struct {
+		data   map[string]interface{}
+		prefix string
+	}
+
+	toVisit := []Map{
+		{
+			data:   contents,
+			prefix: "",
+		},
+	}
+
+	for i := 0; i < len(toVisit); i++ {
+		visiting := toVisit[i]
+		for k, v := range visiting.data {
+			identifier := fmt.Sprintf("%s:%s", visiting.prefix, k)
+			// noop after the first iteration.
+			identifier = strings.TrimLeft(identifier, ":")
+
+			// Recursive function traverses structure to find
+			// identifiers and values. These later get formatted
+			// into doc.Identifiers and doc.Values respectively.
+			var traverseStructure func(interface{})
+			traverseStructure = func(arg interface{}) {
+				switch value := arg.(type) {
+				case map[string]interface{}:
+					toVisit = append(toVisit, Map{
+						data:   value,
+						prefix: identifier,
+					})
+				case []interface{}:
+					for _, val := range value {
+						traverseStructure(val)
+					}
+				case interface{}:
+					esc := fmt.Sprintf("%v", value)
+
+					valuePath := fmt.Sprintf("%s=%v",
+						identifier, esc)
+					valueSet[valuePath] = struct{}{}
+				}
+			}
+			traverseStructure(v)
+
+			identifierSet[identifier] = struct{}{}
+		}
+	}
+}