From 28307bc435ce18189c2a6cb5d70eef32b64bbd66 Mon Sep 17 00:00:00 2001 From: Phani Teja Marupaka Date: Fri, 19 Jun 2020 17:17:32 -0700 Subject: [PATCH] Simplify parsing --- cmd/config/internal/commands/e2e/e2e_test.go | 14 ++-------- cmd/config/internal/commands/sink_test.go | 4 +-- cmd/config/internal/commands/source.go | 2 +- cmd/config/internal/commands/source_test.go | 13 ++++----- kyaml/kio/byteio_reader.go | 29 ++------------------ kyaml/kio/byteio_writer.go | 27 ++++++++---------- kyaml/kio/filters/testyaml/testyaml.go | 14 ++++++++-- kyaml/kio/kioutil/kioutil.go | 10 ++----- kyaml/kio/pkgio_reader.go | 17 +----------- kyaml/kio/pkgio_reader_test.go | 7 ++--- kyaml/runfn/runfn.go | 2 +- kyaml/setters2/set.go | 4 +-- kyaml/yaml/types.go | 2 -- 13 files changed, 47 insertions(+), 98 deletions(-) diff --git a/cmd/config/internal/commands/e2e/e2e_test.go b/cmd/config/internal/commands/e2e/e2e_test.go index c74721d79..962134c99 100644 --- a/cmd/config/internal/commands/e2e/e2e_test.go +++ b/cmd/config/internal/commands/e2e/e2e_test.go @@ -87,18 +87,8 @@ metadata: expectedFiles: func(d string) map[string]string { return map[string]string{ "deployment.json": ` -{ - "apiVersion": "apps/v1", - "kind": "Deployment", - "metadata": { - "annotations": { - "a-bool-value": "false", - "a-int-value": "0", - "a-string-value": "" - }, - "name": "foo" - } -} +{"apiVersion": "apps/v1", "kind": "Deployment", "metadata": {"name": "foo", annotations: { + a-string-value: '', a-int-value: '0', a-bool-value: 'false'}}} `, } }, diff --git a/cmd/config/internal/commands/sink_test.go b/cmd/config/internal/commands/sink_test.go index 83fb47bf1..67d364d62 100644 --- a/cmd/config/internal/commands/sink_test.go +++ b/cmd/config/internal/commands/sink_test.go @@ -148,7 +148,7 @@ func TestSinkCommandJSON(t *testing.T) { kind: ResourceList items: - {"kind": "Deployment", "metadata": {"labels": {"app": "nginx2"}, "name": "foo", - "annotations": {"app": "nginx2", config.kubernetes.io/format: 'json', config.kubernetes.io/index: '0', + "annotations": {"app": "nginx2", config.kubernetes.io/index: '0', config.kubernetes.io/path: 'f1.json'}}, "spec": {"replicas": 1}} `)) r.Command.SetArgs([]string{d}) @@ -309,7 +309,7 @@ func TestSinkCommandJSON_Stdout(t *testing.T) { kind: ResourceList items: - {"kind": "Deployment", "metadata": {"labels": {"app": "nginx2"}, "name": "foo", - "annotations": {"app": "nginx2", config.kubernetes.io/format: 'json', config.kubernetes.io/index: '0', + "annotations": {"app": "nginx2", config.kubernetes.io/index: '0', config.kubernetes.io/path: 'f1.json'}}, "spec": {"replicas": 1}} `)) diff --git a/cmd/config/internal/commands/source.go b/cmd/config/internal/commands/source.go index 8dc6296ae..7873956ce 100644 --- a/cmd/config/internal/commands/source.go +++ b/cmd/config/internal/commands/source.go @@ -74,7 +74,7 @@ func (r *SourceRunner) runE(c *cobra.Command, args []string) error { inputs = append(inputs, kio.LocalPackageReader{PackagePath: a, MatchFilesGlob: kio.MatchAll}) } if len(inputs) == 0 { - inputs = []kio.Reader{&kio.ByteReader{Reader: c.InOrStdin(), AcceptJSON: true}} + inputs = []kio.Reader{&kio.ByteReader{Reader: c.InOrStdin()}} } err := kio.Pipeline{Inputs: inputs, Outputs: outputs}.Execute() diff --git a/cmd/config/internal/commands/source_test.go b/cmd/config/internal/commands/source_test.go index 72d82270f..717b22c4a 100644 --- a/cmd/config/internal/commands/source_test.go +++ b/cmd/config/internal/commands/source_test.go @@ -195,13 +195,12 @@ func TestSourceCommandJSON(t *testing.T) { kind: ResourceList items: - {"kind": "Deployment", "metadata": {"labels": {"app": "nginx2"}, "name": "foo", - "annotations": {"app": "nginx2", config.kubernetes.io/format: 'json', config.kubernetes.io/index: '0', - config.kubernetes.io/path: 'f1.json'}}, "spec": {"replicas": 1}} + "annotations": {"app": "nginx2", config.kubernetes.io/index: '0', config.kubernetes.io/path: 'f1.json'}}, + "spec": {"replicas": 1}} - {"apiVersion": "v1", "kind": "Abstraction", "metadata": {"name": "foo", "annotations": { "config.kubernetes.io/function": "container:\n image: gcr.io/example/reconciler:v1\n", - "config.kubernetes.io/local-config": "true", config.kubernetes.io/format: 'json', - config.kubernetes.io/index: '0', config.kubernetes.io/path: 'f2.json'}}, "spec": { - "replicas": 3}} + "config.kubernetes.io/local-config": "true", config.kubernetes.io/index: '0', + config.kubernetes.io/path: 'f2.json'}}, "spec": {"replicas": 3}} `, b.String()) { return } @@ -309,8 +308,8 @@ func TestSourceCommandJSON_Stdin(t *testing.T) { kind: ResourceList items: - {"kind": "Deployment", "metadata": {"labels": {"app": "nginx2"}, "name": "foo", - "annotations": {"app": "nginx2", config.kubernetes.io/format: 'json', config.kubernetes.io/index: '0'}}, - "spec": {"replicas": 1}} + "annotations": {"app": "nginx2", config.kubernetes.io/index: '0'}}, "spec": { + "replicas": 1}} `, out.String()) { return } diff --git a/kyaml/kio/byteio_reader.go b/kyaml/kio/byteio_reader.go index a3adcae41..670fb96f5 100644 --- a/kyaml/kio/byteio_reader.go +++ b/kyaml/kio/byteio_reader.go @@ -5,7 +5,6 @@ package kio import ( "bytes" - "encoding/json" "fmt" "io" "sort" @@ -102,16 +101,8 @@ type ByteReader struct { // WrappingKind is set by Read(), and is the kind of the object that // the read objects were originally wrapped in. WrappingKind string - - // AcceptJSON indicates if the input json bytes should be processed - AcceptJSON bool } -const ( - YAML = "yaml" - JSON = "json" -) - var _ Reader = &ByteReader{} func (r *ByteReader) Read() ([]*yaml.RNode, error) { @@ -124,27 +115,11 @@ func (r *ByteReader) Read() ([]*yaml.RNode, error) { if err != nil { return nil, errors.Wrap(err) } - inputStr := input.String() - - // check if json is accepted and if input bytes are in json format - if r.AcceptJSON && json.Valid([]byte(inputStr)) { - // convert json to yaml string to generate resource list object - // with appropriate format annotation - if r.SetAnnotations == nil { - r.SetAnnotations = map[string]string{} - } - if !r.OmitReaderAnnotations { - r.SetAnnotations[kioutil.FormatAnnotation] = JSON - } - } - - values := strings.Split(inputStr, "\n---\n") + values := strings.Split(input.String(), "\n---\n") index := 0 for i := range values { - value := values[i] - - decoder := yaml.NewDecoder(bytes.NewBufferString(value)) + decoder := yaml.NewDecoder(bytes.NewBufferString(values[i])) node, err := r.decode(index, decoder) if err == io.EOF { continue diff --git a/kyaml/kio/byteio_writer.go b/kyaml/kio/byteio_writer.go index 72b80331f..c6fe96db4 100644 --- a/kyaml/kio/byteio_writer.go +++ b/kyaml/kio/byteio_writer.go @@ -91,7 +91,7 @@ func (w ByteWriter) Write(nodes []*yaml.RNode) error { // don't wrap the elements if w.WrappingKind == "" { for i := range nodes { - if err := w.encode(encoder, nodes[i]); err != nil { + if err := w.encode(encoder, nodes[i].Document()); err != nil { return err } } @@ -125,26 +125,23 @@ func (w ByteWriter) Write(nodes []*yaml.RNode) error { for i := range nodes { items.Content = append(items.Content, nodes[i].YNode()) } - rNode := yaml.RNode{} - rNode.SetYNode(doc) - err := w.encode(encoder, &rNode) + err := w.encode(encoder, doc) yaml.UndoSerializationHacksOnNodes(nodes) return err } -// encode encodes the input RNode to appropriate node format based on file path extension -func (w ByteWriter) encode(encoder *yaml.Encoder, node *yaml.RNode) error { - _, _, format, err := kioutil.GetFileAnnotations(node) - if !w.KeepReaderAnnotations { - _, err := node.Pipe(yaml.ClearAnnotation(kioutil.FormatAnnotation)) - if err != nil { - return errors.Wrap(err) - } +// encode encodes the input document node to appropriate node format +func (w ByteWriter) encode(encoder *yaml.Encoder, doc *yaml.Node) error { + rNode := &yaml.RNode{} + rNode.SetYNode(doc) + str, err := rNode.String() + if err != nil { + return errors.Wrap(err) } - if err == nil && format == JSON { + if json.Valid([]byte(str)) { je := json.NewEncoder(w.Writer) je.SetIndent("", " ") - return errors.Wrap(je.Encode(node)) + return errors.Wrap(je.Encode(rNode)) } - return encoder.Encode(node.Document()) + return encoder.Encode(doc) } diff --git a/kyaml/kio/filters/testyaml/testyaml.go b/kyaml/kio/filters/testyaml/testyaml.go index 555d95da0..4052e5ffb 100644 --- a/kyaml/kio/filters/testyaml/testyaml.go +++ b/kyaml/kio/filters/testyaml/testyaml.go @@ -56,6 +56,16 @@ status2: - 2 `) -var FormattedJSON1 = []byte(`{"apiVersion": "example.com/v1beta1", "kind": "MyType", "spec": "a", "status": {"conditions": [ - 3, 1, 2]}} +var FormattedJSON1 = []byte(`{ + "apiVersion": "example.com/v1beta1", + "kind": "MyType", + "spec": "a", + "status": { + "conditions": [ + 3, + 1, + 2 + ] + } +} `) diff --git a/kyaml/kio/kioutil/kioutil.go b/kyaml/kio/kioutil/kioutil.go index 8839b7ca4..1d5e3bf58 100644 --- a/kyaml/kio/kioutil/kioutil.go +++ b/kyaml/kio/kioutil/kioutil.go @@ -22,20 +22,16 @@ const ( // PathAnnotation records the path to the file the Resource was read from PathAnnotation AnnotationKey = "config.kubernetes.io/path" - - // FormatAnnotation records the format of the resource ex: json - FormatAnnotation AnnotationKey = "config.kubernetes.io/format" ) -func GetFileAnnotations(rn *yaml.RNode) (string, string, string, error) { +func GetFileAnnotations(rn *yaml.RNode) (string, string, error) { meta, err := rn.GetMeta() if err != nil { - return "", "", "", err + return "", "", err } path := meta.Annotations[PathAnnotation] index := meta.Annotations[IndexAnnotation] - format := meta.Annotations[FormatAnnotation] - return path, index, format, nil + return path, index, nil } // ErrorIfMissingAnnotation validates the provided annotations are present on the given resources diff --git a/kyaml/kio/pkgio_reader.go b/kyaml/kio/pkgio_reader.go index 3b6fa9c3d..d0f4c0ee6 100644 --- a/kyaml/kio/pkgio_reader.go +++ b/kyaml/kio/pkgio_reader.go @@ -123,7 +123,7 @@ func (r *LocalPackageReadWriter) Write(nodes []*yaml.RNode) error { func (r *LocalPackageReadWriter) getFiles(nodes []*yaml.RNode) (sets.String, error) { val := sets.String{} for _, n := range nodes { - path, _, _, err := kioutil.GetFileAnnotations(n) + path, _, err := kioutil.GetFileAnnotations(n) if err != nil { return nil, errors.Wrap(err) } @@ -246,25 +246,10 @@ func (r *LocalPackageReader) readFile(path string, _ os.FileInfo) ([]*yaml.RNode Reader: f, OmitReaderAnnotations: r.OmitReaderAnnotations, SetAnnotations: r.SetAnnotations, - AcceptJSON: r.acceptJSON(path), } return rr.Read() } -// acceptJSON returns true if the input path has json ext and if the MatchFilesGlob -// has any of the JSONMatch ext -func (r *LocalPackageReader) acceptJSON(path string) bool { - for _, gm := range r.MatchFilesGlob { - for _, jm := range JSONMatch { - if filepath.Ext(path) == filepath.Ext(jm) && - filepath.Ext(gm) == filepath.Ext(jm) { - return true - } - } - } - return false -} - // ShouldSkipFile returns true if the file should be skipped func (r *LocalPackageReader) ShouldSkipFile(info os.FileInfo) (bool, error) { // check if the files are in scope diff --git a/kyaml/kio/pkgio_reader_test.go b/kyaml/kio/pkgio_reader_test.go index 2b730b768..62810277d 100644 --- a/kyaml/kio/pkgio_reader_test.go +++ b/kyaml/kio/pkgio_reader_test.go @@ -173,11 +173,10 @@ func TestLocalPackageReader_Read_JSON(t *testing.T) { } // TODO: Fix https://github.com/go-yaml/yaml/issues/44 so these are printed correctly expected := []string{ - `{"a": "b", metadata: {annotations: {config.kubernetes.io/format: 'json', config.kubernetes.io/index: '0', - config.kubernetes.io/path: 'a_test.json'}}} + `{"a": "b", metadata: {annotations: {config.kubernetes.io/index: '0', config.kubernetes.io/path: 'a_test.json'}}} `, - `{"e": "f", "g": {"h": ["i", "j"]}, metadata: {annotations: {config.kubernetes.io/format: 'json', - config.kubernetes.io/index: '0', config.kubernetes.io/path: 'b_test.json'}}} + `{"e": "f", "g": {"h": ["i", "j"]}, metadata: {annotations: {config.kubernetes.io/index: '0', + config.kubernetes.io/path: 'b_test.json'}}} `, } for i := range nodes { diff --git a/kyaml/runfn/runfn.go b/kyaml/runfn/runfn.go index 9b7c662ed..b6a5ce166 100644 --- a/kyaml/runfn/runfn.go +++ b/kyaml/runfn/runfn.go @@ -116,7 +116,7 @@ func (r RunFns) getNodesAndFilters() ( if r.Input == nil { p.Inputs = []kio.Reader{outputPkg} } else { - p.Inputs = []kio.Reader{&kio.ByteReader{Reader: r.Input, AcceptJSON: true}} + p.Inputs = []kio.Reader{&kio.ByteReader{Reader: r.Input}} } if err := p.Execute(); err != nil { return nil, nil, outputPkg, err diff --git a/kyaml/setters2/set.go b/kyaml/setters2/set.go index 8e08cd83d..c95b0f0bc 100644 --- a/kyaml/setters2/set.go +++ b/kyaml/setters2/set.go @@ -447,7 +447,7 @@ func SetAll(s *Set) kio.Filter { return nil, errors.Wrap(err) } if s.Count > preCount { - path, _, _, err := kioutil.GetFileAnnotations(nodes[i]) + path, _, err := kioutil.GetFileAnnotations(nodes[i]) if err != nil { return nil, errors.Wrap(err) } @@ -457,7 +457,7 @@ func SetAll(s *Set) kio.Filter { var nodesInUpdatedFiles []*yaml.RNode // return only the nodes whose corresponding file has at least one node with input setter for i := range nodes { - path, _, _, err := kioutil.GetFileAnnotations(nodes[i]) + path, _, err := kioutil.GetFileAnnotations(nodes[i]) if err != nil { return nil, errors.Wrap(err) } diff --git a/kyaml/yaml/types.go b/kyaml/yaml/types.go index d74cb8471..862a0ecc8 100644 --- a/kyaml/yaml/types.go +++ b/kyaml/yaml/types.go @@ -497,7 +497,6 @@ func String(node *yaml.Node, opts ...string) (string, error) { if node == nil { return "", nil } - DoSerializationHacks(node) optsSet := sets.String{} optsSet.Insert(opts...) if optsSet.Has(Flow) { @@ -516,7 +515,6 @@ func String(node *yaml.Node, opts ...string) (string, error) { if optsSet.Has(Trim) { val = strings.TrimSpace(val) } - UndoSerializationHacks(node) return val, errors.Wrap(err) }