From d0ae8fba13818efbd0f7d3ca82a2afed7c128b20 Mon Sep 17 00:00:00 2001 From: Katrina Verey Date: Wed, 17 Aug 2022 16:09:00 -0400 Subject: [PATCH] Internal copy of go-yaml at f6f7691b1fdeb513f56608cd2c32c51f8194bf51 --- .../yaml/.github/workflows/semgrep.yaml | 16 -- .../forked/github.com/go-yaml/yaml/decode.go | 78 +++++++-- .../github.com/go-yaml/yaml/decode_test.go | 125 ++++++++++++-- .../github.com/go-yaml/yaml/emitterc.go | 42 ++--- .../github.com/go-yaml/yaml/encode_test.go | 156 +----------------- .../go-yaml/yaml/example_embedded_test.go | 2 +- .../forked/github.com/go-yaml/yaml/go.mod | 5 + .../github.com/go-yaml/yaml/limit_test.go | 2 +- .../github.com/go-yaml/yaml/node_test.go | 2 +- .../forked/github.com/go-yaml/yaml/parserc.go | 11 +- .../forked/github.com/go-yaml/yaml/yaml.go | 10 -- .../forked/github.com/go-yaml/yaml/yamlh.go | 2 - 12 files changed, 214 insertions(+), 237 deletions(-) delete mode 100644 kyaml/internal/forked/github.com/go-yaml/yaml/.github/workflows/semgrep.yaml create mode 100644 kyaml/internal/forked/github.com/go-yaml/yaml/go.mod diff --git a/kyaml/internal/forked/github.com/go-yaml/yaml/.github/workflows/semgrep.yaml b/kyaml/internal/forked/github.com/go-yaml/yaml/.github/workflows/semgrep.yaml deleted file mode 100644 index 200c00a5a..000000000 --- a/kyaml/internal/forked/github.com/go-yaml/yaml/.github/workflows/semgrep.yaml +++ /dev/null @@ -1,16 +0,0 @@ ---- -name: Semgrep -on: [push, pull_request] -jobs: - semgrep: - name: semgrep - runs-on: ubuntu-20.04 - steps: - - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - name: Semgrep - id: semgrep - uses: returntocorp/semgrep-action@v1 - with: - config: p/dgryski.semgrep-go diff --git a/kyaml/internal/forked/github.com/go-yaml/yaml/decode.go b/kyaml/internal/forked/github.com/go-yaml/yaml/decode.go index df36e3a30..0173b6982 100644 --- a/kyaml/internal/forked/github.com/go-yaml/yaml/decode.go +++ b/kyaml/internal/forked/github.com/go-yaml/yaml/decode.go @@ -100,7 +100,10 @@ func (p *parser) peek() yaml_event_type_t { if p.event.typ != yaml_NO_EVENT { return p.event.typ } - if !yaml_parser_parse(&p.parser, &p.event) { + // It's curious choice from the underlying API to generally return a + // positive result on success, but on this case return true in an error + // scenario. This was the source of bugs in the past (issue #666). + if !yaml_parser_parse(&p.parser, &p.event) || p.parser.error != yaml_NO_ERROR { p.fail() } return p.event.typ @@ -320,6 +323,8 @@ type decoder struct { decodeCount int aliasCount int aliasDepth int + + mergedFields map[interface{}]bool } var ( @@ -808,6 +813,11 @@ func (d *decoder) mapping(n *Node, out reflect.Value) (good bool) { } } + mergedFields := d.mergedFields + d.mergedFields = nil + + var mergeNode *Node + mapIsNew := false if out.IsNil() { out.Set(reflect.MakeMap(outt)) @@ -815,11 +825,18 @@ func (d *decoder) mapping(n *Node, out reflect.Value) (good bool) { } for i := 0; i < l; i += 2 { if isMerge(n.Content[i]) { - d.merge(n.Content[i+1], out) + mergeNode = n.Content[i+1] continue } k := reflect.New(kt).Elem() if d.unmarshal(n.Content[i], k) { + if mergedFields != nil { + ki := k.Interface() + if mergedFields[ki] { + continue + } + mergedFields[ki] = true + } kkind := k.Kind() if kkind == reflect.Interface { kkind = k.Elem().Kind() @@ -833,6 +850,12 @@ func (d *decoder) mapping(n *Node, out reflect.Value) (good bool) { } } } + + d.mergedFields = mergedFields + if mergeNode != nil { + d.merge(n, mergeNode, out) + } + d.stringMapType = stringMapType d.generalMapType = generalMapType return true @@ -844,7 +867,8 @@ func isStringMap(n *Node) bool { } l := len(n.Content) for i := 0; i < l; i += 2 { - if n.Content[i].ShortTag() != strTag { + shortTag := n.Content[i].ShortTag() + if shortTag != strTag && shortTag != mergeTag { return false } } @@ -861,7 +885,6 @@ func (d *decoder) mappingStruct(n *Node, out reflect.Value) (good bool) { var elemType reflect.Type if sinfo.InlineMap != -1 { inlineMap = out.Field(sinfo.InlineMap) - inlineMap.Set(reflect.New(inlineMap.Type()).Elem()) elemType = inlineMap.Type().Elem() } @@ -870,6 +893,9 @@ func (d *decoder) mappingStruct(n *Node, out reflect.Value) (good bool) { d.prepare(n, field) } + mergedFields := d.mergedFields + d.mergedFields = nil + var mergeNode *Node var doneFields []bool if d.uniqueKeys { doneFields = make([]bool, len(sinfo.FieldsList)) @@ -879,13 +905,20 @@ func (d *decoder) mappingStruct(n *Node, out reflect.Value) (good bool) { for i := 0; i < l; i += 2 { ni := n.Content[i] if isMerge(ni) { - d.merge(n.Content[i+1], out) + mergeNode = n.Content[i+1] continue } if !d.unmarshal(ni, name) { continue } - if info, ok := sinfo.FieldsMap[name.String()]; ok { + sname := name.String() + if mergedFields != nil { + if mergedFields[sname] { + continue + } + mergedFields[sname] = true + } + if info, ok := sinfo.FieldsMap[sname]; ok { if d.uniqueKeys { if doneFields[info.Id] { d.terrors = append(d.terrors, fmt.Sprintf("line %d: field %s already set in type %s", ni.Line, name.String(), out.Type())) @@ -911,6 +944,11 @@ func (d *decoder) mappingStruct(n *Node, out reflect.Value) (good bool) { d.terrors = append(d.terrors, fmt.Sprintf("line %d: field %s not found in type %s", ni.Line, name.String(), out.Type())) } } + + d.mergedFields = mergedFields + if mergeNode != nil { + d.merge(n, mergeNode, out) + } return true } @@ -918,19 +956,29 @@ func failWantMap() { failf("map merge requires map or sequence of maps as the value") } -func (d *decoder) merge(n *Node, out reflect.Value) { - switch n.Kind { +func (d *decoder) merge(parent *Node, merge *Node, out reflect.Value) { + mergedFields := d.mergedFields + if mergedFields == nil { + d.mergedFields = make(map[interface{}]bool) + for i := 0; i < len(parent.Content); i += 2 { + k := reflect.New(ifaceType).Elem() + if d.unmarshal(parent.Content[i], k) { + d.mergedFields[k.Interface()] = true + } + } + } + + switch merge.Kind { case MappingNode: - d.unmarshal(n, out) + d.unmarshal(merge, out) case AliasNode: - if n.Alias != nil && n.Alias.Kind != MappingNode { + if merge.Alias != nil && merge.Alias.Kind != MappingNode { failWantMap() } - d.unmarshal(n, out) + d.unmarshal(merge, out) case SequenceNode: - // Step backwards as earlier nodes take precedence. - for i := len(n.Content) - 1; i >= 0; i-- { - ni := n.Content[i] + for i := 0; i < len(merge.Content); i++ { + ni := merge.Content[i] if ni.Kind == AliasNode { if ni.Alias != nil && ni.Alias.Kind != MappingNode { failWantMap() @@ -943,6 +991,8 @@ func (d *decoder) merge(n *Node, out reflect.Value) { default: failWantMap() } + + d.mergedFields = mergedFields } func isMerge(n *Node) bool { diff --git a/kyaml/internal/forked/github.com/go-yaml/yaml/decode_test.go b/kyaml/internal/forked/github.com/go-yaml/yaml/decode_test.go index 7b34fade2..0364b0bb0 100644 --- a/kyaml/internal/forked/github.com/go-yaml/yaml/decode_test.go +++ b/kyaml/internal/forked/github.com/go-yaml/yaml/decode_test.go @@ -26,7 +26,7 @@ import ( "time" . "gopkg.in/check.v1" - "sigs.k8s.io/kustomize/kyaml/internal/forked/github.com/go-yaml/yaml" + "gopkg.in/yaml.v3" ) var unmarshalIntTest = 123 @@ -767,7 +767,7 @@ var unmarshalTests = []struct { M{"a": 123456e1}, }, { "a: 123456E1\n", - M{"a": 123456E1}, + M{"a": 123456e1}, }, // yaml-test-suite 3GZX: Spec Example 7.1. Alias Nodes { @@ -802,7 +802,6 @@ var unmarshalTests = []struct { "c": []interface{}{"d", "e"}, }, }, - } type M map[string]interface{} @@ -948,16 +947,18 @@ var unmarshalErrorTests = []struct { {"%TAG !%79! tag:yaml.org,2002:\n---\nv: !%79!int '1'", "yaml: did not find expected whitespace"}, {"a:\n 1:\nb\n 2:", ".*could not find expected ':'"}, {"a: 1\nb: 2\nc 2\nd: 3\n", "^yaml: line 3: could not find expected ':'$"}, + {"#\n-\n{", "yaml: line 3: could not find expected ':'"}, // Issue #665 + {"0: [:!00 \xef", "yaml: incomplete UTF-8 octet sequence"}, // Issue #666 { "a: &a [00,00,00,00,00,00,00,00,00]\n" + - "b: &b [*a,*a,*a,*a,*a,*a,*a,*a,*a]\n" + - "c: &c [*b,*b,*b,*b,*b,*b,*b,*b,*b]\n" + - "d: &d [*c,*c,*c,*c,*c,*c,*c,*c,*c]\n" + - "e: &e [*d,*d,*d,*d,*d,*d,*d,*d,*d]\n" + - "f: &f [*e,*e,*e,*e,*e,*e,*e,*e,*e]\n" + - "g: &g [*f,*f,*f,*f,*f,*f,*f,*f,*f]\n" + - "h: &h [*g,*g,*g,*g,*g,*g,*g,*g,*g]\n" + - "i: &i [*h,*h,*h,*h,*h,*h,*h,*h,*h]\n", + "b: &b [*a,*a,*a,*a,*a,*a,*a,*a,*a]\n" + + "c: &c [*b,*b,*b,*b,*b,*b,*b,*b,*b]\n" + + "d: &d [*c,*c,*c,*c,*c,*c,*c,*c,*c]\n" + + "e: &e [*d,*d,*d,*d,*d,*d,*d,*d,*d]\n" + + "f: &f [*e,*e,*e,*e,*e,*e,*e,*e,*e]\n" + + "g: &g [*f,*f,*f,*f,*f,*f,*f,*f,*f]\n" + + "h: &h [*g,*g,*g,*g,*g,*g,*g,*g,*g]\n" + + "i: &i [*h,*h,*h,*h,*h,*h,*h,*h,*h]\n", "yaml: document contains excessive aliasing", }, } @@ -1391,7 +1392,7 @@ inlineSequenceMap: ` func (s *S) TestMerge(c *C) { - var want = map[interface{}]interface{}{ + var want = map[string]interface{}{ "x": 1, "y": 2, "r": 10, @@ -1436,7 +1437,103 @@ func (s *S) TestMergeStruct(c *C) { } } -var unmarshalNullTests = []struct{ input string; pristine, expected func() interface{} }{{ +var mergeTestsNested = ` +mergeouter1: &mergeouter1 + d: 40 + e: 50 + +mergeouter2: &mergeouter2 + e: 5 + f: 6 + g: 70 + +mergeinner1: &mergeinner1 + <<: *mergeouter1 + inner: + a: 1 + b: 2 + +mergeinner2: &mergeinner2 + <<: *mergeouter2 + inner: + a: -1 + b: -2 + +outer: + <<: [*mergeinner1, *mergeinner2] + f: 60 + inner: + a: 10 +` + +func (s *S) TestMergeNestedStruct(c *C) { + // Issue #818: Merging used to just unmarshal twice on the target + // value, which worked for maps as these were replaced by the new map, + // but not on struct values as these are preserved. This resulted in + // the nested data from the merged map to be mixed up with the data + // from the map being merged into. + // + // This test also prevents two potential bugs from showing up: + // + // 1) A simple implementation might just zero out the nested value + // before unmarshaling the second time, but this would clobber previous + // data that is usually respected ({C: 30} below). + // + // 2) A simple implementation might attempt to handle the key skipping + // directly by iterating over the merging map without recursion, but + // there are more complex cases that require recursion. + // + // Quick summary of the fields: + // + // - A must come from outer and not overriden + // - B must not be set as its in the ignored merge + // - C should still be set as it's preset in the value + // - D should be set from the recursive merge + // - E should be set from the first recursive merge, ignored on the second + // - F should be set in the inlined map from outer, ignored later + // - G should be set in the inlined map from the second recursive merge + // + + type Inner struct { + A, B, C int + } + type Outer struct { + D, E int + Inner Inner + Inline map[string]int `yaml:",inline"` + } + type Data struct { + Outer Outer + } + + test := Data{Outer{0, 0, Inner{C: 30}, nil}} + want := Data{Outer{40, 50, Inner{A: 10, C: 30}, map[string]int{"f": 60, "g": 70}}} + + err := yaml.Unmarshal([]byte(mergeTestsNested), &test) + c.Assert(err, IsNil) + c.Assert(test, DeepEquals, want) + + // Repeat test with a map. + + var testm map[string]interface{} + var wantm = map[string]interface {} { + "f": 60, + "inner": map[string]interface{}{ + "a": 10, + }, + "d": 40, + "e": 50, + "g": 70, + } + err = yaml.Unmarshal([]byte(mergeTestsNested), &testm) + c.Assert(err, IsNil) + c.Assert(testm["outer"], DeepEquals, wantm) +} + +var unmarshalNullTests = []struct { + input string + pristine, expected func() interface{} +}{{ "null", func() interface{} { var v interface{}; v = "v"; return &v }, func() interface{} { var v interface{}; v = nil; return &v }, @@ -1487,7 +1584,7 @@ func (s *S) TestUnmarshalNull(c *C) { func (s *S) TestUnmarshalPreservesData(c *C) { var v struct { A, B int - C int `yaml:"-"` + C int `yaml:"-"` } v.A = 42 v.C = 88 diff --git a/kyaml/internal/forked/github.com/go-yaml/yaml/emitterc.go b/kyaml/internal/forked/github.com/go-yaml/yaml/emitterc.go index f0f3d1867..0f47c9ca8 100644 --- a/kyaml/internal/forked/github.com/go-yaml/yaml/emitterc.go +++ b/kyaml/internal/forked/github.com/go-yaml/yaml/emitterc.go @@ -226,7 +226,7 @@ func yaml_emitter_append_tag_directive(emitter *yaml_emitter_t, value *yaml_tag_ } // Increase the indentation level. -func yaml_emitter_increase_indent(emitter *yaml_emitter_t, flow, indentless bool, compact_seq bool) bool { +func yaml_emitter_increase_indent(emitter *yaml_emitter_t, flow, indentless bool) bool { emitter.indents = append(emitter.indents, emitter.indent) if emitter.indent < 0 { if flow { @@ -243,9 +243,6 @@ func yaml_emitter_increase_indent(emitter *yaml_emitter_t, flow, indentless bool // Everything else aligns to the chosen indentation. emitter.indent = emitter.best_indent*((emitter.indent+emitter.best_indent)/emitter.best_indent) } - if compact_seq { - emitter.indent = emitter.indent - 2 - } } return true } @@ -491,7 +488,7 @@ func yaml_emitter_emit_document_content(emitter *yaml_emitter_t, event *yaml_eve if !yaml_emitter_emit_node(emitter, event, true, false, false, false) { return false } - if !yaml_emitter_process_line_comment(emitter, false) { + if !yaml_emitter_process_line_comment(emitter) { return false } if !yaml_emitter_process_foot_comment(emitter) { @@ -537,7 +534,7 @@ func yaml_emitter_emit_flow_sequence_item(emitter *yaml_emitter_t, event *yaml_e if !yaml_emitter_write_indicator(emitter, []byte{'['}, true, true, false) { return false } - if !yaml_emitter_increase_indent(emitter, true, false, false) { + if !yaml_emitter_increase_indent(emitter, true, false) { return false } emitter.flow_level++ @@ -560,7 +557,7 @@ func yaml_emitter_emit_flow_sequence_item(emitter *yaml_emitter_t, event *yaml_e if !yaml_emitter_write_indicator(emitter, []byte{']'}, false, false, false) { return false } - if !yaml_emitter_process_line_comment(emitter, false) { + if !yaml_emitter_process_line_comment(emitter) { return false } if !yaml_emitter_process_foot_comment(emitter) { @@ -605,7 +602,7 @@ func yaml_emitter_emit_flow_sequence_item(emitter *yaml_emitter_t, event *yaml_e return false } } - if !yaml_emitter_process_line_comment(emitter, false) { + if !yaml_emitter_process_line_comment(emitter) { return false } if !yaml_emitter_process_foot_comment(emitter) { @@ -620,7 +617,7 @@ func yaml_emitter_emit_flow_mapping_key(emitter *yaml_emitter_t, event *yaml_eve if !yaml_emitter_write_indicator(emitter, []byte{'{'}, true, true, false) { return false } - if !yaml_emitter_increase_indent(emitter, true, false, false) { + if !yaml_emitter_increase_indent(emitter, true, false) { return false } emitter.flow_level++ @@ -646,7 +643,7 @@ func yaml_emitter_emit_flow_mapping_key(emitter *yaml_emitter_t, event *yaml_eve if !yaml_emitter_write_indicator(emitter, []byte{'}'}, false, false, false) { return false } - if !yaml_emitter_process_line_comment(emitter, false) { + if !yaml_emitter_process_line_comment(emitter) { return false } if !yaml_emitter_process_foot_comment(emitter) { @@ -719,7 +716,7 @@ func yaml_emitter_emit_flow_mapping_value(emitter *yaml_emitter_t, event *yaml_e return false } } - if !yaml_emitter_process_line_comment(emitter, false) { + if !yaml_emitter_process_line_comment(emitter) { return false } if !yaml_emitter_process_foot_comment(emitter) { @@ -731,9 +728,7 @@ func yaml_emitter_emit_flow_mapping_value(emitter *yaml_emitter_t, event *yaml_e // Expect a block item node. func yaml_emitter_emit_block_sequence_item(emitter *yaml_emitter_t, event *yaml_event_t, first bool) bool { if first { - seq := emitter.mapping_context && (emitter.column == 0 || !emitter.indention) && - emitter.compact_sequence_indent - if !yaml_emitter_increase_indent(emitter, false, false, seq){ + if !yaml_emitter_increase_indent(emitter, false, false) { return false } } @@ -757,7 +752,7 @@ func yaml_emitter_emit_block_sequence_item(emitter *yaml_emitter_t, event *yaml_ if !yaml_emitter_emit_node(emitter, event, false, true, false, false) { return false } - if !yaml_emitter_process_line_comment(emitter, false) { + if !yaml_emitter_process_line_comment(emitter) { return false } if !yaml_emitter_process_foot_comment(emitter) { @@ -769,7 +764,7 @@ func yaml_emitter_emit_block_sequence_item(emitter *yaml_emitter_t, event *yaml_ // Expect a block key node. func yaml_emitter_emit_block_mapping_key(emitter *yaml_emitter_t, event *yaml_event_t, first bool) bool { if first { - if !yaml_emitter_increase_indent(emitter, false, false, false) { + if !yaml_emitter_increase_indent(emitter, false, false) { return false } } @@ -833,7 +828,7 @@ func yaml_emitter_emit_block_mapping_value(emitter *yaml_emitter_t, event *yaml_ } else if event.sequence_style() != yaml_FLOW_SEQUENCE_STYLE && (event.typ == yaml_MAPPING_START_EVENT || event.typ == yaml_SEQUENCE_START_EVENT) { // An indented block follows, so write the comment right now. emitter.line_comment, emitter.key_line_comment = emitter.key_line_comment, emitter.line_comment - if !yaml_emitter_process_line_comment(emitter, false) { + if !yaml_emitter_process_line_comment(emitter) { return false } emitter.line_comment, emitter.key_line_comment = emitter.key_line_comment, emitter.line_comment @@ -843,7 +838,7 @@ func yaml_emitter_emit_block_mapping_value(emitter *yaml_emitter_t, event *yaml_ if !yaml_emitter_emit_node(emitter, event, false, false, true, false) { return false } - if !yaml_emitter_process_line_comment(emitter, false) { + if !yaml_emitter_process_line_comment(emitter) { return false } if !yaml_emitter_process_foot_comment(emitter) { @@ -901,7 +896,7 @@ func yaml_emitter_emit_scalar(emitter *yaml_emitter_t, event *yaml_event_t) bool if !yaml_emitter_process_tag(emitter) { return false } - if !yaml_emitter_increase_indent(emitter, true, false, false) { + if !yaml_emitter_increase_indent(emitter, true, false) { return false } if !yaml_emitter_process_scalar(emitter) { @@ -1149,11 +1144,8 @@ func yaml_emitter_process_head_comment(emitter *yaml_emitter_t) bool { } // Write an line comment. -func yaml_emitter_process_line_comment(emitter *yaml_emitter_t, linebreak bool) bool { +func yaml_emitter_process_line_comment(emitter *yaml_emitter_t) bool { if len(emitter.line_comment) == 0 { - if linebreak && !put_break(emitter) { - return false - } return true } if !emitter.whitespace { @@ -1902,7 +1894,7 @@ func yaml_emitter_write_literal_scalar(emitter *yaml_emitter_t, value []byte) bo if !yaml_emitter_write_block_scalar_hints(emitter, value) { return false } - if !yaml_emitter_process_line_comment(emitter, true) { + if !yaml_emitter_process_line_comment(emitter) { return false } //emitter.indention = true @@ -1939,7 +1931,7 @@ func yaml_emitter_write_folded_scalar(emitter *yaml_emitter_t, value []byte) boo if !yaml_emitter_write_block_scalar_hints(emitter, value) { return false } - if !yaml_emitter_process_line_comment(emitter, true) { + if !yaml_emitter_process_line_comment(emitter) { return false } diff --git a/kyaml/internal/forked/github.com/go-yaml/yaml/encode_test.go b/kyaml/internal/forked/github.com/go-yaml/yaml/encode_test.go index b5bf34689..4a8bf2e26 100644 --- a/kyaml/internal/forked/github.com/go-yaml/yaml/encode_test.go +++ b/kyaml/internal/forked/github.com/go-yaml/yaml/encode_test.go @@ -27,7 +27,7 @@ import ( "os" . "gopkg.in/check.v1" - "sigs.k8s.io/kustomize/kyaml/internal/forked/github.com/go-yaml/yaml" + "gopkg.in/yaml.v3" ) var marshalIntTest = 123 @@ -35,142 +35,109 @@ var marshalIntTest = 123 var marshalTests = []struct { value interface{} data string - compact string }{ { nil, "null\n", - "null\n", }, { (*marshalerType)(nil), "null\n", - "null\n", }, { &struct{}{}, "{}\n", - "{}\n", }, { map[string]string{"v": "hi"}, "v: hi\n", - "v: hi\n", }, { map[string]interface{}{"v": "hi"}, "v: hi\n", - "v: hi\n", }, { map[string]string{"v": "true"}, "v: \"true\"\n", - "v: \"true\"\n", }, { map[string]string{"v": "false"}, "v: \"false\"\n", - "v: \"false\"\n", }, { map[string]interface{}{"v": true}, "v: true\n", - "v: true\n", }, { map[string]interface{}{"v": false}, "v: false\n", - "v: false\n", }, { map[string]interface{}{"v": 10}, "v: 10\n", - "v: 10\n", }, { map[string]interface{}{"v": -10}, "v: -10\n", - "v: -10\n", }, { map[string]uint{"v": 42}, "v: 42\n", - "v: 42\n", }, { map[string]interface{}{"v": int64(4294967296)}, "v: 4294967296\n", - "v: 4294967296\n", }, { map[string]int64{"v": int64(4294967296)}, "v: 4294967296\n", - "v: 4294967296\n", }, { map[string]uint64{"v": 4294967296}, "v: 4294967296\n", - "v: 4294967296\n", }, { map[string]interface{}{"v": "10"}, "v: \"10\"\n", - "v: \"10\"\n", }, { map[string]interface{}{"v": 0.1}, "v: 0.1\n", - "v: 0.1\n", }, { map[string]interface{}{"v": float64(0.1)}, "v: 0.1\n", - "v: 0.1\n", }, { map[string]interface{}{"v": float32(0.99)}, "v: 0.99\n", - "v: 0.99\n", }, { map[string]interface{}{"v": -0.1}, "v: -0.1\n", - "v: -0.1\n", }, { map[string]interface{}{"v": math.Inf(+1)}, "v: .inf\n", - "v: .inf\n", }, { map[string]interface{}{"v": math.Inf(-1)}, "v: -.inf\n", - "v: -.inf\n", }, { map[string]interface{}{"v": math.NaN()}, "v: .nan\n", - "v: .nan\n", }, { map[string]interface{}{"v": nil}, "v: null\n", - "v: null\n", }, { map[string]interface{}{"v": ""}, "v: \"\"\n", - "v: \"\"\n", }, { - map[string][]string{"v": {"A", "B"}}, + map[string][]string{"v": []string{"A", "B"}}, "v:\n - A\n - B\n", - "v:\n - A\n - B\n", }, { - map[string][]string{"v": {"A", "B\nC"}}, + map[string][]string{"v": []string{"A", "B\nC"}}, "v:\n - A\n - |-\n B\n C\n", - "v:\n - A\n - |-\n B\n C\n", }, { - map[string][]interface{}{"v": {"A", 1, map[string][]int{"B": {2, 3}}}}, + map[string][]interface{}{"v": []interface{}{"A", 1, map[string][]int{"B": []int{2, 3}}}}, "v:\n - A\n - 1\n - B:\n - 2\n - 3\n", - "v:\n - A\n - 1\n - B:\n - 2\n - 3\n", }, { map[string]interface{}{"a": map[interface{}]interface{}{"b": "c"}}, "a:\n b: c\n", - "a:\n b: c\n", }, { map[string]interface{}{"a": "-"}, "a: '-'\n", - "a: '-'\n", }, // Simple values. { &marshalIntTest, "123\n", - "123\n", }, // Structures { &struct{ Hello string }{"world"}, "hello: world\n", - "hello: world\n", }, { &struct { A struct { @@ -178,7 +145,6 @@ var marshalTests = []struct { } }{struct{ B string }{"c"}}, "a:\n b: c\n", - "a:\n b: c\n", }, { &struct { A *struct { @@ -186,7 +152,6 @@ var marshalTests = []struct { } }{&struct{ B string }{"c"}}, "a:\n b: c\n", - "a:\n b: c\n", }, { &struct { A *struct { @@ -194,37 +159,29 @@ var marshalTests = []struct { } }{}, "a: null\n", - "a: null\n", }, { &struct{ A int }{1}, "a: 1\n", - "a: 1\n", }, { &struct{ A []int }{[]int{1, 2}}, "a:\n - 1\n - 2\n", - "a:\n - 1\n - 2\n", }, { &struct{ A [2]int }{[2]int{1, 2}}, "a:\n - 1\n - 2\n", - "a:\n - 1\n - 2\n", }, { &struct { B int "a" }{1}, "a: 1\n", - "a: 1\n", }, { &struct{ A bool }{true}, "a: true\n", - "a: true\n", }, { &struct{ A string }{"true"}, "a: \"true\"\n", - "a: \"true\"\n", }, { &struct{ A string }{"off"}, "a: \"off\"\n", - "a: \"off\"\n", }, // Conditional flag @@ -234,51 +191,43 @@ var marshalTests = []struct { B int "b,omitempty" }{1, 0}, "a: 1\n", - "a: 1\n", }, { &struct { A int "a,omitempty" B int "b,omitempty" }{0, 0}, "{}\n", - "{}\n", }, { &struct { A *struct{ X, y int } "a,omitempty,flow" }{&struct{ X, y int }{1, 2}}, "a: {x: 1}\n", - "a: {x: 1}\n", }, { &struct { A *struct{ X, y int } "a,omitempty,flow" }{nil}, "{}\n", - "{}\n", }, { &struct { A *struct{ X, y int } "a,omitempty,flow" }{&struct{ X, y int }{}}, "a: {x: 0}\n", - "a: {x: 0}\n", }, { &struct { A struct{ X, y int } "a,omitempty,flow" }{struct{ X, y int }{1, 2}}, "a: {x: 1}\n", - "a: {x: 1}\n", }, { &struct { A struct{ X, y int } "a,omitempty,flow" }{struct{ X, y int }{0, 1}}, "{}\n", - "{}\n", }, { &struct { A float64 "a,omitempty" B float64 "b,omitempty" }{1, 0}, "a: 1\n", - "a: 1\n", }, { &struct { @@ -291,7 +240,6 @@ var marshalTests = []struct { T4: newTime(time.Date(2098, 1, 9, 10, 40, 47, 0, time.UTC)), }, "t2: 2018-01-09T10:40:47Z\nt4: 2098-01-09T10:40:47Z\n", - "t2: 2018-01-09T10:40:47Z\nt4: 2098-01-09T10:40:47Z\n", }, // Nil interface that implements Marshaler. { @@ -299,7 +247,6 @@ var marshalTests = []struct { "a": nil, }, "a: null\n", - "a: null\n", }, // Flow flag @@ -308,13 +255,11 @@ var marshalTests = []struct { A []int "a,flow" }{[]int{1, 2}}, "a: [1, 2]\n", - "a: [1, 2]\n", }, { &struct { A map[string]string "a,flow" }{map[string]string{"b": "c", "d": "e"}}, "a: {b: c, d: e}\n", - "a: {b: c, d: e}\n", }, { &struct { A struct { @@ -322,13 +267,11 @@ var marshalTests = []struct { } "a,flow" }{struct{ B, D string }{"c", "e"}}, "a: {b: c, d: e}\n", - "a: {b: c, d: e}\n", }, { &struct { A string "a,flow" }{"b\nc"}, "a: \"b\\nc\"\n", - "a: \"b\\nc\"\n", }, // Unexported field @@ -338,7 +281,6 @@ var marshalTests = []struct { A int }{0, 1}, "a: 1\n", - "a: 1\n", }, // Ignored field @@ -348,7 +290,6 @@ var marshalTests = []struct { B int "-" }{1, 2}, "a: 1\n", - "a: 1\n", }, // Struct inlining @@ -358,7 +299,6 @@ var marshalTests = []struct { C inlineB `yaml:",inline"` }{1, inlineB{2, inlineC{3}}}, "a: 1\nb: 2\nc: 3\n", - "a: 1\nb: 2\nc: 3\n", }, // Struct inlining as a pointer { @@ -367,21 +307,18 @@ var marshalTests = []struct { C *inlineB `yaml:",inline"` }{1, &inlineB{2, inlineC{3}}}, "a: 1\nb: 2\nc: 3\n", - "a: 1\nb: 2\nc: 3\n", }, { &struct { A int C *inlineB `yaml:",inline"` }{1, nil}, "a: 1\n", - "a: 1\n", }, { &struct { A int D *inlineD `yaml:",inline"` }{1, &inlineD{&inlineC{3}, 4}}, "a: 1\nc: 3\nd: 4\n", - "a: 1\nc: 3\nd: 4\n", }, // Map inlining @@ -391,21 +328,18 @@ var marshalTests = []struct { C map[string]int `yaml:",inline"` }{1, map[string]int{"b": 2, "c": 3}}, "a: 1\nb: 2\nc: 3\n", - "a: 1\nb: 2\nc: 3\n", }, // Duration { map[string]time.Duration{"a": 3 * time.Second}, "a: 3s\n", - "a: 3s\n", }, // Issue #24: bug in map merging logic. { map[string]string{"a": ""}, "a: \n", - "a: \n", }, // Issue #34: marshal unsupported base 60 floats quoted for compatibility @@ -413,110 +347,92 @@ var marshalTests = []struct { { map[string]string{"a": "1:1"}, "a: \"1:1\"\n", - "a: \"1:1\"\n", }, // Binary data. { map[string]string{"a": "\x00"}, "a: \"\\0\"\n", - "a: \"\\0\"\n", }, { map[string]string{"a": "\x80\x81\x82"}, "a: !!binary gIGC\n", - "a: !!binary gIGC\n", }, { map[string]string{"a": strings.Repeat("\x90", 54)}, "a: !!binary |\n " + strings.Repeat("kJCQ", 17) + "kJ\n CQ\n", - "a: !!binary |\n " + strings.Repeat("kJCQ", 17) + "kJ\n CQ\n", }, // Encode unicode as utf-8 rather than in escaped form. { map[string]string{"a": "你好"}, "a: 你好\n", - "a: 你好\n", }, // Support encoding.TextMarshaler. { map[string]net.IP{"a": net.IPv4(1, 2, 3, 4)}, "a: 1.2.3.4\n", - "a: 1.2.3.4\n", }, // time.Time gets a timestamp tag. { map[string]time.Time{"a": time.Date(2015, 2, 24, 18, 19, 39, 0, time.UTC)}, "a: 2015-02-24T18:19:39Z\n", - "a: 2015-02-24T18:19:39Z\n", }, { map[string]*time.Time{"a": newTime(time.Date(2015, 2, 24, 18, 19, 39, 0, time.UTC))}, "a: 2015-02-24T18:19:39Z\n", - "a: 2015-02-24T18:19:39Z\n", }, { // This is confirmed to be properly decoded in Python (libyaml) without a timestamp tag. map[string]time.Time{"a": time.Date(2015, 2, 24, 18, 19, 39, 123456789, time.FixedZone("FOO", -3*60*60))}, "a: 2015-02-24T18:19:39.123456789-03:00\n", - "a: 2015-02-24T18:19:39.123456789-03:00\n", }, // Ensure timestamp-like strings are quoted. { map[string]string{"a": "2015-02-24T18:19:39Z"}, "a: \"2015-02-24T18:19:39Z\"\n", - "a: \"2015-02-24T18:19:39Z\"\n", }, // Ensure strings containing ": " are quoted (reported as PR #43, but not reproducible). { map[string]string{"a": "b: c"}, "a: 'b: c'\n", - "a: 'b: c'\n", }, // Containing hash mark ('#') in string should be quoted { map[string]string{"a": "Hello #comment"}, "a: 'Hello #comment'\n", - "a: 'Hello #comment'\n", }, { map[string]string{"a": "你好 #comment"}, "a: '你好 #comment'\n", - "a: '你好 #comment'\n", }, // Ensure MarshalYAML also gets called on the result of MarshalYAML itself. { &marshalerType{marshalerType{true}}, "true\n", - "true\n", }, { &marshalerType{&marshalerType{true}}, "true\n", - "true\n", }, // Check indentation of maps inside sequences inside maps. { map[string]interface{}{"a": map[string]interface{}{"b": []map[string]int{{"c": 1, "d": 2}}}}, "a:\n b:\n - c: 1\n d: 2\n", - "a:\n b:\n - c: 1\n d: 2\n", }, // Strings with tabs were disallowed as literals (issue #471). { map[string]string{"a": "\tB\n\tC\n"}, "a: |\n \tB\n \tC\n", - "a: |\n \tB\n \tC\n", }, // Ensure that strings do not wrap { map[string]string{"a": "abcdefghijklmnopqrstuvwxyz ABCDEFGHIJKLMNOPQRSTUVWXYZ 1234567890 abcdefghijklmnopqrstuvwxyz ABCDEFGHIJKLMNOPQRSTUVWXYZ 1234567890 "}, "a: 'abcdefghijklmnopqrstuvwxyz ABCDEFGHIJKLMNOPQRSTUVWXYZ 1234567890 abcdefghijklmnopqrstuvwxyz ABCDEFGHIJKLMNOPQRSTUVWXYZ 1234567890 '\n", - "a: 'abcdefghijklmnopqrstuvwxyz ABCDEFGHIJKLMNOPQRSTUVWXYZ 1234567890 abcdefghijklmnopqrstuvwxyz ABCDEFGHIJKLMNOPQRSTUVWXYZ 1234567890 '\n", }, // yaml.Node @@ -532,7 +448,6 @@ var marshalTests = []struct { }, }, "value: 'foo'\n", - "value: 'foo'\n", }, { yaml.Node{ Kind: yaml.ScalarNode, @@ -541,7 +456,6 @@ var marshalTests = []struct { Style: yaml.SingleQuotedStyle, }, "'foo'\n", - "'foo'\n", }, // Enforced tagging with shorthand notation (issue #616). @@ -557,7 +471,6 @@ var marshalTests = []struct { }, }, "value: !!str foo\n", - "value: !!str foo\n", }, { &struct { Value yaml.Node @@ -569,7 +482,6 @@ var marshalTests = []struct { }, }, "value: !!map {}\n", - "value: !!map {}\n", }, { &struct { Value yaml.Node @@ -581,7 +493,6 @@ var marshalTests = []struct { }, }, "value: !!seq []\n", - "value: !!seq []\n", }, } @@ -596,20 +507,6 @@ func (s *S) TestMarshal(c *C) { } } -func (s *S) TestEncoderCompactIndents(c *C) { - for i, item := range marshalTests { - c.Logf("test %d. %q", i, item.data) - var buf bytes.Buffer - enc := yaml.NewEncoder(&buf) - enc.CompactSeqIndent() - err := enc.Encode(item.value) - c.Assert(err, Equals, nil) - err = enc.Close() - c.Assert(err, Equals, nil) - c.Assert(buf.String(), Equals, item.compact) - } -} - func (s *S) TestEncoderSingleDocument(c *C) { for i, item := range marshalTests { c.Logf("test %d. %q", i, item.data) @@ -759,51 +656,6 @@ func (s *S) TestSetIndent(c *C) { c.Assert(buf.String(), Equals, "a:\n b:\n c: d\n") } -func (s *S) TestCompactSeqIndentDefault(c *C) { - var buf bytes.Buffer - enc := yaml.NewEncoder(&buf) - enc.CompactSeqIndent() - err := enc.Encode(map[string]interface{}{"a": []string{"b", "c"}}) - c.Assert(err, Equals, nil) - err = enc.Close() - c.Assert(err, Equals, nil) - // The default indent is 4, so these sequence elements get 2 indents as before - c.Assert(buf.String(), Equals, `a: - - b - - c -`) -} - -func (s *S) TestCompactSequenceWithSetIndent(c *C) { - var buf bytes.Buffer - enc := yaml.NewEncoder(&buf) - enc.CompactSeqIndent() - enc.SetIndent(2) - err := enc.Encode(map[string]interface{}{"a": []string{"b", "c"}}) - c.Assert(err, Equals, nil) - err = enc.Close() - c.Assert(err, Equals, nil) - // The sequence indent is 2, so these sequence elements don't get indented at all - c.Assert(buf.String(), Equals, `a: -- b -- c -`) -} - -func (s *S) TestNewLinePreserved(c *C) { - obj := &marshalerValue{} - obj.Field.value = "a:\n b:\n c: d\n" - data, err := yaml.Marshal(obj) - c.Assert(err, IsNil) - c.Assert(string(data), Equals, "_: |\n a:\n b:\n c: d\n") - - obj.Field.value = "\na:\n b:\n c: d\n" - data, err = yaml.Marshal(obj) - c.Assert(err, IsNil) - // the newline at the start of the file should be preserved - c.Assert(string(data), Equals, "_: |4\n\n a:\n b:\n c: d\n") -} - func (s *S) TestSortedOutput(c *C) { order := []interface{}{ false, diff --git a/kyaml/internal/forked/github.com/go-yaml/yaml/example_embedded_test.go b/kyaml/internal/forked/github.com/go-yaml/yaml/example_embedded_test.go index 82b2c073e..9d17398a5 100644 --- a/kyaml/internal/forked/github.com/go-yaml/yaml/example_embedded_test.go +++ b/kyaml/internal/forked/github.com/go-yaml/yaml/example_embedded_test.go @@ -19,7 +19,7 @@ import ( "fmt" "log" - "sigs.k8s.io/kustomize/kyaml/internal/forked/github.com/go-yaml/yaml" + "gopkg.in/yaml.v3" ) // An example showing how to unmarshal embedded diff --git a/kyaml/internal/forked/github.com/go-yaml/yaml/go.mod b/kyaml/internal/forked/github.com/go-yaml/yaml/go.mod new file mode 100644 index 000000000..f407ea321 --- /dev/null +++ b/kyaml/internal/forked/github.com/go-yaml/yaml/go.mod @@ -0,0 +1,5 @@ +module "gopkg.in/yaml.v3" + +require ( + "gopkg.in/check.v1" v0.0.0-20161208181325-20d25e280405 +) diff --git a/kyaml/internal/forked/github.com/go-yaml/yaml/limit_test.go b/kyaml/internal/forked/github.com/go-yaml/yaml/limit_test.go index 9f6441bf9..07a3cbd4b 100644 --- a/kyaml/internal/forked/github.com/go-yaml/yaml/limit_test.go +++ b/kyaml/internal/forked/github.com/go-yaml/yaml/limit_test.go @@ -5,7 +5,7 @@ import ( "testing" . "gopkg.in/check.v1" - "sigs.k8s.io/kustomize/kyaml/internal/forked/github.com/go-yaml/yaml" + "gopkg.in/yaml.v3" ) var limitTests = []struct { diff --git a/kyaml/internal/forked/github.com/go-yaml/yaml/node_test.go b/kyaml/internal/forked/github.com/go-yaml/yaml/node_test.go index bfed08c96..b7d0c9a30 100644 --- a/kyaml/internal/forked/github.com/go-yaml/yaml/node_test.go +++ b/kyaml/internal/forked/github.com/go-yaml/yaml/node_test.go @@ -21,7 +21,7 @@ import ( "os" . "gopkg.in/check.v1" - "sigs.k8s.io/kustomize/kyaml/internal/forked/github.com/go-yaml/yaml" + "gopkg.in/yaml.v3" "io" "strings" ) diff --git a/kyaml/internal/forked/github.com/go-yaml/yaml/parserc.go b/kyaml/internal/forked/github.com/go-yaml/yaml/parserc.go index ac66fccc0..268558a0d 100644 --- a/kyaml/internal/forked/github.com/go-yaml/yaml/parserc.go +++ b/kyaml/internal/forked/github.com/go-yaml/yaml/parserc.go @@ -687,6 +687,9 @@ func yaml_parser_parse_node(parser *yaml_parser_t, event *yaml_event_t, block, i func yaml_parser_parse_block_sequence_entry(parser *yaml_parser_t, event *yaml_event_t, first bool) bool { if first { token := peek_token(parser) + if token == nil { + return false + } parser.marks = append(parser.marks, token.start_mark) skip_token(parser) } @@ -786,7 +789,7 @@ func yaml_parser_split_stem_comment(parser *yaml_parser_t, stem_len int) { } token := peek_token(parser) - if token.typ != yaml_BLOCK_SEQUENCE_START_TOKEN && token.typ != yaml_BLOCK_MAPPING_START_TOKEN { + if token == nil || token.typ != yaml_BLOCK_SEQUENCE_START_TOKEN && token.typ != yaml_BLOCK_MAPPING_START_TOKEN { return } @@ -813,6 +816,9 @@ func yaml_parser_split_stem_comment(parser *yaml_parser_t, stem_len int) { func yaml_parser_parse_block_mapping_key(parser *yaml_parser_t, event *yaml_event_t, first bool) bool { if first { token := peek_token(parser) + if token == nil { + return false + } parser.marks = append(parser.marks, token.start_mark) skip_token(parser) } @@ -922,6 +928,9 @@ func yaml_parser_parse_block_mapping_value(parser *yaml_parser_t, event *yaml_ev func yaml_parser_parse_flow_sequence_entry(parser *yaml_parser_t, event *yaml_event_t, first bool) bool { if first { token := peek_token(parser) + if token == nil { + return false + } parser.marks = append(parser.marks, token.start_mark) skip_token(parser) } diff --git a/kyaml/internal/forked/github.com/go-yaml/yaml/yaml.go b/kyaml/internal/forked/github.com/go-yaml/yaml/yaml.go index bb6418dba..8cec6da48 100644 --- a/kyaml/internal/forked/github.com/go-yaml/yaml/yaml.go +++ b/kyaml/internal/forked/github.com/go-yaml/yaml/yaml.go @@ -278,16 +278,6 @@ func (e *Encoder) SetIndent(spaces int) { e.encoder.indent = spaces } -// CompactSeqIndent makes it so that '- ' is considered part of the indentation. -func (e *Encoder) CompactSeqIndent() { - e.encoder.emitter.compact_sequence_indent = true -} - -// DefaultSeqIndent makes it so that '- ' is not considered part of the indentation. -func (e *Encoder) DefaultSeqIndent() { - e.encoder.emitter.compact_sequence_indent = false -} - // Close closes the encoder by writing any remaining data. // It does not write a stream terminating string "...". func (e *Encoder) Close() (err error) { diff --git a/kyaml/internal/forked/github.com/go-yaml/yaml/yamlh.go b/kyaml/internal/forked/github.com/go-yaml/yaml/yamlh.go index 40c74de49..7c6d00770 100644 --- a/kyaml/internal/forked/github.com/go-yaml/yaml/yamlh.go +++ b/kyaml/internal/forked/github.com/go-yaml/yaml/yamlh.go @@ -742,8 +742,6 @@ type yaml_emitter_t struct { indent int // The current indentation level. - compact_sequence_indent bool // Is '- ' is considered part of the indentation for sequence elements? - flow_level int // The current flow level. root_context bool // Is it the document root context?