openapi parsing performance improvement with protobuffer (#4568)

* update necessary dependencies

* update openapi test structure

* remove old swagger files and generate new ones

* use protobuffer to parse openapi for performance improvement

kyaml/openapi/kubernetesapi/v1218pb/doc.go

@@ -1,6 +0,0 @@
-// Copyright 2022 The Kubernetes Authors.
-// SPDX-License-Identifier: Apache-2.0
-
-package v1218pb
-
-const OpenAPIProtobuf = "v1218pb"

kyaml/openapi/openapi.go

@@ -11,6 +11,8 @@ import (
 	"reflect"
 	"strings"
 
+	openapi_v2 "github.com/google/gnostic/openapiv2"
+	"google.golang.org/protobuf/proto"
 	"k8s.io/kube-openapi/pkg/validation/spec"
 	"sigs.k8s.io/kustomize/kyaml/errors"
 	"sigs.k8s.io/kustomize/kyaml/openapi/kubernetesapi"
@@ -50,6 +52,13 @@ type openapiData struct {
 	schemaInit bool
 }
 
+type format string
+
+const (
+	JsonOrYaml format = "jsonOrYaml"
+	Proto      format = "proto"
+)
+
 // precomputedIsNamespaceScoped precomputes IsNamespaceScoped for known types. This avoids Schema creation,
 // which is expensive
 // The test output from TestIsNamespaceScopedPrecompute shows the expected map in go syntax,and can be copy and pasted
@@ -264,12 +273,14 @@ func schemaUsingField(object *yaml.RNode, field string) (*spec.Schema, error) {
 
 // AddSchema parses s, and adds definitions from s to the global schema.
 func AddSchema(s []byte) error {
-	return parse(s)
+	return parse(s, JsonOrYaml)
 }
 
 // ResetOpenAPI resets the openapi data to empty
 func ResetOpenAPI() {
 	globalSchema = openapiData{}
+	kubernetesOpenAPIVersion = ""
+	customSchema = nil
 }
 
 // AddDefinitions adds the definitions to the global schema.
@@ -592,26 +603,27 @@ func initSchema() {
 	}
 	globalSchema.schemaInit = true
 
+	// TODO(natasha41575): Accept proto-formatted schema files
 	if customSchema != nil {
-		err := parse(customSchema)
+		err := parse(customSchema, JsonOrYaml)
 		if err != nil {
 			panic("invalid schema file")
 		}
-		if err = parse(kustomizationapi.MustAsset(kustomizationAPIAssetName)); err != nil {
-			// this should never happen
-			panic(err)
+	} else {
+		if kubernetesOpenAPIVersion == "" {
+			parseBuiltinSchema(kubernetesOpenAPIDefaultVersion)
+		} else {
+			parseBuiltinSchema(kubernetesOpenAPIVersion)
 		}
-		return
 	}
 
-	if kubernetesOpenAPIVersion == "" {
-		parseBuiltinSchema(kubernetesOpenAPIDefaultVersion)
-	} else {
-		parseBuiltinSchema(kubernetesOpenAPIVersion)
+	if err := parse(kustomizationapi.MustAsset(kustomizationAPIAssetName), JsonOrYaml); err != nil {
+		// this should never happen
+		panic(err)
 	}
 }
 
-// parseBuiltinSchema calls parse to parse the json schemas
+// parseBuiltinSchema calls parse to parse the json or proto schemas
 func parseBuiltinSchema(version string) {
 	if globalSchema.noUseBuiltInSchema {
 		// don't parse the built in schema
@@ -622,36 +634,45 @@ func parseBuiltinSchema(version string) {
 	assetName := filepath.Join(
 		"kubernetesapi",
 		version,
-		"swagger.json")
+		"swagger.pb")
 
-	if err := parse(kubernetesapi.OpenAPIMustAsset[version](assetName)); err != nil {
-		// this should never happen
-		panic(err)
-	}
-
-	if err := parse(kustomizationapi.MustAsset(kustomizationAPIAssetName)); err != nil {
+	if err := parse(kubernetesapi.OpenAPIMustAsset[version](assetName), Proto); err != nil {
 		// this should never happen
 		panic(err)
 	}
 }
 
-// parse parses and indexes a single json schema
-func parse(b []byte) error {
+// parse parses and indexes a single json or proto schema
+func parse(b []byte, format format) error {
 	var swagger spec.Swagger
-	s := string(b)
-	if len(s) > 0 && s[0] != '{' {
-		var err error
-		b, err = k8syaml.YAMLToJSON(b)
+	switch {
+	case format == Proto:
+		doc := &openapi_v2.Document{}
+		// We parse protobuf and get an openapi_v2.Document here.
+		if err := proto.Unmarshal(b, doc); err != nil {
+			return fmt.Errorf("openapi proto unmarshalling failed: %w", err)
+		}
+		// convert the openapi_v2.Document back to Swagger
+		_, err := swagger.FromGnostic(doc)
 		if err != nil {
 			return errors.Wrap(err)
 		}
+
+	case format == JsonOrYaml:
+		if len(b) > 0 && b[0] != byte('{') {
+			var err error
+			b, err = k8syaml.YAMLToJSON(b)
+			if err != nil {
+				return errors.Wrap(err)
+			}
+		}
+		if err := swagger.UnmarshalJSON(b); err != nil {
+			return errors.Wrap(err)
+		}
 	}
-	if err := swagger.UnmarshalJSON(b); err != nil {
-		return errors.Wrap(err)
-	}
+
 	AddDefinitions(swagger.Definitions)
 	findNamespaceability(swagger.Paths)
-
 	return nil
 }
 
@@ -695,6 +716,9 @@ func findNamespaceability(paths *spec.Paths) {
 }
 
 func resolve(root interface{}, ref *spec.Ref) (*spec.Schema, error) {
+	if s, ok := root.(*spec.Schema); ok && s == nil {
+		return nil, nil
+	}
 	res, _, err := ref.GetPointer().Get(root)
 	if err != nil {
 		return nil, errors.Wrap(err)

kyaml/openapi/openapi_benchmark_test.go

@@ -9,56 +9,16 @@ import (
 
 	openapi_v2 "github.com/google/gnostic/openapiv2"
 	"google.golang.org/protobuf/proto"
-	"k8s.io/kube-openapi/pkg/validation/spec"
-	"sigs.k8s.io/kustomize/kyaml/openapi/kubernetesapi"
-	"sigs.k8s.io/kustomize/kyaml/openapi/kubernetesapi/v1218pb"
+	v1212 "sigs.k8s.io/kustomize/kyaml/openapi/kubernetesapi/v1212"
 )
 
-// Benchmark for swagger parsing (UnmarshalJSON)
-func BenchmarkSwaggerUnmarshalJSON(t *testing.B) {
-	version := kubernetesOpenAPIDefaultVersion
-
-	// parse the swagger, this should never fail
-	assetName := filepath.Join(
-		"kubernetesapi",
-		version,
-		"swagger.json")
-
-	b := kubernetesapi.OpenAPIMustAsset[version](assetName)
-
-	for i := 0; i < t.N; i++ {
-		var swagger spec.Swagger
-		if err := swagger.UnmarshalJSON(b); err != nil {
-			t.Fatalf("swagger.UnmarshalJSON failed: %v", err)
-		}
-	}
-}
-
-func BenchmarkOpenAPIV2ParseDocument(t *testing.B) {
-	version := kubernetesOpenAPIDefaultVersion
-
-	assetName := filepath.Join(
-		"kubernetesapi",
-		version,
-		"swagger.json")
-
-	b := kubernetesapi.OpenAPIMustAsset[version](assetName)
-
-	for i := 0; i < t.N; i++ {
-		// We parse JSON and get an openapiv2.Document here.
-		if _, err := openapi_v2.ParseDocument(b); err != nil {
-			t.Fatalf("openapi_v2.ParseDocument failed: %v", err)
-		}
-	}
-}
-
 func BenchmarkProtoUnmarshal(t *testing.B) {
 	assetName := filepath.Join(
 		"kubernetesapi",
-		"v1218pb",
+		"v1212",
 		"swagger.pb")
 
-	b := v1218pb.MustAsset(assetName)
+	b := v1212.MustAsset(assetName)
 
 	for i := 0; i < t.N; i++ {
 		// We parse protobuf and get an openapiv2.Document here.
@@ -67,18 +27,3 @@ func BenchmarkProtoUnmarshal(t *testing.B) {
 		}
 	}
 }
-
-// Benchmark for loading assets packed into the binary
-func BenchmarkAssetRead(t *testing.B) {
-	for i := 0; i < t.N; i++ {
-		version := kubernetesOpenAPIDefaultVersion
-
-		// parse the swagger, this should never fail
-		assetName := filepath.Join(
-			"kubernetesapi",
-			version,
-			"swagger.json")
-
-		kubernetesapi.OpenAPIMustAsset[version](assetName)
-	}
-}