update openapi scripts to fetch protobuffer schemas (#4582)

* update openapi scripts to fetch protobuffer schema

* code review

* code review

* restore makeOpenApiInfoDotGo.sh

* code review

api/krusty/openapitests/openapicustomschema_test.go

@@ -421,7 +421,7 @@ spec:
       - image: nginx
         name: server
 `)
-		assert.Equal(t, "v1212", openapi.GetSchemaVersion())
+		assert.Equal(t, "v1.21.2", openapi.GetSchemaVersion())
 	})
 }
 
@@ -580,7 +580,7 @@ spec:
       containers:
       - image: whatever
 `)
-		assert.Equal(t, "v1212", openapi.GetSchemaVersion())
+		assert.Equal(t, "v1.21.2", openapi.GetSchemaVersion())
 	})
 }
 

kyaml/openapi/Makefile

@@ -5,8 +5,8 @@ MYGOBIN = $(shell go env GOBIN)
 ifeq ($(MYGOBIN),)
 MYGOBIN = $(shell go env GOPATH)/bin
 endif
-API_VERSION := "v1.21.2"
 KIND_VERSION := "v0.11.1"
+API_VERSION ?= "v1.21.2"
 
 .PHONY: all
 all: \
@@ -28,7 +28,7 @@ nuke: clean
 	rm -r kubernetesapi/*
 
 $(MYGOBIN)/go-bindata:
-	go install github.com/go-bindata/go-bindata/v3/go-bindata
+	go install github.com/go-bindata/go-bindata/v3/go-bindata@latest
 
 $(MYGOBIN)/kind:
 	( \
@@ -40,22 +40,22 @@ $(MYGOBIN)/kind:
 		rm -rf $$d; \
 	)
 
+.PHONY: kubernetesapi/openapiinfo.go
+kubernetesapi/openapiinfo.go:
+	./scripts/makeOpenApiInfoDotGo.sh
+
 kustomizationapi/swagger.go: $(MYGOBIN)/go-bindata kustomizationapi/swagger.json
 	$(MYGOBIN)/go-bindata \
 		--pkg kustomizationapi \
 		-o kustomizationapi/swagger.go \
 		kustomizationapi/swagger.json
 
-.PHONY: kubernetesapi/openapiinfo.go
-kubernetesapi/openapiinfo.go:
-	./scripts/makeOpenApiInfoDotGo.sh
-
-.PHONY: kubernetesapi/swagger.json
-kubernetesapi/swagger.json: $(MYGOBIN)/kind $(MYGOBIN)/kustomize
+.PHONY: kubernetesapi/swagger.pb
+kubernetesapi/swagger.pb: $(MYGOBIN)/kind $(MYGOBIN)/kustomize
 	./scripts/fetchSchemaFromCluster.sh $(API_VERSION)
 
 .PHONY: kubernetesapi/swagger.go
-kubernetesapi/swagger.go: $(MYGOBIN)/go-bindata kubernetesapi/swagger.json
+kubernetesapi/swagger.go: $(MYGOBIN)/go-bindata kubernetesapi/swagger.pb
 	./scripts/generateSwaggerDotGo.sh $(API_VERSION)
 
 $(MYGOBIN)/kustomize:

kyaml/openapi/README.md

@@ -22,22 +22,31 @@ make nuke
 
 The compiled-in schema version should maximize API availability with respect to all actively supported Kubernetes versions. For example, while 1.20, 1.21 and 1.22 are the actively supported versions, 1.21 is the best choice. This is because 1.21 introduces at least one new API and does not remove any, while 1.22 removes a large set of long-deprecated APIs that are still supported in 1.20/1.21.
 
-### Update the built-in schema to a new version
+### Generating additional schema
 
-In the Makefile in this directory, update the `API_VERSION` to your desired version.
+If you'd like to change the default schema version, then in the Makefile in this directory, update the `API_VERSION` to your desired version.
 
 You may need to update the version of Kind these scripts use by changing `KIND_VERSION` in the Makefile in this directory. You can find compatibility information in the [kind release notes](https://github.com/kubernetes-sigs/kind/releases).
 
-In this directory, fetch the openapi schema and generate the 
-corresponding swagger.go for the kubernetes api: 
+In this directory, fetch the openapi schema, generate the
+corresponding swagger.go for the kubernetes api, and update `kubernetesapi/openapiinfo.go`:
 
 ```
 make all
 ```
 
-The above command will update the [OpenAPI schema] and the [Kustomization schema]. It will
-create a directory kubernetesapi/v1212 and store the resulting
-swagger.json and swagger.go files there.
+If you want to run the steps individually instead of using `make all`, you can run
+the following commands:
+
+```
+make kustomizationapi/swagger.go
+make kubernetesapi/swagger.go
+make kubernetesapi/openapiinfo.go
+```
+
+You can optionally delete the old `swagger.pb` and `swagger.go` files if we no longer need to support that kubernetes version of
+openapi data. Make sure you rerun `make kubernetesapi/openapiinfo.go` after deleting any old schemas.
+
 
 #### Precomputations
 
@@ -55,25 +64,6 @@ make prow-presubmit-check >& /tmp/k.txt; echo $?
 
 The exit code should be zero; if not, examine `/tmp/k.txt`.
 
-## Generating additional schemas
-
-Instead of replacing the default version, you can specify a desired version as part of the make invocation:
-
-```
-rm kubernetesapi/swagger.go
-make kubernetesapi/swagger.go API_VERSION=v1.21.2
-```
-
-While the above commands generate the swagger.go files, they
-do not make them available for use nor do they update the
-info field reported by `kustomize openapi info`. To make the
-newly fetched schema and swagger.go available:
-
-```
-rm kubernetesapi/openapiinfo.go
-make kubernetesapi/openapiinfo.go
-```
-
 ## Partial regeneration
 
 You can also regenerate the kubernetes api schemas specifically with:
@@ -87,8 +77,8 @@ To fetch the schema without generating the swagger.go, you can
 run:
 
 ```
-rm kubernetesapi/swagger.json
-make kubernetesapi/swagger.json
+rm kubernetesapi/swagger.pb
+make kubernetesapi/swagger.pb
 ```
 
 Note that generating the swagger.go will re-fetch the schema.

kyaml/openapi/kubernetesapi/openapiinfo.go

@@ -6,13 +6,13 @@
 package kubernetesapi
 
 import (
-	"sigs.k8s.io/kustomize/kyaml/openapi/kubernetesapi/v1212"
+	"sigs.k8s.io/kustomize/kyaml/openapi/kubernetesapi/v1_21_2"
 )
 
 const Info = "{title:Kubernetes,version:v1.21.2}"
 
 var OpenAPIMustAsset = map[string]func(string) []byte{
-	"v1212": v1212.MustAsset,
+	"v1.21.2": v1_21_2.MustAsset,
 }
 
-const DefaultOpenAPI = "v1212"
+const DefaultOpenAPI = "v1.21.2"

kyaml/openapi/openapi.go

@@ -400,7 +400,7 @@ func SuppressBuiltInSchemaUse() {
 
 // Elements returns the Schema for the elements of an array.
 func (rs *ResourceSchema) Elements() *ResourceSchema {
-	// load the schema from swagger.json
+	// load the schema from swagger files
 	initSchema()
 
 	if len(rs.Schema.Type) != 1 || rs.Schema.Type[0] != "array" {
@@ -446,7 +446,7 @@ func (rs *ResourceSchema) Lookup(path ...string) *ResourceSchema {
 
 // Field returns the Schema for a field.
 func (rs *ResourceSchema) Field(field string) *ResourceSchema {
-	// load the schema from swagger.json
+	// load the schema from swagger files
 	initSchema()
 
 	// locate the Schema
@@ -459,7 +459,7 @@ func (rs *ResourceSchema) Field(field string) *ResourceSchema {
 		// (the key doesn't matter, they all have the same value type)
 		s = *rs.Schema.AdditionalProperties.Schema
 	default:
-		// no Schema found from either swagger.json or line comments
+		// no Schema found from either swagger files or line comments
 		return nil
 	}
 
@@ -557,12 +557,13 @@ func SetSchema(openAPIField map[string]string, schema []byte, reset bool) error
 		return nil
 	}
 
-	version, exists := openAPIField["version"]
-	if exists && schema != nil {
+	version, versionProvided := openAPIField["version"]
+
+	// use custom schema
+	if schema != nil {
+		if versionProvided {
 			return fmt.Errorf("builtin version and custom schema provided, cannot use both")
 		}
-
-	if schema != nil { // use custom schema
 		customSchema = schema
 		kubernetesOpenAPIVersion = "custom"
 		// if the schema is changed, initSchema should parse the new schema
@@ -571,13 +572,14 @@ func SetSchema(openAPIField map[string]string, schema []byte, reset bool) error
 	}
 
 	// use builtin version
-	kubernetesOpenAPIVersion = strings.ReplaceAll(version, ".", "")
+	kubernetesOpenAPIVersion = version
 	if kubernetesOpenAPIVersion == "" {
 		return nil
 	}
 	if _, ok := kubernetesapi.OpenAPIMustAsset[kubernetesOpenAPIVersion]; !ok {
 		return fmt.Errorf("the specified OpenAPI version is not built in")
 	}
+
 	customSchema = nil
 	// if the schema is changed, initSchema should parse the new schema
 	globalSchema.schemaInit = false
@@ -629,11 +631,10 @@ func parseBuiltinSchema(version string) {
 		// don't parse the built in schema
 		return
 	}
-
 	// parse the swagger, this should never fail
 	assetName := filepath.Join(
 		"kubernetesapi",
-		version,
+		strings.ReplaceAll(version, ".", "_"),
 		"swagger.pb")
 
 	if err := parse(kubernetesapi.OpenAPIMustAsset[version](assetName), Proto); err != nil {

kyaml/openapi/openapi_benchmark_test.go

@@ -5,20 +5,24 @@ package openapi
 
 import (
 	"path/filepath"
+	"strings"
 	"testing"
 
 	openapi_v2 "github.com/google/gnostic/openapiv2"
 	"google.golang.org/protobuf/proto"
-	v1212 "sigs.k8s.io/kustomize/kyaml/openapi/kubernetesapi/v1212"
+	"sigs.k8s.io/kustomize/kyaml/openapi/kubernetesapi"
 )
 
 func BenchmarkProtoUnmarshal(t *testing.B) {
+	version := kubernetesOpenAPIDefaultVersion
+
+	// parse the swagger, this should never fail
 	assetName := filepath.Join(
 		"kubernetesapi",
-		"v1212",
+		strings.ReplaceAll(version, ".", "_"),
 		"swagger.pb")
 
-	b := v1212.MustAsset(assetName)
+	b := kubernetesapi.OpenAPIMustAsset[version](assetName)
 
 	for i := 0; i < t.N; i++ {
 		// We parse protobuf and get an openapiv2.Document here.

kyaml/openapi/scripts/fetchSchemaFromCluster.sh

@@ -8,8 +8,13 @@ VERSION=$1
 
 cp $HOME/.kube/config /tmp/kubeconfig.txt | true
 $MYGOBIN/kind create cluster --image kindest/node:$VERSION --name=getopenapidata
-$MYGOBIN/kustomize openapi fetch > /tmp/new_swagger.json
+
+# TODO (natasha41575) Add a `kustomize openapi fetch --proto` option
+kubectl proxy &
+sleep 5
+curl -k -H "Accept: application/com.github.proto-openapi.spec.v2@v1.0+protobuf" http://localhost:8001/openapi/v2 > /tmp/new_swagger.pb
+
 $MYGOBIN/kind delete cluster --name=getopenapidata
 cp /tmp/kubeconfig.txt $HOME/.kube/config | true
-mkdir -p kubernetesapi/"${VERSION//.}"
-cp /tmp/new_swagger.json kubernetesapi/"${VERSION//.}"/swagger.json
+mkdir -p kubernetesapi/"${VERSION//./_}"
+cp /tmp/new_swagger.pb kubernetesapi/"${VERSION//./_}"/swagger.pb

kyaml/openapi/scripts/generateSwaggerDotGo.sh

@@ -7,6 +7,6 @@ MYGOBIN="${MYGOBIN:-$(go env GOPATH)/bin}"
 VERSION=$1
 
 $MYGOBIN/go-bindata \
-  --pkg "${VERSION//.}" \
-  -o kubernetesapi/"${VERSION//.}"/swagger.go \
-  kubernetesapi/"${VERSION//.}"/swagger.json
+  --pkg "${VERSION//./_}" \
+  -o kubernetesapi/"${VERSION//./_}"/swagger.go \
+  kubernetesapi/"${VERSION//./_}"/swagger.pb

kyaml/openapi/scripts/makeOpenApiInfoDotGo.sh

@@ -23,10 +23,7 @@ version_list=()
 V=`ls kubernetesapi | grep v.*`
 for VERSION in $V
 do
-  openapiinfo=$(\
-    jq -r '.info' kubernetesapi/$VERSION/swagger.json  | \
-    sed 's/[\" *]//g' | \
-    tr -d '\n' )
+  openapiinfo="{title:Kubernetes,version:${VERSION//_/.}}"
   info_list+=( $openapiinfo )
   version_list+=( ${VERSION} )
 done
@@ -70,7 +67,7 @@ for version in ${version_list[@]}
 do
   latest=$version
   cat <<EOF >>kubernetesapi/openapiinfo.go
-  "$version": $version.MustAsset,
+  "${version//_/.}": $version.MustAsset,
 EOF
 done
 
@@ -78,7 +75,7 @@ done
 cat <<EOF >>kubernetesapi/openapiinfo.go
 }
 
-const DefaultOpenAPI = "$latest"
+const DefaultOpenAPI = "${latest//_/.}"
 EOF
 
-
+gofmt -s -w kubernetesapi/openapiinfo.go