Merge pull request #3548 from monopole/moveCopier

Move the k8s-related code generator.
This commit is contained in:
Jeff Regan
2021-02-08 12:49:41 -08:00
committed by GitHub
13 changed files with 404 additions and 243 deletions

View File

@@ -79,6 +79,11 @@ $(MYGOBIN)/gorepomod:
cd cmd/gorepomod; \
go install .
# Build from local source.
$(MYGOBIN)/k8scopy:
cd cmd/k8scopy; \
go install .
# Build from local source.
$(MYGOBIN)/pluginator:
cd cmd/pluginator; \
@@ -98,13 +103,13 @@ $(MYGOBIN)/kustomize:
install-tools: \
$(MYGOBIN)/goimports \
$(MYGOBIN)/golangci-lint-kustomize \
$(MYGOBIN)/gh \
$(MYGOBIN)/gorepomod \
$(MYGOBIN)/helm \
$(MYGOBIN)/k8scopy \
$(MYGOBIN)/mdrip \
$(MYGOBIN)/pluginator \
$(MYGOBIN)/prchecker \
$(MYGOBIN)/stringer \
$(MYGOBIN)/helm
$(MYGOBIN)/stringer
### Begin kustomize plugin rules.
#
@@ -221,7 +226,7 @@ build-kustomize-api: $(builtinplugins)
cd api; go build ./...
.PHONY: generate-kustomize-api
generate-kustomize-api:
generate-kustomize-api: $(MYGOBIN)/k8scopy
cd api; go generate ./...
.PHONY: test-unit-kustomize-api

8
cmd/k8scopy/go.mod Normal file
View File

@@ -0,0 +1,8 @@
module sigs.k8s.io/kustomize/cmd/k8scopy
go 1.15
require (
github.com/stretchr/testify v1.4.0
sigs.k8s.io/yaml v1.2.0
)

15
cmd/k8scopy/go.sum Normal file
View File

@@ -0,0 +1,15 @@
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q=
sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc=

View File

@@ -0,0 +1,111 @@
// Copyright 2020 The Kubernetes Authors.
// SPDX-License-Identifier: Apache-2.0
package internal
import (
"bufio"
"fmt"
"log"
"os"
"path/filepath"
"strings"
)
const (
sigsK8sIo = "sigs.k8s.io"
)
type Copier struct {
spec *ModuleSpec
goModCache string
topPackage string
srcDir string
pgmName string
}
func (c Copier) replacementPath() string {
return filepath.Join(c.topPackage, c.subPath())
}
func (c Copier) subPath() string {
return filepath.Join("internal", c.pgmName)
}
func (c Copier) print() {
fmt.Printf(" apiMachineryModule: %s\n", c.spec.Module)
fmt.Printf(" replacementPath: %s\n", c.replacementPath())
fmt.Printf(" goModCache: %s\n", c.goModCache)
fmt.Printf(" topPackage: %s\n", c.topPackage)
fmt.Printf(" subPath: %s\n", c.subPath())
fmt.Printf(" srcDir: %s\n", c.srcDir)
fmt.Printf(" apiMachineryModSpec: %s\n", c.spec.Name())
fmt.Printf(" pgmName: %s\n", c.pgmName)
fmt.Printf(" pwd: %s\n", os.Getenv("PWD"))
}
func NewCopier(s *ModuleSpec) Copier {
tmp := Copier{
spec: s,
pgmName: os.Getenv("GOPACKAGE"),
goModCache: RunGetOutputCommand("go", "env", "GOMODCACHE"),
}
goMod := RunGetOutputCommand("go", "env", "GOMOD")
topPackage := filepath.Join(goMod[:len(goMod)-len("go.mod")-1], "yaml")
k := strings.Index(topPackage, sigsK8sIo)
if k < 1 {
log.Fatalf("cannot find %s in %s", sigsK8sIo, topPackage)
}
tmp.srcDir = topPackage[:k-1]
tmp.topPackage = topPackage[k:]
return tmp
}
func (c Copier) CopyFile(dir, name string) error {
inFile, err := os.Open(
filepath.Join(c.goModCache, c.spec.Name(), dir, name))
if err != nil {
return err
}
defer inFile.Close()
scanner := bufio.NewScanner(inFile)
w, err := newWriter(dir, name)
if err != nil {
return err
}
defer w.close()
w.write(
fmt.Sprintf(
// This particular phrasing is required.
"// Code generated by %s/generator from %s; DO NOT EDIT.",
c.pgmName, c.spec.Name()))
w.write(
fmt.Sprintf(
"// Copied from %s\n",
filepath.Join(c.spec.Name(), dir, name)))
for scanner.Scan() {
l := scanner.Text()
// Disallow recursive generation.
if strings.HasPrefix(l, "//go:generate") {
continue
}
// Don't want it to appear double generated.
if strings.HasPrefix(l, "// Code generated") {
continue
}
// Fix self-imports.
l = strings.Replace(l, c.spec.Module, c.replacementPath(), 1)
// Replace klog with generic log (eschewing k8s.io entirely).
l = strings.Replace(l, "\"k8s.io/klog\"", "\"log\"", 1)
l = strings.Replace(l, "klog.V(10).Infof(", "log.Printf(", 1)
w.write(l)
}
if err := scanner.Err(); err != nil {
return err
}
w.write("")
return nil
}

View File

@@ -0,0 +1,37 @@
// Copyright 2020 The Kubernetes Authors.
// SPDX-License-Identifier: Apache-2.0
package internal
import (
"io/ioutil"
"sigs.k8s.io/yaml"
)
type ModuleSpec struct {
Module string `json:"module,omitempty" yaml:"module,omitempty"`
Version string `json:"version,omitempty" yaml:"version,omitempty"`
Packages []PackageSpec `json:"packages,omitempty" yaml:"packages,omitempty"`
}
func (s ModuleSpec) Name() string {
return s.Module + "@" + s.Version
}
type PackageSpec struct {
Name string `json:"name,omitempty" yaml:"name,omitempty"`
Files []string `json:"files,omitempty" yaml:"files,omitempty"`
}
func ReadSpec(fileName string) *ModuleSpec {
bytes, err := ioutil.ReadFile(fileName)
if err != nil {
panic(err)
}
var spec ModuleSpec
if err = yaml.Unmarshal(bytes, &spec); err != nil {
panic(err)
}
return &spec
}

View File

@@ -0,0 +1,65 @@
// Copyright 2021 The Kubernetes Authors.
// SPDX-License-Identifier: Apache-2.0
package internal_test
import (
"io/ioutil"
"os"
"testing"
"github.com/stretchr/testify/assert"
. "sigs.k8s.io/kustomize/cmd/k8scopy/internal"
)
var data = []byte(`module: k8s.io/apimachinery
version: v0.17.0
packages:
- name: pkg/labels
files:
- labels.go
- selector.go
- zz_generated.deepcopy.go
- name: pkg/selection
files:
- operator.go
- name: pkg/util/sets
files:
- empty.go
- string.go
- name: pkg/util/errors
files:
- errors.go
- name: pkg/util/validation
files:
- validation.go
- name: pkg/util/validation/field
files:
- errors.go
- path.go
`)
func TestReadSpec(t *testing.T) {
fn := writeFile(t, data)
defer os.Remove(fn)
x := ReadSpec(fn)
assert.Equal(t, "k8s.io/apimachinery@v0.17.0", x.Name())
assert.Equal(t, 6, len(x.Packages))
assert.Equal(t, "pkg/util/validation/field", x.Packages[5].Name)
assert.Equal(t, "path.go", x.Packages[5].Files[1])
}
// Write content to temp file, returning file name.
func writeFile(t *testing.T, content []byte) string {
f, err := ioutil.TempFile("", "testjunk")
if err != nil {
t.Fatal(err)
}
if _, err = f.Write(content); err != nil {
t.Fatal(err)
}
if err := f.Close(); err != nil {
t.Fatal(err)
}
return f.Name()
}

View File

@@ -0,0 +1,32 @@
// Copyright 2020 The Kubernetes Authors.
// SPDX-License-Identifier: Apache-2.0
package internal
import (
"bytes"
"fmt"
"log"
"os/exec"
"strings"
)
func RunNoOutputCommand(n string, args ...string) {
o := RunGetOutputCommand(n, args...)
if len(o) > 0 {
log.Fatalf("unexpected output: %q", o)
}
}
func RunGetOutputCommand(n string, args ...string) string {
cmd := exec.Command(n, args...)
var outBuf bytes.Buffer
cmd.Stdout = &outBuf
var errBuf bytes.Buffer
cmd.Stderr = &errBuf
if err := cmd.Run(); err != nil {
fmt.Printf("err: %q\n", errBuf.String())
log.Fatal(err)
}
return strings.TrimSpace(outBuf.String())
}

View File

@@ -0,0 +1,40 @@
// Copyright 2020 The Kubernetes Authors.
// SPDX-License-Identifier: Apache-2.0
package internal
import (
"fmt"
"log"
"os"
"path/filepath"
)
type writer struct {
root string
f *os.File
}
func newWriter(toDir, name string) (*writer, error) {
if err := os.MkdirAll(toDir, 0755); err != nil {
log.Printf("unable to create directory: %s", toDir)
return nil, err
}
n := filepath.Join(toDir, name)
f, err := os.Create(n)
if err != nil {
return nil, fmt.Errorf("unable to create `%s`; %v", n, err)
}
return &writer{root: toDir, f: f}, nil
}
func (w *writer) close() {
w.f.Close()
}
func (w *writer) write(line string) {
if _, err := w.f.WriteString(line + "\n"); err != nil {
log.Printf("Trouble writing: %s", line)
log.Fatalf("Error: %s", err)
}
}

54
cmd/k8scopy/main.go Normal file
View File

@@ -0,0 +1,54 @@
// Copyright 2020 The Kubernetes Authors.
// SPDX-License-Identifier: Apache-2.0
// k8scopy is a code reuse mechanism for k8s.io code.
//
// kustomize, kyaml, cmd/config are kustomize repo modules that want to reuse
// some small bits of k8s.io code. These modules cannot depend k8s.io via
// normal Go import or vendoring because kubectl will depend on these things
// eventually (see kubernetes-sigs/kustomize/issues/1500), and kubectl's code
// reuse is tricky. While kubectl remains in the k/k repo, it depends on local
// relative symlinked paths to a 'staging' version of k8s.io code. No code
// imported by kubectl can refer to any other version of k8s.io code, not by
// Go importing, not by Go vendoring.
//
// This main exists to allow "go generate" to copy select k8s.io packages into
// the kustomize repo at well defined tags in reproducible fashion. It's
// a form of vendoring reuse that avoids the problems created by k8s staging.
// The copied code is labelled as generated and is not otherwise edited.
//
// When/if kubectl is finally extracted from k/k to its own repo, it can
// depend on k8s.io code via normal imports, and then so can kustomize,
// so this technique can be dropped.
//
// Until then, if a bug is found in a particular instance of copied k8s.io
// (highly unlikely, since only old stable versions are copied), just update
// the version being copied, re-generate, and if need be adjust call points.
package main
import (
"log"
"os"
"sigs.k8s.io/kustomize/cmd/k8scopy/internal"
)
func main() {
if len(os.Args) < 2 {
log.Fatal("Need name of yaml file containing module specs.")
}
spec := internal.ReadSpec(os.Args[1])
c := internal.NewCopier(spec)
internal.RunNoOutputCommand("go", "get", spec.Name())
for _, p := range spec.Packages {
for _, n := range p.Files {
if err := c.CopyFile(p.Name, n); err != nil {
log.Fatal(err)
}
}
}
internal.RunNoOutputCommand(
"go", "mod", "edit", "-droprequire="+spec.Module)
internal.RunNoOutputCommand("go", "mod", "tidy")
internal.RunGetOutputCommand("go", "fmt", "./...")
}

View File

@@ -1,27 +1,7 @@
// Copyright 2020 The Kubernetes Authors.
// SPDX-License-Identifier: Apache-2.0
// Package k8sgen contains code generated by copying bits of k8s.io code.
// The copier is in generator/main.go.
//
// kyaml (and indirectly kustomize) wants to reuse the k/k label selection code.
// Labels and annotations are a key part of k8s metadata.
//
// kyaml cannot depend on it via a normal Go import because kubectl will
// depend on kyaml eventually (see kubernetes-sigs/kustomize/issues/1500).
// While kubectl remains in the k/k repo, it depends on local relative symlinked
// path to a 'staging' version of k8s.io code. No code imported by kubectl can
// refer to any other version of k8s.io code.
//
// So to do what that code does, kyaml must either reimplement it or copy it.
// When/if kubectl is finally extracted from k/k to its own repo, it can
// depend on apimachinery code via normal imports, and so can kyaml.
// At that point, this copy (and copying mechanism) can be dropped.
//
// Until then, if a bug is found in a particular instance of copied k8s.io
// (unlikely, since this code is old and stable), just increment
// apiMachineryVersion below to match the fixed version, possibly change
// the files copied, re-generate, and if need be adjust call points.
//
//go:generate go run generator/main.go
// All code below this directory is generated.
// See {repo}/cmd/k8scopy/main.go for more info.
//go:generate k8scopy k8scopy.yaml
package k8sgen

View File

@@ -1,3 +0,0 @@
module sigs.k8s.io/kustomize/kyaml/yaml/internal/k8sgen/generator
go 1.15

View File

@@ -1,213 +0,0 @@
// Copyright 2020 The Kubernetes Authors.
// SPDX-License-Identifier: Apache-2.0
// A code copier launched via `go generate`.
// See k8segen/doc.go for further discussion.
package main
import (
"bufio"
"bytes"
"fmt"
"log"
"os"
"os/exec"
"path/filepath"
"strings"
)
const (
// Splitting this so it doesn't show up in grep.
apiMachineryModule = "k8s.io" + "/" + "apimachinery"
apiMachineryVersion = "v0.17.0"
sigsK8sIo = "sigs.k8s.io"
)
var (
filesToCopy = map[string][]string{
filepath.Join("pkg", "labels"): {
"labels.go",
"selector.go",
"zz_generated.deepcopy.go",
},
filepath.Join("pkg", "selection"): {
"operator.go",
},
filepath.Join("pkg", "util", "sets"): {
"empty.go",
"string.go",
},
filepath.Join("pkg", "util", "errors"): {
"errors.go",
},
filepath.Join("pkg", "util", "validation"): {
"validation.go",
},
filepath.Join("pkg", "util", "validation", "field"): {
"errors.go",
"path.go",
},
}
)
func main() {
c := newCopier()
// c.print()
runNoOutputCommand("go", "get", c.apiMachineryModSpec())
for dir, files := range filesToCopy {
for _, n := range files {
if err := c.copyFile(dir, n); err != nil {
log.Fatal(err)
}
}
}
runNoOutputCommand(
"go", "mod", "edit", "-droprequire="+apiMachineryModule)
runNoOutputCommand("go", "mod", "tidy")
runGetOutputCommand("go", "fmt", "./...")
}
type copier struct {
goModCache string
topPackage string
srcDir string
pgmName string
}
func (c copier) apiMachineryModSpec() string {
return apiMachineryModule + "@" + apiMachineryVersion
}
func (c copier) replacementPath() string {
return filepath.Join(c.topPackage, c.subPath())
}
func (c copier) subPath() string {
return filepath.Join("internal", c.pgmName)
}
func (c copier) print() {
fmt.Printf(" apiMachineryModule: %s\n", apiMachineryModule)
fmt.Printf(" replacementPath: %s\n", c.replacementPath())
fmt.Printf(" goModCache: %s\n", c.goModCache)
fmt.Printf(" topPackage: %s\n", c.topPackage)
fmt.Printf(" subPath: %s\n", c.subPath())
fmt.Printf(" srcDir: %s\n", c.srcDir)
fmt.Printf(" apiMachineryModSpec: %s\n", c.apiMachineryModSpec())
fmt.Printf(" pgmName: %s\n", c.pgmName)
fmt.Printf(" pwd: %s\n", os.Getenv("PWD"))
}
func newCopier() copier {
tmp := copier{
pgmName: os.Getenv("GOPACKAGE"),
goModCache: runGetOutputCommand("go", "env", "GOMODCACHE"),
}
goMod := runGetOutputCommand("go", "env", "GOMOD")
topPackage := filepath.Join(goMod[:len(goMod)-len("go.mod")-1], "yaml")
k := strings.Index(topPackage, sigsK8sIo)
if k < 1 {
log.Fatalf("cannot find %s in %s", sigsK8sIo, topPackage)
}
tmp.srcDir = topPackage[:k-1]
tmp.topPackage = topPackage[k:]
return tmp
}
func (c copier) copyFile(dir, name string) error {
inFile, err := os.Open(
filepath.Join(c.goModCache, c.apiMachineryModSpec(), dir, name))
if err != nil {
return err
}
defer inFile.Close()
scanner := bufio.NewScanner(inFile)
w, err := newWriter(dir, name)
if err != nil {
return err
}
defer w.close()
w.write(
fmt.Sprintf(
// This particular phrasing is required.
"// Code generated by %s/generator from %s; DO NOT EDIT.",
c.pgmName, c.apiMachineryModSpec()))
w.write(
fmt.Sprintf(
"// Copied from %s\n",
filepath.Join(c.apiMachineryModSpec(), dir, name)))
for scanner.Scan() {
l := scanner.Text()
// Disallow recursive generation.
if strings.HasPrefix(l, "//go:generate") {
continue
}
// Don't want it to appear double generated.
if strings.HasPrefix(l, "// Code generated") {
continue
}
// Fix self-imports.
l = strings.Replace(l, apiMachineryModule, c.replacementPath(), 1)
// Replace klog with generic log (eschewing k8s.io entirely).
l = strings.Replace(l, "\"k8s.io/klog\"", "\"log\"", 1)
l = strings.Replace(l, "klog.V(10).Infof(", "log.Printf(", 1)
w.write(l)
}
if err := scanner.Err(); err != nil {
return err
}
w.write("")
return nil
}
type writer struct {
root string
f *os.File
}
func newWriter(toDir, name string) (*writer, error) {
if err := os.MkdirAll(toDir, 0755); err != nil {
log.Printf("unable to create directory: %s", toDir)
return nil, err
}
n := filepath.Join(toDir, name)
f, err := os.Create(n)
if err != nil {
return nil, fmt.Errorf("unable to create `%s`; %v", n, err)
}
return &writer{root: toDir, f: f}, nil
}
func (w *writer) close() {
w.f.Close()
}
func (w *writer) write(line string) {
if _, err := w.f.WriteString(line + "\n"); err != nil {
log.Printf("Trouble writing: %s", line)
log.Fatalf("Error: %s", err)
}
}
func runNoOutputCommand(n string, args ...string) {
o := runGetOutputCommand(n, args...)
if len(o) > 0 {
log.Fatalf("unexpected output: %q", o)
}
}
func runGetOutputCommand(n string, args ...string) string {
cmd := exec.Command(n, args...)
var outBuf bytes.Buffer
cmd.Stdout = &outBuf
var errBuf bytes.Buffer
cmd.Stderr = &errBuf
if err := cmd.Run(); err != nil {
fmt.Printf("err: %q\n", errBuf.String())
log.Fatal(err)
}
return strings.TrimSpace(outBuf.String())
}

View File

@@ -0,0 +1,30 @@
# Copyright 2020 The Kubernetes Authors.
# SPDX-License-Identifier: Apache-2.0
#
# The files to vendor (copy).
# See {repo}/cmd/k8scopy/main.go for more info.
module: k8s.io/apimachinery
version: v0.17.0
packages:
- name: pkg/labels
files:
- labels.go
- selector.go
- zz_generated.deepcopy.go
- name: pkg/selection
files:
- operator.go
- name: pkg/util/sets
files:
- empty.go
- string.go
- name: pkg/util/errors
files:
- errors.go
- name: pkg/util/validation
files:
- validation.go
- name: pkg/util/validation/field
files:
- errors.go
- path.go