Refactor container function runtime into its own package

This commit is contained in:
Phillip Wittrock
2020-05-04 08:11:04 -07:00
parent 096ad8c95c
commit 594c48d19a
11 changed files with 773 additions and 2161 deletions

View File

@@ -0,0 +1,202 @@
// Copyright 2019 The Kubernetes Authors.
// SPDX-License-Identifier: Apache-2.0
package container
import (
"fmt"
"os"
"strings"
runtimeexec "sigs.k8s.io/kustomize/kyaml/fn/runtime/exec"
"sigs.k8s.io/kustomize/kyaml/fn/runtime/runtimeutil"
"sigs.k8s.io/kustomize/kyaml/yaml"
)
// Filter filters Resources using a container image.
// The container must start a process that reads the list of
// input Resources from stdin, reads the Configuration from the env
// API_CONFIG, and writes the filtered Resources to stdout.
// If there is a error or validation failure, the process must exit
// non-zero.
// The full set of environment variables from the parent process
// are passed to the container.
//
// Function Scoping:
// Filter applies the function only to Resources to which it is scoped.
//
// Resources are scoped to a function if any of the following are true:
// - the Resource were read from the same directory as the function config
// - the Resource were read from a subdirectory of the function config directory
// - the function config is in a directory named "functions" and
// they were read from a subdirectory of "functions" parent
// - the function config doesn't have a path annotation (considered globally scoped)
// - the Filter has GlobalScope == true
//
// In Scope Examples:
//
// Example 1: deployment.yaml and service.yaml in function.yaml scope
// same directory as the function config directory
// .
// ├── function.yaml
// ├── deployment.yaml
// └── service.yaml
//
// Example 2: apps/deployment.yaml and apps/service.yaml in function.yaml scope
// subdirectory of the function config directory
// .
// ├── function.yaml
// └── apps
//    ├── deployment.yaml
//    └── service.yaml
//
// Example 3: apps/deployment.yaml and apps/service.yaml in functions/function.yaml scope
// function config is in a directory named "functions"
// .
// ├── functions
// │   └── function.yaml
// └── apps
//    ├── deployment.yaml
//    └── service.yaml
//
// Out of Scope Examples:
//
// Example 1: apps/deployment.yaml and apps/service.yaml NOT in stuff/function.yaml scope
// .
// ├── stuff
// │   └── function.yaml
// └── apps
//    ├── deployment.yaml
//    └── service.yaml
//
// Example 2: apps/deployment.yaml and apps/service.yaml NOT in stuff/functions/function.yaml scope
// .
// ├── stuff
// │   └── functions
// │    └── function.yaml
// └── apps
//    ├── deployment.yaml
//    └── service.yaml
//
// Default Paths:
// Resources emitted by functions will have default path applied as annotations
// if none is present.
// The default path will be the function-dir/ (or parent directory in the case of "functions")
// + function-file-name/ + namespace/ + kind_name.yaml
//
// Example 1: Given a function in fn.yaml that produces a Deployment name foo and a Service named bar
// dir
// └── fn.yaml
//
// Would default newly generated Resources to:
//
// dir
// ├── fn.yaml
// └── fn
//    ├── deployment_foo.yaml
//    └── service_bar.yaml
//
// Example 2: Given a function in functions/fn.yaml that produces a Deployment name foo and a Service named bar
// dir
// └── fn.yaml
//
// Would default newly generated Resources to:
//
// dir
// ├── functions
// │   └── fn.yaml
// └── fn
//    ├── deployment_foo.yaml
//    └── service_bar.yaml
//
// Example 3: Given a function in fn.yaml that produces a Deployment name foo, namespace baz and a Service named bar namespace baz
// dir
// └── fn.yaml
//
// Would default newly generated Resources to:
//
// dir
// ├── fn.yaml
// └── fn
// └── baz
//    ├── deployment_foo.yaml
//    └── service_bar.yaml
type Filter struct {
// Image is the container image to use to create a container.
Image string `yaml:"image,omitempty"`
// Network is the container network to use.
Network string `yaml:"network,omitempty"`
// StorageMounts is a list of storage options that the container will have mounted.
StorageMounts []runtimeutil.StorageMount `yaml:"mounts,omitempty"`
Exec runtimeexec.Filter
}
func (c Filter) String() string {
if c.Exec.DeferFailure {
return fmt.Sprintf("%s deferFailure: %v", c.Image, c.Exec.DeferFailure)
}
return c.Image
}
func (c Filter) GetExit() error {
return c.Exec.GetExit()
}
func (c *Filter) Filter(nodes []*yaml.RNode) ([]*yaml.RNode, error) {
c.setupExec()
return c.Exec.Filter(nodes)
}
func (c *Filter) setupExec() {
// don't init 2x
if c.Exec.Path != "" {
return
}
path, args := c.getCommand()
c.Exec.Path = path
c.Exec.Args = args
}
// getArgs returns the command + args to run to spawn the container
func (c *Filter) getCommand() (string, []string) {
// run the container using docker. this is simpler than using the docker
// libraries, and ensures things like auth work the same as if the container
// was run from the cli.
network := "none"
if c.Network != "" {
network = c.Network
}
args := []string{"run",
"--rm", // delete the container afterward
"-i", "-a", "STDIN", "-a", "STDOUT", "-a", "STDERR", // attach stdin, stdout, stderr
"--network", network,
// added security options
"--user", "nobody", // run as nobody
"--security-opt=no-new-privileges", // don't allow the user to escalate privileges
// note: don't make fs readonly because things like heredoc rely on writing tmp files
}
// TODO(joncwong): Allow StorageMount fields to have default values.
for _, storageMount := range c.StorageMounts {
args = append(args, "--mount", storageMount.String())
}
os.Setenv("LOG_TO_STDERR", "true")
os.Setenv("STRUCTURED_RESULTS", "true")
// export the local environment vars to the container
for _, pair := range os.Environ() {
args = append(args, "-e", strings.Split(pair, "=")[0])
}
a := append(args, c.Image)
return "docker", a
}

View File

@@ -0,0 +1,203 @@
// Copyright 2019 The Kubernetes Authors.
// SPDX-License-Identifier: Apache-2.0
package container
import (
"bytes"
"fmt"
"os"
"strings"
"testing"
"github.com/stretchr/testify/assert"
"sigs.k8s.io/kustomize/kyaml/fn/runtime/runtimeutil"
"sigs.k8s.io/kustomize/kyaml/kio"
"sigs.k8s.io/kustomize/kyaml/yaml"
)
func TestFilter_setupExec(t *testing.T) {
var tests = []struct {
name string
functionConfig string
expectedArgs []string
instance Filter
}{
{
name: "command",
functionConfig: `apiVersion: apps/v1
kind: Deployment
metadata:
name: foo
`,
expectedArgs: []string{
"run",
"--rm",
"-i", "-a", "STDIN", "-a", "STDOUT", "-a", "STDERR",
"--network", "none",
"--user", "nobody",
"--security-opt=no-new-privileges",
},
instance: Filter{Image: "example.com:version"},
},
{
name: "network",
functionConfig: `apiVersion: apps/v1
kind: Deployment
metadata:
name: foo
`,
expectedArgs: []string{
"run",
"--rm",
"-i", "-a", "STDIN", "-a", "STDOUT", "-a", "STDERR",
"--network", "test-1",
"--user", "nobody",
"--security-opt=no-new-privileges",
},
instance: Filter{Image: "example.com:version", Network: "test-1"},
},
{
name: "storage_mounts",
functionConfig: `apiVersion: apps/v1
kind: Deployment
metadata:
name: foo
`,
expectedArgs: []string{
"run",
"--rm",
"-i", "-a", "STDIN", "-a", "STDOUT", "-a", "STDERR",
"--network", "none",
"--user", "nobody",
"--security-opt=no-new-privileges",
"--mount", fmt.Sprintf("type=%s,src=%s,dst=%s:ro", "bind", "/mount/path", "/local/"),
"--mount", fmt.Sprintf("type=%s,src=%s,dst=%s:ro", "volume", "myvol", "/local/"),
"--mount", fmt.Sprintf("type=%s,src=%s,dst=%s:ro", "tmpfs", "", "/local/"),
},
instance: Filter{
Image: "example.com:version",
StorageMounts: []runtimeutil.StorageMount{
{MountType: "bind", Src: "/mount/path", DstPath: "/local/"},
{MountType: "volume", Src: "myvol", DstPath: "/local/"},
{MountType: "tmpfs", Src: "", DstPath: "/local/"},
},
},
},
}
for i := range tests {
tt := tests[i]
t.Run(tt.name, func(t *testing.T) {
cfg, err := yaml.Parse(tt.functionConfig)
if !assert.NoError(t, err) {
t.FailNow()
}
tt.instance.Exec.FunctionConfig = cfg
os.Setenv("KYAML_TEST", "FOO")
tt.instance.setupExec()
// configure expected env
for _, e := range os.Environ() {
// the process env
tt.expectedArgs = append(tt.expectedArgs, "-e", strings.Split(e, "=")[0])
}
tt.expectedArgs = append(tt.expectedArgs, tt.instance.Image)
if !assert.Equal(t, "docker", tt.instance.Exec.Path) {
t.FailNow()
}
if !assert.Equal(t, tt.expectedArgs, tt.instance.Exec.Args) {
t.FailNow()
}
})
}
}
func TestFilter_Filter(t *testing.T) {
cfg, err := yaml.Parse(`apiVersion: apps/v1
kind: Deployment
metadata:
name: foo
`)
if !assert.NoError(t, err) {
return
}
input, err := (&kio.ByteReader{Reader: bytes.NewBufferString(`
apiVersion: apps/v1
kind: Deployment
metadata:
name: deployment-foo
---
apiVersion: v1
kind: Service
metadata:
name: service-foo
`)}).Read()
if !assert.NoError(t, err) {
return
}
instance := Filter{}
instance.Exec.FunctionConfig = cfg
instance.Exec.Path = "sed"
instance.Exec.Args = []string{"s/Deployment/StatefulSet/g"}
output, err := instance.Filter(input)
if !assert.NoError(t, err) {
t.FailNow()
}
b := &bytes.Buffer{}
err = kio.ByteWriter{Writer: b, KeepReaderAnnotations: true}.Write(output)
if !assert.NoError(t, err) {
t.FailNow()
}
if !assert.Equal(t, `apiVersion: apps/v1
kind: StatefulSet
metadata:
name: deployment-foo
annotations:
config.kubernetes.io/index: '0'
config.kubernetes.io/path: 'statefulset_deployment-foo.yaml'
---
apiVersion: v1
kind: Service
metadata:
name: service-foo
annotations:
config.kubernetes.io/index: '1'
config.kubernetes.io/path: 'service_service-foo.yaml'
`, b.String()) {
t.FailNow()
}
}
func TestFilter_String(t *testing.T) {
instance := Filter{Image: "foo"}
if !assert.Equal(t, "foo", instance.String()) {
t.FailNow()
}
instance.Exec.DeferFailure = true
if !assert.Equal(t, "foo deferFailure: true", instance.String()) {
t.FailNow()
}
}
func TestFilter_ExitCode(t *testing.T) {
instance := Filter{}
instance.Exec.Path = "/not/real/command"
instance.Exec.DeferFailure = true
_, err := instance.Filter(nil)
if !assert.NoError(t, err) {
t.FailNow()
}
if !assert.EqualError(t, instance.GetExit(), "fork/exec /not/real/command: no such file or directory") {
t.FailNow()
}
}

View File

@@ -0,0 +1,177 @@
// Copyright 2019 The Kubernetes Authors.
// SPDX-License-Identifier: Apache-2.0
package runtimeutil
import (
"fmt"
"strings"
"sigs.k8s.io/kustomize/kyaml/yaml"
)
const (
FunctionAnnotationKey = "config.kubernetes.io/function"
oldFunctionAnnotationKey = "config.k8s.io/function"
)
var functionAnnotationKeys = []string{FunctionAnnotationKey, oldFunctionAnnotationKey}
// FunctionSpec defines a spec for running a function
type FunctionSpec struct {
// Network is the name of the network to use from a container
Network string `json:"network,omitempty" yaml:"network,omitempty"`
DeferFailure bool `json:"deferFailure,omitempty" yaml:"deferFailure,omitempty"`
// Container is the spec for running a function as a container
Container ContainerSpec `json:"container,omitempty" yaml:"container,omitempty"`
// Starlark is the spec for running a function as a starlark script
Starlark StarlarkSpec `json:"starlark,omitempty" yaml:"starlark,omitempty"`
// Mounts are the storage or directories to mount into the container
StorageMounts []StorageMount `json:"mounts,omitempty" yaml:"mounts,omitempty"`
}
// ContainerSpec defines a spec for running a function as a container
type ContainerSpec struct {
// Image is the container image to run
Image string `json:"image,omitempty" yaml:"image,omitempty"`
// Network defines network specific configuration
Network ContainerNetwork `json:"network,omitempty" yaml:"network,omitempty"`
// Mounts are the storage or directories to mount into the container
StorageMounts []StorageMount `json:"mounts,omitempty" yaml:"mounts,omitempty"`
}
// ContainerNetwork
type ContainerNetwork struct {
// Required specifies that function requires a network
Required bool `json:"required,omitempty" yaml:"required,omitempty"`
}
// StarlarkSpec defines how to run a function as a starlark program
type StarlarkSpec struct {
Name string `json:"name,omitempty" yaml:"name,omitempty"`
// Path specifies a path to a starlark script
Path string `json:"path,omitempty" yaml:"path,omitempty"`
}
// StorageMount represents a container's mounted storage option(s)
type StorageMount struct {
// Type of mount e.g. bind mount, local volume, etc.
MountType string `json:"type,omitempty" yaml:"type,omitempty"`
// Source for the storage to be mounted.
// For named volumes, this is the name of the volume.
// For anonymous volumes, this field is omitted (empty string).
// For bind mounts, this is the path to the file or directory on the host.
Src string `json:"src,omitempty" yaml:"src,omitempty"`
// The path where the file or directory is mounted in the container.
DstPath string `json:"dst,omitempty" yaml:"dst,omitempty"`
}
func (s *StorageMount) String() string {
return fmt.Sprintf("type=%s,src=%s,dst=%s:ro", s.MountType, s.Src, s.DstPath)
}
// GetFunctionSpec returns the FunctionSpec for a resource. Returns
// nil if the resource does not have a FunctionSpec.
//
// The FunctionSpec is read from the resource metadata.annotation
// "config.kubernetes.io/function"
func GetFunctionSpec(n *yaml.RNode) *FunctionSpec {
meta, err := n.GetMeta()
if err != nil {
return nil
}
if fn := getFunctionSpecFromAnnotation(n, meta); fn != nil {
fn.Network = ""
fn.StorageMounts = []StorageMount{}
return fn
}
// legacy function specification for backwards compatibility
container := meta.Annotations["config.kubernetes.io/container"]
if container != "" {
return &FunctionSpec{Container: ContainerSpec{Image: container}}
}
return nil
}
// getFunctionSpecFromAnnotation parses the config function from an annotation
// if it is found
func getFunctionSpecFromAnnotation(n *yaml.RNode, meta yaml.ResourceMeta) *FunctionSpec {
var fs FunctionSpec
for _, s := range functionAnnotationKeys {
fn := meta.Annotations[s]
if fn != "" {
_ = yaml.Unmarshal([]byte(fn), &fs)
return &fs
}
}
n, err := n.Pipe(yaml.Lookup("metadata", "configFn"))
if err != nil || yaml.IsEmpty(n) {
return nil
}
s, err := n.String()
if err != nil {
return nil
}
_ = yaml.Unmarshal([]byte(s), &fs)
return &fs
}
func StringToStorageMount(s string) StorageMount {
m := make(map[string]string)
options := strings.Split(s, ",")
for _, option := range options {
keyVal := strings.SplitN(option, "=", 2)
m[keyVal[0]] = keyVal[1]
}
var sm StorageMount
for key, value := range m {
switch {
case key == "type":
sm.MountType = value
case key == "src":
sm.Src = value
case key == "dst":
sm.DstPath = value
}
}
return sm
}
// IsReconcilerFilter filters Resources based on whether or not they are Reconciler Resource.
// Resources with an apiVersion starting with '*.gcr.io', 'gcr.io' or 'docker.io' are considered
// Reconciler Resources.
type IsReconcilerFilter struct {
// ExcludeReconcilers if set to true, then Reconcilers will be excluded -- e.g.
// Resources with a reconcile container through the apiVersion (gcr.io prefix) or
// through the annotations
ExcludeReconcilers bool `yaml:"excludeReconcilers,omitempty"`
// IncludeNonReconcilers if set to true, the NonReconciler will be included.
IncludeNonReconcilers bool `yaml:"includeNonReconcilers,omitempty"`
}
// Filter implements kio.Filter
func (c *IsReconcilerFilter) Filter(inputs []*yaml.RNode) ([]*yaml.RNode, error) {
var out []*yaml.RNode
for i := range inputs {
isFnResource := GetFunctionSpec(inputs[i]) != nil
if isFnResource && !c.ExcludeReconcilers {
out = append(out, inputs[i])
}
if !isFnResource && c.IncludeNonReconcilers {
out = append(out, inputs[i])
}
}
return out, nil
}

View File

@@ -996,3 +996,272 @@ metadata:
})
}
}
func Test_GetFunction(t *testing.T) {
var tests = []struct {
name string
resource string
expectedFn string
missingFn bool
}{
// fn annotation
{
name: "fn annotation",
resource: `
apiVersion: v1beta1
kind: Example
metadata:
annotations:
config.kubernetes.io/function: |-
container:
image: foo:v1.0.0
`,
expectedFn: `
container:
image: foo:v1.0.0`,
},
{
name: "storage mounts json style",
resource: `
apiVersion: v1beta1
kind: Example
metadata:
annotations:
config.kubernetes.io/function: |-
container:
image: foo:v1.0.0
mounts: [ {type: bind, src: /mount/path, dst: /local/}, {src: myvol, dst: /local/, type: volume}, {dst: /local/, type: tmpfs} ]
`,
expectedFn: `
container:
image: foo:v1.0.0
mounts:
- type: bind
src: /mount/path
dst: /local/
- type: volume
src: myvol
dst: /local/
- type: tmpfs
dst: /local/
`,
},
{
name: "storage mounts yaml style",
resource: `
apiVersion: v1beta1
kind: Example
metadata:
annotations:
config.kubernetes.io/function: |-
container:
image: foo:v1.0.0
mounts:
- src: /mount/path
type: bind
dst: /local/
- dst: /local/
src: myvol
type: volume
- type: tmpfs
dst: /local/
`,
expectedFn: `
container:
image: foo:v1.0.0
mounts:
- type: bind
src: /mount/path
dst: /local/
- type: volume
src: myvol
dst: /local/
- type: tmpfs
dst: /local/
`,
},
{
name: "network",
resource: `
apiVersion: v1beta1
kind: Example
metadata:
annotations:
config.kubernetes.io/function: |-
container:
image: foo:v1.0.0
network:
required: true
`,
expectedFn: `
container:
image: foo:v1.0.0
network:
required: true
`,
},
{
name: "path",
resource: `
apiVersion: v1beta1
kind: Example
metadata:
annotations:
config.kubernetes.io/function: |-
path: foo
container:
image: foo:v1.0.0
`,
// path should be erased
expectedFn: `
container:
image: foo:v1.0.0
`,
},
{
name: "network",
resource: `
apiVersion: v1beta1
kind: Example
metadata:
annotations:
config.kubernetes.io/function: |-
network: foo
container:
image: foo:v1.0.0
`,
// network should be erased
expectedFn: `
container:
image: foo:v1.0.0
`,
},
// legacy fn style
{name: "legacy fn meta",
resource: `
apiVersion: v1beta1
kind: Example
metadata:
configFn:
container:
image: foo:v1.0.0
`,
expectedFn: `
container:
image: foo:v1.0.0
`,
},
// no fn
{name: "no fn",
resource: `
apiVersion: v1beta1
kind: Example
metadata:
annotations: {}
`,
missingFn: true,
},
// test network, etc...
}
for i := range tests {
tt := tests[i]
t.Run(tt.name, func(t *testing.T) {
resource := yaml.MustParse(tt.resource)
fn := GetFunctionSpec(resource)
if tt.missingFn {
if !assert.Nil(t, fn) {
t.FailNow()
}
} else {
b, err := yaml.Marshal(fn)
if !assert.NoError(t, err) {
t.FailNow()
}
if !assert.Equal(t,
strings.TrimSpace(tt.expectedFn),
strings.TrimSpace(string(b))) {
t.FailNow()
}
}
})
}
}
func Test_GetContainerNetworkRequired(t *testing.T) {
tests := []struct {
input string
required bool
}{
{
input: `apiVersion: v1
kind: Foo
metadata:
name: foo
configFn:
container:
image: gcr.io/kustomize-functions/example-tshirt:v0.1.0
network:
required: true
`,
required: true,
},
{
input: `apiVersion: v1
kind: Foo
metadata:
name: foo
configFn:
container:
image: gcr.io/kustomize-functions/example-tshirt:v0.1.0
network:
required: false
`,
required: false,
},
{
input: `apiVersion: v1
kind: Foo
metadata:
name: foo
configFn:
container:
image: gcr.io/kustomize-functions/example-tshirt:v0.1.0
`,
required: false,
},
{
input: `apiVersion: v1
kind: Foo
metadata:
name: foo
annotations:
config.kubernetes.io/function: |
container:
image: gcr.io/kustomize-functions/example-tshirt:v0.1.0
network:
required: true
`,
required: true,
},
}
for _, tc := range tests {
cfg, err := yaml.Parse(tc.input)
if !assert.NoError(t, err) {
return
}
fn := GetFunctionSpec(cfg)
assert.Equal(t, tc.required, fn.Container.Network.Required)
}
}