kubernetes-sigs/kustomize
1
0
Fork 0
mirror of https://github.com/kubernetes-sigs/kustomize.git synced 2026-06-11 17:12:51 +00:00
Code Issues Packages Projects Releases 1 Wiki Activity

Merge pull request #3939 from KnVerey/framework_test_helpers

Browse Source 
[kyaml] Improvements to frameworktestutil
This commit is contained in:
Jeff Regan
2021-06-03 11:14:54 -07:00
committed by GitHub
parent 636b9c7aeb 382f09a126
commit da1bd901b4
5 changed files with 285 additions and 203 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
kyaml/fn/framework/example_test.go
View File

@@ -8,6 +8,7 @@ import (
"fmt"
"log"
"path/filepath"
"strings"
"sigs.k8s.io/kustomize/kyaml/errors"
"sigs.k8s.io/kustomize/kyaml/fn/framework"
@@ -962,10 +963,27 @@ func (a *v1alpha1JavaSpringBoot) Default() error {
}
func (a *v1alpha1JavaSpringBoot) Validate() error {
var messages []string
if a.Metadata.Name == "" {
return errors.Errorf("Name is required")
messages = append(messages, "name is required")
}
return nil
if a.Spec.Replicas > 10 {
messages = append(messages, "replicas must be less than 10")
}
if !strings.HasSuffix(a.Spec.Domain, "example.com") {
messages = append(messages, "domain must be a subdomain of example.com")
}
if strings.HasSuffix(a.Spec.Image, ":latest") {
messages = append(messages, "image should not have latest tag")
}
if len(messages) == 0 {
return nil
}
errMsg := fmt.Sprintf("JavaSpringBoot had %d errors:\n", len(messages))
for i, msg := range messages {
errMsg += fmt.Sprintf(" [%d] %s\n", i+1, msg)
}
return errors.Errorf(errMsg)
}
// ExampleVersionedAPIProcessor shows how to use the VersionedAPIProcessor and TemplateProcessor to

430
kyaml/fn/framework/frameworktestutil/frameworktestutil.go
View File

@@ -9,27 +9,60 @@ import (
"io/ioutil"
"os"
"path/filepath"
"regexp"
"strings"
"testing"
"github.com/spf13/cobra"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"sigs.k8s.io/kustomize/kyaml/fn/framework"
"sigs.k8s.io/kustomize/kyaml/kio"
)
// CommandResultsChecker tests a function by running it with predefined inputs and comparing
// the outputs to expected results.
const (
DefaultTestDataDirectory = "testdata"
DefaultConfigInputFilename = "config.yaml"
DefaultInputFilename = "input.yaml"
DefaultInputFilenameGlob = "input*.yaml"
DefaultOutputFilename = "expected.yaml"
DefaultErrorFilename = "errors.txt"
)
// CommandResultsChecker tests a command-wrapped function by running it with predefined inputs
// and comparing the outputs to expected results.
type CommandResultsChecker struct {
// TestDataDirectory is the directory containing the testdata subdirectories.
// CommandResultsChecker will recurse into each test directory and run the Command
// if the directory contains both the ConfigInputFilename and at least one
// of ExpectedOutputFilname or ExpectedErrorFilename.
// if the directory contains at least one of ExpectedOutputFilename or ExpectedErrorFilename.
// Defaults to "testdata"
TestDataDirectory string
// ExpectedOutputFilename is the file with the expected output of the function
// Defaults to "expected.yaml". Directories containing neither this file
// nor ExpectedErrorFilename will be skipped.
ExpectedOutputFilename string
// ExpectedErrorFilename is the file containing elements of an expected error message.
// Each line of the file will be treated as a regex that must match the actual error.
// Defaults to "errors.txt". Directories containing neither this file
// nor ExpectedOutputFilename will be skipped.
ExpectedErrorFilename string
// UpdateExpectedFromActual if set to true will write the actual results to the
// expected testdata files. This is useful for updating test data.
UpdateExpectedFromActual bool
// OutputAssertionFunc allows you to swap out the logic used to compare the expected output
// from the fixture file to the actual output.
// By default, it performs a string comparison after normalizing whitespace.
OutputAssertionFunc AssertionFunc
// ErrorAssertionFunc allows you to swap out the logic used to compare the expected error
// message from the fixture file to the actual error message.
// By default, it interprets each line of the fixture as a regex that the actual error must match.
ErrorAssertionFunc AssertionFunc
// ConfigInputFilename is the name of the config file provided as the first
// argument to the function. Directories without this file will be skipped.
// Defaults to "config.yaml"
@@ -39,226 +72,114 @@ type CommandResultsChecker struct {
// Defaults to "input*.yaml"
InputFilenameGlob string
// ExpectedOutputFilename is the file with the expected output of the function
// Defaults to "expected.yaml". Directories containing neither this file
// nor ExpectedErrorFilename will be skipped.
ExpectedOutputFilename string
// ExpectedErrorFilename is the file containing part of an expected error message
// Defaults to "error.yaml". Directories containing neither this file
// nor ExpectedOutputFilename will be skipped.
ExpectedErrorFilename string
// Command provides the function to run.
Command func() *cobra.Command
// UpdateExpectedFromActual if set to true will write the actual results to the
// expected testdata files. This is useful for updating test data.
UpdateExpectedFromActual bool
testsCasesRun int
}
// Assert asserts the results for functions
// Assert runs the command with the input provided in each valid test directory
// and verifies that the actual output and error match the fixtures in the directory.
func (rc *CommandResultsChecker) Assert(t *testing.T) bool {
if rc.TestDataDirectory == "" {
rc.TestDataDirectory = "testdata"
}
if rc.ConfigInputFilename == "" {
rc.ConfigInputFilename = "config.yaml"
}
if rc.ExpectedOutputFilename == "" {
rc.ExpectedOutputFilename = "expected.yaml"
}
if rc.ExpectedErrorFilename == "" {
rc.ExpectedErrorFilename = "error.yaml"
rc.ConfigInputFilename = DefaultConfigInputFilename
}
if rc.InputFilenameGlob == "" {
rc.InputFilenameGlob = "input*.yaml"
rc.InputFilenameGlob = DefaultInputFilenameGlob
}
err := filepath.Walk(rc.TestDataDirectory, func(path string, info os.FileInfo, err error) error {
require.NoError(t, err)
if !info.IsDir() {
// skip non-directories
return nil
checker := newResultsChecker(
rc.TestDataDirectory, rc.ExpectedOutputFilename, rc.ExpectedErrorFilename,
rc.OutputAssertionFunc, rc.ErrorAssertionFunc,
rc.UpdateExpectedFromActual,
)
checker.assert(t, func() (string, string) {
_, err := os.Stat(rc.ConfigInputFilename)
if os.IsNotExist(err) {
t.Errorf("Test case is missing FunctionConfig input file (default: %s)", DefaultConfigInputFilename)
}
rc.compare(t, path)
return nil
})
require.NoError(t, err)
require.NoError(t, err)
args := []string{rc.ConfigInputFilename}
require.NotZero(t, rc.testsCasesRun, "No complete test cases found in %s", rc.TestDataDirectory)
return true
}
func (rc *CommandResultsChecker) compare(t *testing.T, path string) {
// cd into the directory so we can test functions that refer
// local files by relative paths
d, err := os.Getwd()
require.NoError(t, err)
defer func() { require.NoError(t, os.Chdir(d)) }()
require.NoError(t, os.Chdir(path))
// make sure this directory contains test data
_, err = os.Stat(rc.ConfigInputFilename)
if os.IsNotExist(err) {
// missing input
return
}
args := []string{rc.ConfigInputFilename}
expectedOutput, expectedError := getExpected(t, rc.ExpectedOutputFilename, rc.ExpectedErrorFilename)
if expectedError == "" && expectedOutput == "" && !rc.UpdateExpectedFromActual {
// missing expected and UpdateExpectedFromActual == false, return and skip this test
return
}
require.NoError(t, err)
// run the test
t.Run(path, func(t *testing.T) {
rc.testsCasesRun += 1
if rc.InputFilenameGlob != "" {
inputs, err := filepath.Glob(rc.InputFilenameGlob)
require.NoError(t, err)
args = append(args, inputs...)
}
var actualOutput, actualError bytes.Buffer
var stdOut, stdErr bytes.Buffer
cmd := rc.Command()
cmd.SetArgs(args)
cmd.SetOut(&actualOutput)
cmd.SetErr(&actualError)
cmd.SetOut(&stdOut)
cmd.SetErr(&stdErr)
err = cmd.Execute()
// Update the fixtures if configured to
if rc.UpdateExpectedFromActual {
if actualError.String() != "" {
assert.NoError(t, ioutil.WriteFile(rc.ExpectedErrorFilename, actualError.Bytes(), 0600))
}
if actualOutput.String() != "" {
assert.NoError(t, ioutil.WriteFile(rc.ExpectedOutputFilename, actualOutput.Bytes(), 0600))
}
return
}
// Compare the results
if expectedError != "" {
// We expected an error, so make sure there was one and it matches
require.Error(t, err, actualOutput.String())
require.Contains(t,
standardizeSpacing(actualError.String()),
standardizeSpacing(expectedError), actualOutput.String())
} else {
// We didn't expect an error, and the output should match
require.NoError(t, err, actualError.String())
require.Equal(t,
standardizeSpacing(expectedOutput),
standardizeSpacing(actualOutput.String()), actualError.String())
}
return stdOut.String(), stdErr.String()
})
return true
}
func standardizeSpacing(s string) string {
// remove extra whitespace and convert Windows line endings
return strings.ReplaceAll(strings.TrimSpace(s), "\r\n", "\n")
}
// ProcessorResultsChecker tests a function by running it with predefined inputs and comparing
// the outputs to expected results.
// ProcessorResultsChecker tests a processor function by running it with predefined inputs
// and comparing the outputs to expected results.
type ProcessorResultsChecker struct {
// TestDataDirectory is the directory containing the testdata subdirectories.
// CommandResultsChecker will recurse into each test directory and run the Processor
// if the directory contains both the InputFilename and at least one
// of ExpectedOutputFilename or ExpectedErrorFilename.
// ProcessorResultsChecker will recurse into each test directory and run the Command
// if the directory contains at least one of ExpectedOutputFilename or ExpectedErrorFilename.
// Defaults to "testdata"
TestDataDirectory string
// InputFilename is the name of the file containing the ResourceList input.
// Directories without this file will be skipped.
// Defaults to "input.yaml"
InputFilename string
// ExpectedOutputFilename is the file with the expected output of the function
// Defaults to "expected.yaml". Directories containing neither this file
// nor ExpectedErrorFilename will be skipped.
ExpectedOutputFilename string
// ExpectedErrorFilename is the file containing part of an expected error message
// Defaults to "error.yaml". Directories containing neither this file
// ExpectedErrorFilename is the file containing elements of an expected error message.
// Each line of the file will be treated as a regex that must match the actual error.
// Defaults to "errors.txt". Directories containing neither this file
// nor ExpectedOutputFilename will be skipped.
ExpectedErrorFilename string
// Processor returns a ResourceListProcessor to run.
Processor func() framework.ResourceListProcessor
// UpdateExpectedFromActual if set to true will write the actual results to the
// expected testdata files. This is useful for updating test data.
UpdateExpectedFromActual bool
testsCasesRun int
// InputFilename is the name of the file containing the ResourceList input.
// Directories without this file will be skipped.
// Defaults to "input.yaml"
InputFilename string
// OutputAssertionFunc allows you to swap out the logic used to compare the expected output
// from the fixture file to the actual output.
// By default, it performs a string comparison after normalizing whitespace.
OutputAssertionFunc AssertionFunc
// ErrorAssertionFunc allows you to swap out the logic used to compare the expected error
// message from the fixture file to the actual error message.
// By default, it interprets each line of the fixture as a regex that the actual error must match.
ErrorAssertionFunc AssertionFunc
// Processor returns a ResourceListProcessor to run.
Processor func() framework.ResourceListProcessor
}
// Assert asserts the results for functions
// Assert runs the processor with the input provided in each valid test directory
// and verifies that the actual output and error match the fixtures in the directory.
func (rc *ProcessorResultsChecker) Assert(t *testing.T) bool {
if rc.TestDataDirectory == "" {
rc.TestDataDirectory = "testdata"
}
if rc.InputFilename == "" {
rc.InputFilename = "input.yaml"
}
if rc.ExpectedOutputFilename == "" {
rc.ExpectedOutputFilename = "expected.yaml"
}
if rc.ExpectedErrorFilename == "" {
rc.ExpectedErrorFilename = "error.yaml"
rc.InputFilename = DefaultInputFilename
}
err := filepath.Walk(rc.TestDataDirectory, func(path string, info os.FileInfo, err error) error {
require.NoError(t, err)
if !info.IsDir() {
// skip non-directories
return nil
checker := newResultsChecker(
rc.TestDataDirectory, rc.ExpectedOutputFilename, rc.ExpectedErrorFilename,
rc.OutputAssertionFunc, rc.ErrorAssertionFunc,
rc.UpdateExpectedFromActual,
)
checker.assert(t, func() (string, string) {
_, err := os.Stat(rc.InputFilename)
if os.IsNotExist(err) {
t.Error("Test case is missing input file")
}
rc.compare(t, path)
return nil
})
require.NoError(t, err)
require.NoError(t, err)
require.NotZero(t, rc.testsCasesRun, "No complete test cases found in %s", rc.TestDataDirectory)
return true
}
func (rc *ProcessorResultsChecker) compare(t *testing.T, path string) {
// cd into the directory so we can test functions that refer
// local files by relative paths
d, err := os.Getwd()
require.NoError(t, err)
defer func() { require.NoError(t, os.Chdir(d)) }()
require.NoError(t, os.Chdir(path))
// make sure this directory contains test data
_, err = os.Stat(rc.InputFilename)
if os.IsNotExist(err) {
// missing input
return
}
require.NoError(t, err)
expectedOutput, expectedError := getExpected(t, rc.ExpectedOutputFilename, rc.ExpectedErrorFilename)
if expectedError == "" && expectedOutput == "" && !rc.UpdateExpectedFromActual {
// missing expected and UpdateExpectedFromActual == false, return and skip this test
return
}
// run the test
t.Run(path, func(t *testing.T) {
rc.testsCasesRun += 1
actualOutput := bytes.NewBuffer([]byte{})
rlBytes, err := ioutil.ReadFile(rc.InputFilename)
require.NoError(t, err)
@@ -267,63 +188,170 @@ func (rc *ProcessorResultsChecker) compare(t *testing.T, path string) {
Reader: bytes.NewBuffer(rlBytes),
Writer: actualOutput,
}
err = framework.Execute(rc.Processor(), &rw)
if err != nil {
require.NotEmptyf(t, err.Error(), "processor returned error with empty message")
return actualOutput.String(), err.Error()
}
return actualOutput.String(), ""
})
return true
}
// Update the fixtures if configured to
if rc.UpdateExpectedFromActual {
if err != nil {
require.NoError(t, ioutil.WriteFile(rc.ExpectedErrorFilename, []byte(err.Error()), 0600))
type AssertionFunc func(t *testing.T, expected string, actual string)
// RequireEachLineMatches is an AssertionFunc that treats each line of expected string
// as a regex that must match the actual string.
func RequireEachLineMatches(t *testing.T, expected, actual string) {
// Check that each expected line matches the output
actual = standardizeSpacing(actual)
for _, msg := range strings.Split(standardizeSpacing(expected), "\n") {
require.Regexp(t, regexp.MustCompile(msg), actual)
}
}
// RequireStrippedStringsEqual is an AssertionFunc that does a simple string comparison
// of expected and actual after normalizing whitespace.
func RequireStrippedStringsEqual(t *testing.T, expected, actual string) {
require.Equal(t,
standardizeSpacing(expected),
standardizeSpacing(actual))
}
func standardizeSpacing(s string) string {
// remove extra whitespace and convert Windows line endings
return strings.ReplaceAll(strings.TrimSpace(s), "\r\n", "\n")
}
// resultsChecker implements the core logic shared by all results checking types.
type resultsChecker struct {
testDataDirectory string
expectedOutputFilename string
expectedErrorFilename string
updateExpectedFromActual bool
outputAssertionFunc AssertionFunc
errorAssertionFunc AssertionFunc
testsCasesRun int
}
func newResultsChecker(testDataDir string, outputFilename string, errorFilename string,
outputAsserter AssertionFunc, errorAsserter AssertionFunc,
updateFixtures bool) *resultsChecker {
rc := resultsChecker{
testDataDirectory: testDataDir,
expectedOutputFilename: outputFilename,
expectedErrorFilename: errorFilename,
updateExpectedFromActual: updateFixtures,
outputAssertionFunc: outputAsserter,
errorAssertionFunc: errorAsserter,
}
if rc.testDataDirectory == "" {
rc.testDataDirectory = DefaultTestDataDirectory
}
if rc.expectedOutputFilename == "" {
rc.expectedOutputFilename = DefaultOutputFilename
}
if rc.expectedErrorFilename == "" {
rc.expectedErrorFilename = DefaultErrorFilename
}
if rc.outputAssertionFunc == nil {
rc.outputAssertionFunc = RequireStrippedStringsEqual
}
if rc.errorAssertionFunc == nil {
rc.errorAssertionFunc = RequireEachLineMatches
}
return &rc
}
// assert traverses TestDataDirectory to find test cases, calls getResult to invoke the function
// under test in each directory, and then runs assertions on the returned output and error results.
// It triggers a test failure if no valid test directories were found.
func (rc *resultsChecker) assert(t *testing.T, getResult func() (string, string)) {
err := filepath.Walk(rc.testDataDirectory,
func(path string, info os.FileInfo, err error) error {
require.NoError(t, err)
if !info.IsDir() {
// skip non-directories
return nil
}
if len(actualOutput.String()) > 0 {
require.NoError(t, ioutil.WriteFile(rc.ExpectedOutputFilename, actualOutput.Bytes(), 0600))
rc.runDirectoryTestCase(t, path, getResult)
return nil
})
require.NoError(t, err)
require.NotZero(t, rc.testsCasesRun, "No complete test cases found in %s", rc.testDataDirectory)
}
func (rc *resultsChecker) runDirectoryTestCase(t *testing.T, path string, getResult func() (string, string)) {
// cd into the directory so we can test functions that refer
// local files by relative paths
d, err := os.Getwd()
require.NoError(t, err)
defer func() { require.NoError(t, os.Chdir(d)) }()
require.NoError(t, os.Chdir(path))
expectedOutput, expectedError := rc.readAssertionFiles(t)
if expectedError == "" && expectedOutput == "" && !rc.updateExpectedFromActual {
// not a test directory: missing expectations and updateExpectedFromActual == false
return
}
// run the test
t.Run(path, func(t *testing.T) {
rc.testsCasesRun += 1
actualOutput, actualError := getResult()
// Configured to update the assertion files instead of comparing them
if rc.updateExpectedFromActual {
if actualError != "" {
require.NoError(t, ioutil.WriteFile(rc.expectedErrorFilename, []byte(actualError), 0600))
}
return
if len(actualOutput) > 0 {
require.NoError(t, ioutil.WriteFile(rc.expectedOutputFilename, []byte(actualOutput), 0600))
}
t.Skip("Updated fixtures for test case")
}
// Compare the results
// Compare the results to the assertion files
if expectedError != "" {
// We expected an error, so make sure there was one and it matches
require.Error(t, err, actualOutput.String())
require.Contains(t,
standardizeSpacing(err.Error()),
standardizeSpacing(expectedError), actualOutput.String())
// We expected an error, so make sure there was one
require.NotEmptyf(t, actualError, "test expected an error but message was empty, and output was:\n%s", actualOutput)
rc.errorAssertionFunc(t, expectedError, actualError)
} else {
// We didn't expect an error, and the output should match
require.NoError(t, err)
require.Equal(t,
standardizeSpacing(expectedOutput),
standardizeSpacing(actualOutput.String()))
require.Emptyf(t, actualError, "test expected no error but got an error message, and output was:\n%s", actualOutput)
rc.outputAssertionFunc(t, expectedOutput, actualOutput)
}
})
}
// getExpected reads the expected results and error files
func getExpected(t *testing.T, expectedOutFilename, expectedErrFilename string) (string, string) {
// readAssertionFiles reads the expected results and error files
func (rc *resultsChecker) readAssertionFiles(t *testing.T) (string, string) {
// read the expected results
var expectedOutput, expectedError string
if expectedOutFilename != "" {
_, err := os.Stat(expectedOutFilename)
if rc.expectedOutputFilename != "" {
_, err := os.Stat(rc.expectedOutputFilename)
if !os.IsNotExist(err) && err != nil {
t.FailNow()
}
if err == nil {
// only read the file if it exists
b, err := ioutil.ReadFile(expectedOutFilename)
b, err := ioutil.ReadFile(rc.expectedOutputFilename)
if !assert.NoError(t, err) {
t.FailNow()
}
expectedOutput = string(b)
}
}
if expectedErrFilename != "" {
_, err := os.Stat(expectedErrFilename)
if rc.expectedErrorFilename != "" {
_, err := os.Stat(rc.expectedErrorFilename)
if !os.IsNotExist(err) && err != nil {
t.FailNow()
}
if err == nil {
// only read the file if it exists
b, err := ioutil.ReadFile(expectedErrFilename)
b, err := ioutil.ReadFile(rc.expectedErrorFilename)
if !assert.NoError(t, err) {
t.FailNow()
}

17
kyaml/fn/framework/processors_test.go
View File

@@ -11,6 +11,7 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"sigs.k8s.io/kustomize/kyaml/errors"
"sigs.k8s.io/kustomize/kyaml/fn/framework/frameworktestutil"
"sigs.k8s.io/kustomize/kyaml/fn/framework/parser"
"sigs.k8s.io/kustomize/kyaml/openapi"
"sigs.k8s.io/kustomize/kyaml/yaml"
@@ -589,3 +590,19 @@ items:
})
require.Nil(t, found, "openAPI schema was not reset")
}
func TestTemplateProcessor_Validator(t *testing.T) {
// This test proves the Validate method is called when implemented
// and demonstrates the use of ProcessorResultsChecker's error matching
p := func() framework.ResourceListProcessor {
return &framework.VersionedAPIProcessor{FilterProvider: framework.GVKFilterMap{
"JavaSpringBoot": {
"example.com/v1alpha1": &v1alpha1JavaSpringBoot{},
}}}
}
c := frameworktestutil.ProcessorResultsChecker{
TestDataDirectory: "testdata/validation",
Processor: p,
}
c.Assert(t)
}

5
kyaml/fn/framework/testdata/validation/error/errors.txt vendored Normal file
View File

@@ -0,0 +1,5 @@
JavaSpringBoot had 4 errors:
\[\d\] replicas must be less than 10
\[\d\] name is required
\[\d\] image should not have latest tag
\[\d\] domain must be a subdomain of example.com

14
kyaml/fn/framework/testdata/validation/error/input.yaml vendored Normal file
View File

@@ -0,0 +1,14 @@
# Copyright 2021 The Kubernetes Authors.
# SPDX-License-Identifier: Apache-2.0
kind: ResourceList
apiVersion: config.kubernetes.io/v1alpha1
functionConfig:
apiVersion: example.com/v1alpha1
kind: JavaSpringBoot
metadata:
name: ""
spec:
replicas: 1000
image: foo:latest
domain: bad