Localize fields: openapi, configurations, crds (#4907)

* Localize openapi, configurations, crds

* Add integration test

* Move krusty test

* Address code review feedback
This commit is contained in:
Anna Song
2022-12-22 15:07:26 -05:00
committed by GitHub
parent a1bfab382a
commit de6162625f
5 changed files with 407 additions and 166 deletions

View File

@@ -118,12 +118,38 @@ func (lc *localizer) load() (*types.Kustomization, string, error) {
// localizeNativeFields localizes paths on kustomize-native fields, like configMapGenerator, that kustomize has a
// built-in understanding of. This excludes helm-related fields, such as `helmGlobals` and `helmCharts`.
func (lc *localizer) localizeNativeFields(kust *types.Kustomization) error {
for i, path := range kust.Components {
newPath, err := lc.localizeDir(path)
if path, exists := kust.OpenAPI["path"]; exists {
newPath, err := lc.localizeFile(path)
if err != nil {
return errors.WrapPrefixf(err, "unable to localize components field")
return errors.WrapPrefixf(err, "unable to localize openapi path")
}
kust.OpenAPI["path"] = newPath
}
for fieldName, field := range map[string]struct {
paths []string
locFn func(string) (string, error)
}{
"components": {
kust.Components,
lc.localizeDir,
},
"configurations": {
kust.Configurations,
lc.localizeFile,
},
"crds": {
kust.Crds,
lc.localizeFile,
},
} {
for i, path := range field.paths {
newPath, err := field.locFn(path)
if err != nil {
return errors.WrapPrefixf(err, "unable to localize %s path", fieldName)
}
field.paths[i] = newPath
}
kust.Components[i] = newPath
}
for i := range kust.ConfigMapGenerator {
@@ -168,8 +194,7 @@ func (lc *localizer) localizeNativeFields(kust *types.Kustomization) error {
}
}
// TODO(annasong): localize all other kustomization fields: resources, bases, crds, configurations,
// openapi, configMapGenerator.env, secretGenerator.env
// TODO(annasong): localize all other kustomization fields: resources, bases, configMapGenerator.env, secretGenerator.env
return nil
}

View File

@@ -103,6 +103,21 @@ func reportFSysDiff(t *testing.T, fSysExpected filesys.FileSystem, fSysActual fi
require.NoError(t, err)
}
func checkLocalizeInTargetSuccess(t *testing.T, files map[string]string) {
t.Helper()
fSys := makeMemoryFs(t)
addFiles(t, fSys, "/a", files)
err := Run("/a", "/", "dst", fSys)
require.NoError(t, err)
fSysExpected := makeMemoryFs(t)
addFiles(t, fSysExpected, "/a", files)
addFiles(t, fSysExpected, "/dst/a", files)
checkFSys(t, fSysExpected, fSys)
}
func TestTargetIsScope(t *testing.T) {
kustomization := map[string]string{
"kustomization.yaml": `apiVersion: kustomize.config.k8s.io/v1beta1
@@ -151,13 +166,7 @@ commonLabels:
kind: Kustomization
`,
}
fSysExpected, fSysActual := makeFileSystems(t, "/a", kustomization)
err := Run("/a", "/", "/dst", fSysActual)
require.NoError(t, err)
addFiles(t, fSysExpected, "/dst/a", kustomization)
checkFSys(t, fSysExpected, fSysActual)
checkLocalizeInTargetSuccess(t, kustomization)
}
func TestLoadGVKNN(t *testing.T) {
@@ -171,13 +180,7 @@ func TestLoadGVKNN(t *testing.T) {
files := map[string]string{
"kustomization.yaml": kustomization,
}
fSysExpected, fSysActual := makeFileSystems(t, "/a", files)
err := Run("/a", "/a", "/dst", fSysActual)
require.NoError(t, err)
addFiles(t, fSysExpected, "/dst", files)
checkFSys(t, fSysExpected, fSysActual)
checkLocalizeInTargetSuccess(t, files)
})
}
}
@@ -197,25 +200,7 @@ imageTags:
kind: Kustomization
`,
}
fSysExpected, fSysActual := makeFileSystems(t, "/alpha", kustomization)
err := Run("/alpha", "/alpha", "/beta", fSysActual)
require.NoError(t, err)
addFiles(t, fSysExpected, "/beta", map[string]string{
"kustomization.yaml": `apiVersion: kustomize.config.k8s.io/v1beta1
bases:
- beta
configMapGenerator:
- env: env.properties
imageTags:
- name: postgres
newName: my-registry/my-postgres
newTag: v1
kind: Kustomization
`,
})
checkFSys(t, fSysExpected, fSysActual)
checkLocalizeInTargetSuccess(t, kustomization)
}
func TestLoadUnknownKustFields(t *testing.T) {
@@ -247,13 +232,7 @@ patches:
`, path),
path: podConfiguration,
}
expected, actual := makeFileSystems(t, "/a", kustAndPatch)
err := Run("/a", "/", "/a/dst", actual)
require.NoError(t, err)
addFiles(t, expected, "/a/dst/a", kustAndPatch)
checkFSys(t, expected, actual)
checkLocalizeInTargetSuccess(t, kustAndPatch)
})
}
}
@@ -292,9 +271,9 @@ configMapGenerator:
name: referenced-file
kind: Kustomization
`,
"env": "APPLE=orange",
"env.properties": "USERNAME=password",
"resource.yaml": podConfiguration,
"env": "APPLE=orange",
"env.properties": "USERNAME=password",
"dir/resource.yaml": podConfiguration,
}
expected, actual := makeFileSystems(t, "/alpha/beta", targetAndUnreferenced)
@@ -326,25 +305,95 @@ patches:
allowNameChange: true
path: patch.yaml
`,
"patch.yaml": `apiVersion: kustomize.config.k8s.io/v1beta1
kind: Deployment
metadata:
name: not-used
spec:
template:
spec:
containers:
- name: nginx
image: nginx:1.21.0
`,
"patch.yaml": podConfiguration,
}
expected, actual := makeFileSystems(t, "/", kustAndPatch)
checkLocalizeInTargetSuccess(t, kustAndPatch)
}
err := Run("/", "", "", actual)
require.NoError(t, err)
func TestLocalizeOpenAPI(t *testing.T) {
type testCase struct {
name string
files map[string]string
}
for _, test := range []testCase{
{
name: "no_path",
files: map[string]string{
"kustomization.yaml": `apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
openapi:
version: v1.20.4
`,
},
},
{
name: "path",
files: map[string]string{
"kustomization.yaml": `apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
openapi:
path: openapi.json
`,
"openapi.json": `{
"definitions": {
"io.k8s.apimachinery.pkg.apis.meta.v1.ObjectMeta": {
"properties": {
"name": {
"type": "string"
}
},
"type": "object"
}
}
}`,
},
},
} {
t.Run(test.name, func(t *testing.T) {
checkLocalizeInTargetSuccess(t, test.files)
})
}
}
addFiles(t, expected, "/localized", kustAndPatch)
checkFSys(t, expected, actual)
func TestLocalizeConfigurations(t *testing.T) {
kustAndConfigs := map[string]string{
"kustomization.yaml": `apiVersion: kustomize.config.k8s.io/v1beta1
configurations:
- commonLabels.yaml
- namePrefix.yaml
kind: Kustomization
`,
"commonLabels.yaml": `commonLabels:
- path: new/path
create: true`,
"namePrefix.yaml": `namePrefix:
- version: v1
path: metadata/name
- group: custom
path: metadata/name`,
}
checkLocalizeInTargetSuccess(t, kustAndConfigs)
}
func TestLocalizeCrds(t *testing.T) {
kustAndCrds := map[string]string{
"kustomization.yaml": `apiVersion: kustomize.config.k8s.io/v1beta1
crds:
- crd1.yaml
- crd2.yaml
kind: Kustomization
`,
"crd1.yaml": `apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
name: controller.stable.example.com`,
"crd2.yaml": `apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
name: crontabs.stable.example.com
scope: Cluster`,
}
checkLocalizeInTargetSuccess(t, kustAndCrds)
}
func TestLocalizePatchesJson(t *testing.T) {
@@ -376,13 +425,7 @@ patchesJson6902:
{"op": "remove", "path": "/some/existing/path"},
]`,
}
expected, actual := makeFileSystems(t, "/alpha/beta", kustAndPatches)
err := Run("/alpha/beta", "/", "/beta", actual)
require.NoError(t, err)
addFiles(t, expected, "/beta/alpha/beta", kustAndPatches)
checkFSys(t, expected, actual)
checkLocalizeInTargetSuccess(t, kustAndPatches)
}
func TestLocalizePatchesSM(t *testing.T) {
@@ -401,13 +444,7 @@ patchesStrategicMerge:
`,
"patch.yaml": podConfiguration,
}
expected, actual := makeFileSystems(t, "/a", kustAndPatches)
err := Run("/a", "", "/dst", actual)
require.NoError(t, err)
addFiles(t, expected, "/dst", kustAndPatches)
checkFSys(t, expected, actual)
checkLocalizeInTargetSuccess(t, kustAndPatches)
}
func TestLocalizeReplacements(t *testing.T) {
@@ -426,7 +463,7 @@ replacements:
name: my-map
`,
"replacement.yaml": `source:
fieldPath: path.*.to.[some=field]
fieldPath: path.to.some.field
kind: Pod
options:
delimiter: /
@@ -434,19 +471,14 @@ targets:
- fieldPaths:
- config\.kubernetes\.io.annotations
- second.path
- path.*.to.[some=field]
reject:
- group: apps
version: v2
select:
namespace: my`,
}
expected, actual := makeFileSystems(t, "/a", kustAndReplacement)
err := Run("/a", "/", "/dst", actual)
require.NoError(t, err)
addFiles(t, expected, "/dst/a", kustAndReplacement)
checkFSys(t, expected, actual)
checkLocalizeInTargetSuccess(t, kustAndReplacement)
}
func TestLocalizeConfigMapGenerator(t *testing.T) {
@@ -471,13 +503,7 @@ metadata:
IS_GLOBAL=true`,
"key.properties": "value",
}
expected, actual := makeFileSystems(t, "/a/b", kustAndData)
err := Run("/a/b", "", "", actual)
require.NoError(t, err)
addFiles(t, expected, "/localized-b", kustAndData)
checkFSys(t, expected, actual)
checkLocalizeInTargetSuccess(t, kustAndData)
}
func TestLocalizeSecretGenerator(t *testing.T) {
@@ -504,13 +530,7 @@ secretGenerator:
"b/value.properties": "dmFsdWU=",
"b/value": "dmFsdWU=",
}
expected, actual := makeFileSystems(t, "/a", kustAndData)
err := Run("/a", "/", "/localized-a", actual)
require.NoError(t, err)
addFiles(t, expected, "/localized-a/a", kustAndData)
checkFSys(t, expected, actual)
checkLocalizeInTargetSuccess(t, kustAndData)
}
func TestLocalizeFileNoFile(t *testing.T) {
@@ -551,13 +571,7 @@ path: patchSM-two.yaml
"patchSM-one.yaml": podConfiguration,
"patchSM-two.yaml": podConfiguration,
}
expected, actual := makeFileSystems(t, "/a", kustAndPlugins)
err := Run("/a", "", "/dst", actual)
require.NoError(t, err)
addFiles(t, expected, "/dst", kustAndPlugins)
checkFSys(t, expected, actual)
checkLocalizeInTargetSuccess(t, kustAndPlugins)
}
func TestLocalizeMultiplePluginsInEntry(t *testing.T) {
@@ -581,13 +595,7 @@ transformers:
"patchSM-one.yaml": podConfiguration,
"patchSM-two.yaml": podConfiguration,
}
expected, actual := makeFileSystems(t, "/a", kustAndPlugins)
err := Run("/a", "", "/dst", actual)
require.NoError(t, err)
addFiles(t, expected, "/dst", kustAndPlugins)
checkFSys(t, expected, actual)
checkLocalizeInTargetSuccess(t, kustAndPlugins)
}
func TestLocalizeCleanedPathInPath(t *testing.T) {
@@ -649,13 +657,7 @@ metadata:
name: map
`,
}
expected, actual := makeFileSystems(t, "/a", kustAndPlugins)
err := Run("/a", "", "/alpha/dst", actual)
require.NoError(t, err)
addFiles(t, expected, "/alpha/dst", kustAndPlugins)
checkFSys(t, expected, actual)
checkLocalizeInTargetSuccess(t, kustAndPlugins)
}
func TestLocalizeTransformersPatch(t *testing.T) {
@@ -681,13 +683,7 @@ path: patchSM.yaml
`,
"patchSM.yaml": podConfiguration,
}
expected, actual := makeFileSystems(t, "/a", kustAndPatches)
err := Run("/a", "", "/dst", actual)
require.NoError(t, err)
addFiles(t, expected, "/dst", kustAndPatches)
checkFSys(t, expected, actual)
checkLocalizeInTargetSuccess(t, kustAndPatches)
}
func TestLocalizeTransformersPatchJson(t *testing.T) {
@@ -722,13 +718,7 @@ target:
]
`,
}
expected, actual := makeFileSystems(t, "/a", kustAndPatches)
err := Run("/a", "", "/dst", actual)
require.NoError(t, err)
addFiles(t, expected, "/dst", kustAndPatches)
checkFSys(t, expected, actual)
checkLocalizeInTargetSuccess(t, kustAndPatches)
}
func TestLocalizePluginsNoPaths(t *testing.T) {
@@ -751,13 +741,7 @@ metadata:
prefix: copy
`,
}
expected, actual := makeFileSystems(t, "/a", kustAndPlugins)
err := Run("/a", "", "/dst", actual)
require.NoError(t, err)
addFiles(t, expected, "/dst", kustAndPlugins)
checkFSys(t, expected, actual)
checkLocalizeInTargetSuccess(t, kustAndPlugins)
}
func TestLocalizeValidators(t *testing.T) {
@@ -792,18 +776,12 @@ replacements:
namespace: test
targets:
- fieldPaths:
- path
- path.*.to.[some=field]
select:
namespace: test
`,
}
expected, actual := makeFileSystems(t, "/", kustAndPlugin)
err := Run("/", "", "/dst", actual)
require.NoError(t, err)
addFiles(t, expected, "/dst", kustAndPlugin)
checkFSys(t, expected, actual)
checkLocalizeInTargetSuccess(t, kustAndPlugin)
}
func TestLocalizeBuiltinPluginsNotResource(t *testing.T) {
@@ -959,13 +937,7 @@ namespace: kustomize-namespace
},
} {
t.Run(tc.name, func(t *testing.T) {
expected, actual := makeFileSystems(t, "/alpha/beta/gamma", tc.files)
err := Run("/alpha/beta/gamma", "/alpha/beta", "/dst", actual)
require.NoError(t, err)
addFiles(t, expected, "/dst/gamma", tc.files)
checkFSys(t, expected, actual)
checkLocalizeInTargetSuccess(t, tc.files)
})
}
}
@@ -1017,11 +989,5 @@ kind: Component
nameSuffix: -test
`,
}
expected, actual := makeFileSystems(t, "/", kustAndComponents)
err := Run("/", "", "", actual)
require.NoError(t, err)
addFiles(t, expected, "/localized", kustAndComponents)
checkFSys(t, expected, actual)
checkLocalizeInTargetSuccess(t, kustAndComponents)
}

View File

@@ -92,7 +92,7 @@ func (ll *Loader) Load(path string) ([]byte, error) {
if filepath.IsAbs(path) {
return nil, errors.Errorf("absolute paths not yet supported in alpha: file path %q is absolute", path)
}
if ll.local {
if !loader.IsRemoteFile(path) && ll.local {
cleanPath := cleanFilePath(ll.fSys, filesys.ConfirmedDir(ll.Root()), path)
cleanAbs := filepath.Join(ll.Root(), cleanPath)
dir := filesys.ConfirmedDir(filepath.Dir(cleanAbs))

View File

@@ -0,0 +1,15 @@
// Copyright 2022 The Kubernetes Authors.
// SPDX-License-Identifier: Apache-2.0
package localizer
import (
"sigs.k8s.io/kustomize/api/internal/localizer"
"sigs.k8s.io/kustomize/kyaml/errors"
"sigs.k8s.io/kustomize/kyaml/filesys"
)
// Run `kustomize localize`s files referenced by kustomization target in scope to destination newDir on fSys.
func Run(fSys filesys.FileSystem, target, scope, newDir string) error {
return errors.Wrap(localizer.Run(target, scope, newDir, fSys))
}

View File

@@ -0,0 +1,235 @@
// Copyright 2022 The Kubernetes Authors.
// SPDX-License-Identifier: Apache-2.0
package localizer_test
import (
"fmt"
"io/fs"
"os"
"path/filepath"
"testing"
"github.com/stretchr/testify/require"
. "sigs.k8s.io/kustomize/api/internal/localizer"
"sigs.k8s.io/kustomize/api/krusty/localizer"
"sigs.k8s.io/kustomize/kyaml/filesys"
)
const (
customSchema = `{
"definitions": {
"v1alpha1.MyCRD": {
"properties": {
"apiVersion": {
"type": "string"
},
"kind": {
"type": "string"
},
"metadata": {
"type": "object"
},
"spec": {
"properties": {
"template": {
"$ref": "#/definitions/io.k8s.api.core.v1.PodTemplateSpec"
}
},
"type": "object"
},
"status": {
"properties": {
"success": {
"type": "boolean"
}
},
"type": "object"
}
},
"type": "object",
"x-kubernetes-group-version-kind": [
{
"group": "example.com",
"kind": "MyCRD",
"version": "v1alpha1"
},
{
"group": "",
"kind": "MyCRD",
"version": "v1alpha1"
}
]
},
"io.k8s.api.core.v1.PodTemplateSpec": {
"properties": {
"metadata": {
"$ref": "#/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.ObjectMeta"
},
"spec": {
"$ref": "#/definitions/io.k8s.api.core.v1.PodSpec"
}
},
"type": "object"
},
"io.k8s.apimachinery.pkg.apis.meta.v1.ObjectMeta": {
"properties": {
"name": {
"type": "string"
}
},
"type": "object"
},
"io.k8s.api.core.v1.PodSpec": {
"properties": {
"containers": {
"items": {
"$ref": "#/definitions/io.k8s.api.core.v1.Container"
},
"type": "array",
"x-kubernetes-patch-merge-key": "name",
"x-kubernetes-patch-strategy": "merge"
}
},
"type": "object"
},
"io.k8s.api.core.v1.Container": {
"properties": {
"command": {
"items": {
"type": "string"
},
"type": "array"
},
"image": {
"type": "string"
},
"name": {
"type": "string"
},
"ports": {
"items": {
"$ref": "#/definitions/io.k8s.api.core.v1.ContainerPort"
},
"type": "array",
"x-kubernetes-list-map-keys": [
"containerPort",
"protocol"
],
"x-kubernetes-list-type": "map",
"x-kubernetes-patch-merge-key": "containerPort",
"x-kubernetes-patch-strategy": "merge"
}
},
"type": "object"
},
"io.k8s.api.core.v1.ContainerPort": {
"properties": {
"containerPort": {
"format": "int32",
"type": "integer"
},
"name": {
"type": "string"
},
"protocol": {
"type": "string"
}
},
"type": "object"
}
}
}
`
)
func prepareFs(t *testing.T, files map[string]string) (memoryFs filesys.FileSystem, actualFs filesys.FileSystem, testDir filesys.ConfirmedDir) {
t.Helper()
memoryFs = filesys.MakeFsInMemory()
actualFs = filesys.MakeFsOnDisk()
testDir, err := filesys.NewTmpConfirmedDir()
require.NoError(t, err)
setupDir(t, memoryFs, testDir.String(), files)
setupDir(t, actualFs, testDir.String(), files)
t.Cleanup(func() {
_ = actualFs.RemoveAll(testDir.String())
})
return memoryFs, actualFs, testDir
}
func setupDir(t *testing.T, targetFs filesys.FileSystem, parentDir string, files map[string]string) {
t.Helper()
for file, content := range files {
require.NoError(t, targetFs.WriteFile(filepath.Join(parentDir, file), []byte(content)))
}
}
func getLocFilePath(t *testing.T, pathFromTestdata []string) string {
t.Helper()
localizedPathDirs := []string{LocalizeDir, "raw.githubusercontent.com", "kubernetes-sigs", "kustomize",
"kustomize", "v4.5.7", "api", "krusty", "testdata"}
return filepath.Join(append(localizedPathDirs, pathFromTestdata...)...)
}
// checkFs checks fsActual, the real file system, against fsExpected, a file system in memory, for contents
// in directory walkDir.
func checkFs(t *testing.T, walkDir string, fsExpected filesys.FileSystem, fsActual filesys.FileSystem) {
t.Helper()
err := fsExpected.Walk(walkDir, func(path string, info fs.FileInfo, err error) error {
require.NoError(t, err)
if info.IsDir() {
require.DirExists(t, path)
} else {
require.FileExists(t, path)
expectedContent, err := fsExpected.ReadFile(path)
require.NoError(t, err)
actualContent, err := fsActual.ReadFile(path)
require.NoError(t, err)
require.Equal(t, string(expectedContent), string(actualContent))
}
return nil
})
require.NoError(t, err)
err = fsActual.Walk(walkDir, func(path string, info fs.FileInfo, err error) error {
require.NoError(t, err)
// no symlinks yet
require.NotEqual(t, os.ModeSymlink, info.Mode()&os.ModeSymlink)
require.True(t, fsExpected.Exists(path))
return nil
})
require.NoError(t, err)
}
func TestRemoteFile(t *testing.T) {
const kustf = `apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
openapi:
path: %s
`
fsExpected, fsActual, testDir := prepareFs(t, map[string]string{
"kustomization.yaml": fmt.Sprintf(kustf, `https://raw.githubusercontent.com/kubernetes-sigs/kustomize/kustomize/v4.5.7/api/krusty/testdata/customschema.json`),
})
dst := testDir.Join("dst")
err := localizer.Run(fsActual, testDir.String(), "", dst)
require.NoError(t, err)
localizedPath := getLocFilePath(t, []string{"customschema.json"})
setupDir(t, fsExpected, dst, map[string]string{
"kustomization.yaml": fmt.Sprintf(kustf, localizedPath),
localizedPath: customSchema,
})
checkFs(t, testDir.String(), fsExpected, fsActual)
}