diff --git a/api/internal/localizer/localizer.go b/api/internal/localizer/localizer.go index 49a0093fe..3897ad2f8 100644 --- a/api/internal/localizer/localizer.go +++ b/api/internal/localizer/localizer.go @@ -11,6 +11,7 @@ import ( pLdr "sigs.k8s.io/kustomize/api/internal/plugins/loader" "sigs.k8s.io/kustomize/api/internal/target" "sigs.k8s.io/kustomize/api/konfig" + "sigs.k8s.io/kustomize/api/loader" "sigs.k8s.io/kustomize/api/resmap" "sigs.k8s.io/kustomize/api/types" "sigs.k8s.io/kustomize/kyaml/errors" @@ -38,8 +39,7 @@ type Localizer struct { func NewLocalizer(ldr *Loader, validator ifc.Validator, rFactory *resmap.Factory, pLdr *pLdr.Loader) (*Localizer, error) { toDst, err := filepath.Rel(ldr.args.Scope.String(), ldr.Root()) if err != nil { - log.Fatalf("cannot find path from directory %q to %q inside directory: %s", ldr.args.Scope.String(), - ldr.Root(), err.Error()) + log.Fatalf("cannot find path from %q to child directory %q: %s", ldr.args.Scope, ldr.Root(), err) } dst := ldr.args.NewDir.Join(toDst) if err = ldr.fSys.MkdirAll(dst); err != nil { @@ -62,9 +62,10 @@ func (lc *Localizer) Localize() error { if err != nil { return errors.Wrap(err) } - - kust := lc.processKust(kt) - + kust, err := lc.processKust(kt) + if err != nil { + return err + } content, err := yaml.Marshal(kust) if err != nil { return errors.WrapPrefixf(err, "unable to serialize localized kustomization file") @@ -75,9 +76,54 @@ func (lc *Localizer) Localize() error { return nil } -// TODO(annasong): implement // processKust returns a copy of the kustomization at kt with paths localized. -func (lc *Localizer) processKust(kt *target.KustTarget) *types.Kustomization { +func (lc *Localizer) processKust(kt *target.KustTarget) (*types.Kustomization, error) { kust := kt.Kustomization() - return &kust + for i := range kust.Patches { + if kust.Patches[i].Path != "" { + newPath, err := lc.localizeFile(kust.Patches[i].Path) + if err != nil { + return nil, errors.WrapPrefixf(err, "unable to localize patches path %q", kust.Patches[i].Path) + } + kust.Patches[i].Path = newPath + } + } + // TODO(annasong): localize all other kustomization fields: resources, components, crds, configurations, + // openapi, patchesStrategicMerge, replacements, configMapGenerators, secretGenerators + // TODO(annasong): localize built-in plugins under generators, transformers, and validators fields + return &kust, nil +} + +// localizeFile localizes file path and returns the localized path +func (lc *Localizer) localizeFile(path string) (string, error) { + content, err := lc.ldr.Load(path) + if err != nil { + return "", errors.Wrap(err) + } + + var locPath string + if loader.IsRemoteFile(path) { + // TODO(annasong): check if able to add localize directory + locPath = locFilePath(path) + } else { + // ldr has checked that path must be relative; this is subject to change in beta. + + // We must clean path to: + // 1. avoid symlinks. A `kustomize build` run will fail if we write files to + // symlink paths outside the current root, given that we don't want to recreate + // the symlinks. Even worse, we could be writing files outside the localize destination. + // 2. avoid paths that temporarily traverse outside the current root, + // i.e. ../../../scope/target/current-root. The localized file will be surrounded by + // different directories than its source, and so an uncleaned path may no longer be valid. + locPath = cleanFilePath(lc.fSys, filesys.ConfirmedDir(lc.ldr.Root()), path) + // TODO(annasong): check if hits localize directory + } + absPath := lc.dst.Join(locPath) + if err = lc.fSys.MkdirAll(filepath.Dir(absPath)); err != nil { + return "", errors.WrapPrefixf(err, "unable to create directories to localize file %q", path) + } + if err = lc.fSys.WriteFile(absPath, content); err != nil { + return "", errors.WrapPrefixf(err, "unable to localize file %q", path) + } + return locPath, nil } diff --git a/api/internal/localizer/localizer_test.go b/api/internal/localizer/localizer_test.go index 596881cca..82a5709c6 100644 --- a/api/internal/localizer/localizer_test.go +++ b/api/internal/localizer/localizer_test.go @@ -4,9 +4,12 @@ package localizer_test import ( + "fmt" + "io/fs" "path/filepath" "testing" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "sigs.k8s.io/kustomize/api/hasher" . "sigs.k8s.io/kustomize/api/internal/localizer" @@ -27,7 +30,7 @@ spec: - name: nginx image: nginx:1.14.2 ports: - -containerPort: 80 + - containerPort: 80 ` func makeMemoryFs(t *testing.T) filesys.FileSystem { @@ -71,36 +74,101 @@ func createLocalizer(t *testing.T, fSys filesys.FileSystem, target string, scope return lc } +func checkFSys(t *testing.T, fSysExpected filesys.FileSystem, fSysActual filesys.FileSystem) { + t.Helper() + + assert.Equal(t, fSysExpected, fSysActual) + if t.Failed() { + reportFSysDiff(t, fSysExpected, fSysActual) + } +} + +func reportFSysDiff(t *testing.T, fSysExpected filesys.FileSystem, fSysActual filesys.FileSystem) { + t.Helper() + + visited := make(map[string]struct{}) + err := fSysActual.Walk("/", func(path string, info fs.FileInfo, err error) error { + require.NoError(t, err) + visited[path] = struct{}{} + + if info.IsDir() { + assert.Truef(t, fSysExpected.IsDir(path), "unexpected directory %q", path) + } else { + actualContent, readErr := fSysActual.ReadFile(path) + require.NoError(t, readErr) + expectedContent, findErr := fSysExpected.ReadFile(path) + assert.NoError(t, findErr) + if findErr == nil { + assert.Equal(t, string(expectedContent), string(actualContent)) + } + } + return nil + }) + require.NoError(t, err) + + err = fSysExpected.Walk("/", func(path string, info fs.FileInfo, err error) error { + require.NoError(t, err) + visited[path] = struct{}{} + + if _, exists := visited[path]; !exists { + t.Errorf("expected path %q not found", path) + } + return nil + }) + require.NoError(t, err) +} + func TestNewLocalizerTargetIsScope(t *testing.T) { fSys := makeMemoryFs(t) - _ = createLocalizer(t, fSys, "/a", "", "/a/b/dst") + kustomization := map[string]string{ + "kustomization.yaml": `apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namePrefix: my- +`, + } + addFiles(t, fSys, "/a", kustomization) + lclzr := createLocalizer(t, fSys, "/a", "", "/a/b/dst") + require.NoError(t, lclzr.Localize()) fSysExpected := makeMemoryFs(t) - require.NoError(t, fSysExpected.MkdirAll("/a/b/dst")) - require.Equal(t, fSysExpected, fSys) + addFiles(t, fSysExpected, "/a", kustomization) + addFiles(t, fSysExpected, "/a/b/dst", kustomization) + checkFSys(t, fSysExpected, fSys) } func TestNewLocalizerTargetNestedInScope(t *testing.T) { fSys := makeMemoryFs(t) - _ = createLocalizer(t, fSys, "/a/b", "/", "/a/b/dst") + kustomization := map[string]string{ + "kustomization.yaml": `apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +patches: +- patch: |- + - op: replace + path: /some/existing/path + value: new value + target: + kind: Deployment + labelSelector: env=dev +`, + } + addFiles(t, fSys, "/a/b", kustomization) + lclzr := createLocalizer(t, fSys, "/a/b", "/", "/a/b/dst") + require.NoError(t, lclzr.Localize()) fSysExpected := makeMemoryFs(t) - require.NoError(t, fSysExpected.MkdirAll("/a/b/dst/a/b")) - require.Equal(t, fSysExpected, fSys) + addFiles(t, fSysExpected, "/a/b", kustomization) + addFiles(t, fSysExpected, "/a/b/dst/a/b", kustomization) + checkFSys(t, fSysExpected, fSys) } func TestLocalizeKustomizationName(t *testing.T) { fSys := makeMemoryFs(t) kustomization := map[string]string{ "Kustomization": `apiVersion: kustomize.config.k8s.io/v1beta1 -configMapGenerator: -- behavior: create - literals: - - APPLE=orange - name: map +commonLabels: + label-one: value-one + label-two: value-two kind: Kustomization -resources: -- pod.yaml `, } addFiles(t, fSys, "/a", kustomization) @@ -113,5 +181,121 @@ resources: addFiles(t, fSysExpected, "/dst/a", map[string]string{ "kustomization.yaml": kustomization["Kustomization"], }) - require.Equal(t, fSysExpected, fSys) + checkFSys(t, fSysExpected, fSys) +} + +func TestLocalizeFileName(t *testing.T) { + for name, path := range map[string]string{ + "nested_directories": "a/b/c/d/patch.yaml", + "localize_dir_name_when_absent": LocalizeDir, + "in_localize_dir_name_when_absent": fmt.Sprintf("%s/patch.yaml", LocalizeDir), + "no_file_extension": "patch", + "kustomization_name": "a/kustomization.yaml", + } { + t.Run(name, func(t *testing.T) { + fSys := makeMemoryFs(t) + kustAndPatch := map[string]string{ + "kustomization.yaml": fmt.Sprintf(`apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +patches: +- path: %s +`, path), + path: podConfiguration, + } + addFiles(t, fSys, "/a", kustAndPatch) + + lclzr := createLocalizer(t, fSys, "/a", "/", "/a/dst") + require.NoError(t, lclzr.Localize()) + + fSysExpected := makeMemoryFs(t) + addFiles(t, fSysExpected, "/a", kustAndPatch) + addFiles(t, fSysExpected, "/a/dst/a", kustAndPatch) + checkFSys(t, fSysExpected, fSys) + }) + } +} + +func TestLocalizeFileCleaned(t *testing.T) { + fSys := makeMemoryFs(t) + kustAndPatch := map[string]string{ + "kustomization.yaml": `apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +patches: +- path: ../gamma/../../../alpha/beta/./gamma/patch.yaml +`, + "patch.yaml": podConfiguration, + } + addFiles(t, fSys, "/alpha/beta/gamma", kustAndPatch) + + lclzr := createLocalizer(t, fSys, "/alpha/beta/gamma", "/", "") + require.NoError(t, lclzr.Localize()) + + fSysExpected := makeMemoryFs(t) + addFiles(t, fSysExpected, "/alpha/beta/gamma", kustAndPatch) + addFiles(t, fSysExpected, "/localized-gamma/alpha/beta/gamma", map[string]string{ + "kustomization.yaml": `apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +patches: +- path: patch.yaml +`, + "patch.yaml": podConfiguration, + }) + checkFSys(t, fSysExpected, fSys) +} + +func TestLocalizePatches(t *testing.T) { + fSys := makeMemoryFs(t) + kustAndPatch := map[string]string{ + "kustomization.yaml": `apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +patches: +- patch: |- + apiVersion: v1 + kind: Deployment + metadata: + labels: + app.kubernetes.io/version: 1.21.0 + name: dummy-app + target: + labelSelector: app.kubernetes.io/name=nginx +- options: + allowNameChange: true + path: patch.yaml +`, + "patch.yaml": `apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Deployment +metadata: + name: not-used +spec: + template: + spec: + containers: + - name: nginx + image: nginx:1.21.0 +`, + } + addFiles(t, fSys, "/", kustAndPatch) + + lclzr := createLocalizer(t, fSys, "/", "", "") + require.NoError(t, lclzr.Localize()) + + fSysExpected := makeMemoryFs(t) + addFiles(t, fSysExpected, "/", kustAndPatch) + addFiles(t, fSysExpected, "/localized", kustAndPatch) + checkFSys(t, fSysExpected, fSys) +} + +func TestLocalizeFileNoFile(t *testing.T) { + fSys := makeMemoryFs(t) + kustAndPatch := map[string]string{ + "kustomization.yaml": `apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +patches: +- path: name-DNE.yaml +`, + } + addFiles(t, fSys, "/a/b", kustAndPatch) + + lclzr := createLocalizer(t, fSys, "/a/b", "", "/dst") + require.Error(t, lclzr.Localize()) } diff --git a/api/internal/localizer/locloader.go b/api/internal/localizer/locloader.go index c29d355ed..68ed745a9 100644 --- a/api/internal/localizer/locloader.go +++ b/api/internal/localizer/locloader.go @@ -4,7 +4,6 @@ package localizer import ( - "log" "path/filepath" "sigs.k8s.io/kustomize/api/ifc" @@ -14,14 +13,13 @@ import ( "sigs.k8s.io/kustomize/kyaml/filesys" ) -const DstPrefix = "localized" - // Args holds localize arguments type Args struct { // target; local copy if remote Target filesys.ConfirmedDir - // directory that bounds target's local references, empty string if target is remote + // directory that bounds target's local references + // repo directory of local copy if target is remote Scope filesys.ConfirmedDir // localize destination @@ -95,18 +93,14 @@ func (ll *Loader) Load(path string) ([]byte, error) { return nil, errors.Errorf("absolute paths not yet supported in alpha: file path %q is absolute", path) } if ll.local { - abs := filepath.Join(ll.Root(), path) - dir, f, err := ll.fSys.CleanedAbs(abs) - if err != nil { - // should never happen - log.Fatalf(errors.WrapPrefixf(err, "cannot clean validated file path %q", abs).Error()) - } + cleanPath := cleanFilePath(ll.fSys, filesys.ConfirmedDir(ll.Root()), path) + cleanAbs := filepath.Join(ll.Root(), cleanPath) + dir := filesys.ConfirmedDir(filepath.Dir(cleanAbs)) // target cannot reference newDir, as this load would've failed prior to localize; // not a problem if remote because then reference could only be in newDir if repo copy, // which will be cleaned, is inside newDir if dir.HasPrefix(ll.args.NewDir) { - return nil, errors.Errorf( - "file path %q references into localize destination %q", dir.Join(f), ll.args.NewDir) + return nil, errors.Errorf("file %q at %q enters localize destination %q", path, cleanAbs, ll.args.NewDir) } } return content, nil diff --git a/api/internal/localizer/util.go b/api/internal/localizer/util.go index dfbd8fc91..500da8ccc 100644 --- a/api/internal/localizer/util.go +++ b/api/internal/localizer/util.go @@ -14,7 +14,13 @@ import ( "sigs.k8s.io/kustomize/kyaml/filesys" ) -// establishScope returns the effective scope given localize arguments and targetLdr at rawTarget. For remote targetArg, +// DstPrefix prefixes the target and ref, if target is remote, in the default localize destination directory name +const DstPrefix = "localized" + +// LocalizeDir is the name of the localize directories used to store remote content in the localize destination +const LocalizeDir = "localized-files" + +// establishScope returns the effective scope given localize arguments and targetLdr at rawTarget. For remote rawTarget, // the effective scope is the downloaded repo. func establishScope(rawScope string, rawTarget string, targetLdr ifc.Loader, fSys filesys.FileSystem) (filesys.ConfirmedDir, error) { if repo := targetLdr.Repo(); repo != "" { @@ -54,7 +60,7 @@ func createNewDir(rawNewDir string, targetLdr ifc.Loader, spec *git.RepoSpec, fS newDir, err := filesys.ConfirmDir(fSys, rawNewDir) if err != nil { if errCleanup := fSys.RemoveAll(newDir.String()); errCleanup != nil { - log.Printf("%s", errors.WrapPrefixf(errCleanup, "unable to clean localize destination").Error()) + log.Printf("%s", errors.WrapPrefixf(errCleanup, "unable to clean localize destination")) } return "", errors.WrapPrefixf(err, "unable to establish localize destination") } @@ -95,7 +101,27 @@ func urlBase(url string) string { func hasRef(repoURL string) bool { repoSpec, err := git.NewRepoSpecFromURL(repoURL) if err != nil { - log.Fatalf("unable to parse validated root url: %s", err.Error()) + log.Fatalf("unable to parse validated root url: %s", err) } return repoSpec.Ref != "" } + +// cleanFilePath returns file cleaned, where file is a relative path to root on fSys +func cleanFilePath(fSys filesys.FileSystem, root filesys.ConfirmedDir, file string) string { + abs := root.Join(file) + dir, f, err := fSys.CleanedAbs(abs) + if err != nil { + log.Fatalf("cannot clean validated file path %q: %s", abs, err) + } + locPath, err := filepath.Rel(root.String(), dir.Join(f)) + if err != nil { + log.Fatalf("cannot find path from parent %q to file %q: %s", root, dir.Join(f), err) + } + return locPath +} + +// locFilePath returns the relative localized path of validated file url fileURL +// TODO(annasong): implement +func locFilePath(_ string) string { + return filepath.Join(LocalizeDir, "") +}