Merge pull request #204 from Liujingfang1/diamond

Add support for using common base

examples/README.md

@@ -36,3 +36,5 @@ go get github.com/kubernetes-sigs/kustomize
  * [container args](wordpress/README.md) - Injecting k8s runtime data into container arguments (e.g. to point wordpress to a SQL service).
  
  * [image tags](imageTags.md) - Updating image tags without applying a patch.
+
+ * [multibases](multibases/README.md) - Composing three variants (dev, staging, production) with a common base.

examples/multibases/README.md

@@ -0,0 +1,126 @@
+# Demo: multibases with a common base
+
+`kustomize` encourages defining multiple variants - e.g. dev, staging and prod, as overlays on a common base.
+
+It's possible to create an additional overlay to compose these variants together - just declare the overlays as the bases of a new kustomization.
+
+This is also a means to apply a common label or annotation across the variants, if for some reason the base isn't under your control. It also allows one to define a left-most namePrefix across the variants - something that cannot be done by modifying the common base.
+
+The following demonstrates this using a base that's just one pod.
+
+Define a place to work:
+
+<!-- @makeWorkplace @test -->
+```
+DEMO_HOME=$(mktemp -d)
+```
+
+Define a common base:
+<!-- @makeBase @test -->
+```
+BASE=$DEMO_HOME/base
+mkdir $BASE
+
+cat <<EOF >$BASE/kustomization.yaml
+resources:
+- pod.yaml
+EOF
+
+cat <<EOF >$BASE/pod.yaml
+apiVersion: v1
+kind: Pod
+metadata:
+  name: myapp-pod
+  labels:
+    app: myapp
+spec:
+  containers:
+  - name: nginx
+    image: nginx:1.7.9
+EOF
+```
+
+Define a dev variant overlaying base:
+<!-- @makeDev @test -->
+```
+DEV=$DEMO_HOME/dev
+mkdir $DEV
+
+cat <<EOF >$DEV/kustomization.yaml
+bases:
+- ./../base
+namePrefix: dev-
+EOF
+```
+
+Define a staging variant overlaying base:
+<!-- @makeStaging @test -->
+```
+STAG=$DEMO_HOME/staging
+mkdir $STAG
+
+cat <<EOF >$STAG/kustomization.yaml
+bases:
+- ./../base
+namePrefix: stag-
+EOF
+```
+
+Define a production variant overlaying base:
+<!-- @makeProd @test -->
+```
+PROD=$DEMO_HOME/production
+mkdir $PROD
+
+cat <<EOF >$PROD/kustomization.yaml
+bases:
+- ./../base
+namePrefix: prod-
+EOF
+```
+
+Then define a _Kustomization_ composing three variants together:
+<!-- @makeTopLayer @test -->
+```
+cat <<EOF >$DEMO_HOME/kustomization.yaml
+bases:
+- ./dev
+- ./staging
+- ./production
+
+namePrefix: cluster-a-
+EOF
+```
+
+Now the workspace has following directories
+> ```
+> .
+> ├── base
+> │   ├── kustomization.yaml
+> │   └── pod.yaml
+> ├── dev
+> │   └── kustomization.yaml
+> ├── kustomization.yaml
+> ├── production
+> │   └── kustomization.yaml
+> └── staging
+>     └── kustomization.yaml
+> ```
+
+Confirm that the `kustomize build` output contains three pod objects from dev, staing and production variants.
+
+<!-- @confirmVariants @test -->
+```
+test 1 == \
+  $(kustomize build $DEMO_HOME | grep cluster-a-dev-myapp-pod | wc -l); \
+  echo $?
+  
+test 1 == \
+  $(kustomize build $DEMO_HOME | grep cluster-a-stag-myapp-pod | wc -l); \
+  echo $?
+  
+test 1 == \
+  $(kustomize build $DEMO_HOME | grep cluster-a-prod-myapp-pod | wc -l); \
+  echo $?    
+```
+

examples/multibases/base/kustomization.yaml

@@ -0,0 +1,2 @@
+resources:
+- pod.yaml

examples/multibases/base/pod.yaml

@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: myapp-pod
+  labels:
+    app: myapp
+spec:
+  containers:
+  - name: nginx
+    image: nginx:1.7.9

examples/multibases/dev/kustomization.yaml

@@ -0,0 +1,4 @@
+bases:
+- ./../base
+
+namePrefix: dev-

examples/multibases/kustomization.yaml

@@ -0,0 +1,6 @@
+bases:
+- ./dev
+- ./staging
+- ./production
+
+namePrefix: cluster-a-

examples/multibases/production/kustomization.yaml

@@ -0,0 +1,4 @@
+bases:
+- ./../base
+
+namePrefix: prod-

examples/multibases/staging/kustomization.yaml

@@ -0,0 +1,4 @@
+bases:
+- ./../base
+
+namePrefix: staging-

pkg/app/application.go

@@ -292,7 +292,7 @@ func (a *Application) resolveRefVars(m resmap.ResMap) (map[string]string, error)
 	}
 	for _, v := range vars {
 		id := resource.NewResId(v.ObjRef.GroupVersionKind(), v.ObjRef.Name)
-		if r, found := m[id]; found {
+		if r, found := m.DemandOneMatchForId(id); found {
 			s, err := r.GetFieldValue(v.FieldRef.FieldPath)
 			if err != nil {
 				return nil, fmt.Errorf("failed to resolve referred var: %+v", v)

pkg/app/application_test.go

@@ -91,7 +91,7 @@ var svc = schema.GroupVersionKind{Version: "v1", Kind: "Service"}
 
 func TestResources1(t *testing.T) {
 	expected := resmap.ResMap{
-		resource.NewResId(deploy, "dply1"): resource.NewResourceFromMap(
+		resource.NewResIdWithPrefix(deploy, "dply1", "foo-"): resource.NewResourceFromMap(
 			map[string]interface{}{
 				"apiVersion": "apps/v1",
 				"kind":       "Deployment",
@@ -123,7 +123,7 @@ func TestResources1(t *testing.T) {
 					},
 				},
 			}),
-		resource.NewResId(cmap, "literalConfigMap"): resource.NewResourceFromMap(
+		resource.NewResIdWithPrefix(cmap, "literalConfigMap", "foo-"): resource.NewResourceFromMap(
 			map[string]interface{}{
 				"apiVersion": "v1",
 				"kind":       "ConfigMap",
@@ -143,7 +143,7 @@ func TestResources1(t *testing.T) {
 					"DB_PASSWORD": "somepw",
 				},
 			}).SetBehavior(resource.BehaviorCreate),
-		resource.NewResId(secret, "secret"): resource.NewResourceFromMap(
+		resource.NewResIdWithPrefix(secret, "secret", "foo-"): resource.NewResourceFromMap(
 			map[string]interface{}{
 				"apiVersion": "v1",
 				"kind":       "Secret",
@@ -164,7 +164,7 @@ func TestResources1(t *testing.T) {
 					"DB_PASSWORD": base64.StdEncoding.EncodeToString([]byte("somepw")),
 				},
 			}).SetBehavior(resource.BehaviorCreate),
-		resource.NewResId(ns, "ns1"): resource.NewResourceFromMap(
+		resource.NewResIdWithPrefix(ns, "ns1", "foo-"): resource.NewResourceFromMap(
 			map[string]interface{}{
 				"apiVersion": "v1",
 				"kind":       "Namespace",
@@ -289,7 +289,7 @@ func makeLoader2(t *testing.T) loader.Loader {
 // perhaps it's not worth supporting the command.
 func TestRawResources2(t *testing.T) {
 	expected := resmap.ResMap{
-		resource.NewResId(deploy, "dply1"): resource.NewResourceFromMap(
+		resource.NewResIdWithPrefix(deploy, "dply1", "foo-"): resource.NewResourceFromMap(
 			map[string]interface{}{
 				"apiVersion": "apps/v1",
 				"kind":       "Deployment",

pkg/diff/rundiff.go

@@ -56,7 +56,7 @@ func writeYamlToNewDir(in resmap.ResMap, prefix string) (*directory, error) {
 	}
 
 	for id, obj := range in {
-		f, err := dir.newFile(id.String())
+		f, err := dir.newFile(id.GvknString())
 		if err != nil {
 			return nil, err
 		}

pkg/resmap/resmap.go

@@ -37,6 +37,26 @@ import (
 // ResMap is a map from ResId to Resource.
 type ResMap map[resource.ResId]*resource.Resource
 
+// FindByGVKN find the matched ResIds by Group/Version/Kind and Name
+func (m ResMap) FindByGVKN(inputId resource.ResId) []resource.ResId {
+	var result []resource.ResId
+	for id := range m {
+		if id.GvknEquals(inputId) {
+			result = append(result, id)
+		}
+	}
+	return result
+}
+
+// DemandOneMatchForId find the matched resource by Group/Version/Kind and Name
+func (m ResMap) DemandOneMatchForId(inputId resource.ResId) (*resource.Resource, bool) {
+	result := m.FindByGVKN(inputId)
+	if len(result) == 1 {
+		return m[result[0]], true
+	}
+	return nil, false
+}
+
 // EncodeAsYaml encodes a ResMap to YAML; encoded objects separated by `---`.
 func (m ResMap) EncodeAsYaml() ([]byte, error) {
 	var ids []resource.ResId
@@ -217,10 +237,12 @@ func MergeWithoutOverride(maps ...ResMap) (ResMap, error) {
 // must be BehaviorMerge or BehaviorReplace.  If X is not in the map, then it's
 // behavior cannot be merge or replace.
 func MergeWithOverride(maps ...ResMap) (ResMap, error) {
-	result := ResMap{}
-	for _, m := range maps {
+	result := maps[0]
+	for _, m := range maps[1:] {
 		for id, r := range m {
-			if _, found := result[id]; found {
+			matchedId := result.FindByGVKN(id)
+			if len(matchedId) == 1 {
+				id = matchedId[0]
 				switch r.Behavior() {
 				case resource.BehaviorReplace:
 					glog.V(4).Infof("Replace %v with %v", result[id].Object, r.Object)
@@ -236,13 +258,15 @@ func MergeWithOverride(maps ...ResMap) (ResMap, error) {
 				default:
 					return nil, fmt.Errorf("id %#v exists; must merge or replace", id)
 				}
-			} else {
+			} else if len(matchedId) == 0 {
 				switch r.Behavior() {
 				case resource.BehaviorMerge, resource.BehaviorReplace:
 					return nil, fmt.Errorf("id %#v does not exist; cannot merge or replace", id)
 				default:
 					result[id] = r
 				}
+			} else {
+				return nil, fmt.Errorf("Merge conflict, found multiple objects %v the Resmap %v can merge into", matchedId, id)
 			}
 		}
 	}

pkg/resource/resid.go

@@ -28,18 +28,42 @@ type ResId struct {
 	gvk schema.GroupVersionKind
 	// original name of the resource before transformation.
 	name string
+	// namePrefix of the resource
+	// an untransformed resource has no prefix, fully transformed resource has an arbitrary number of prefixes
+	// concatenated together.
+	prefix string
+}
+
+// NewResIdWithPrefix creates new resource identifier with a prefix
+func NewResIdWithPrefix(g schema.GroupVersionKind, n, p string) ResId {
+	return ResId{gvk: g, name: n, prefix: p}
 }
 
 // NewResId creates new resource identifier
 func NewResId(g schema.GroupVersionKind, n string) ResId {
-	return ResId{gvk: g, name: n}
+	return NewResIdWithPrefix(g, n, "")
 }
 
+// String of ResId based on GVK, name and prefix
 func (n ResId) String() string {
+	fields := []string{n.gvk.Group, n.gvk.Version, n.gvk.Kind, n.prefix, n.name}
+	return strings.Join(fields, "_") + ".yaml"
+}
+
+// GvknString of ResId based on GVK and name
+func (n ResId) GvknString() string {
 	if n.gvk.Group == "" {
 		return strings.Join([]string{n.gvk.Version, n.gvk.Kind, n.name}, "_") + ".yaml"
 	}
 	return strings.Join([]string{n.gvk.Group, n.gvk.Version, n.gvk.Kind, n.name}, "_") + ".yaml"
+
+}
+
+// GvknEquals return if two ResId have the same Group/Version/Kind and name
+// The comparison excludes prefix
+func (n ResId) GvknEquals(id ResId) bool {
+	return n.gvk.Group == id.gvk.Group && n.gvk.Version == id.gvk.Version &&
+		n.gvk.Kind == id.gvk.Kind && n.name == id.name
 }
 
 // Gvk returns Group/Version/Kind of the resource.
@@ -51,3 +75,13 @@ func (n ResId) Gvk() schema.GroupVersionKind {
 func (n ResId) Name() string {
 	return n.name
 }
+
+// Prefix returns name prefix.
+func (n ResId) Prefix() string {
+	return n.prefix
+}
+
+// CopyWithNewPrefix make a new copy from current ResId and append a new prefix
+func (n ResId) CopyWithNewPrefix(p string) ResId {
+	return ResId{gvk: n.gvk, name: n.name, prefix: p + n.prefix}
+}

pkg/transformers/patch.go

@@ -55,10 +55,15 @@ func (pt *patchTransformer) Transform(baseResourceMap resmap.ResMap) error {
 	for _, patch := range patches {
 		// Merge patches with base resource.
 		id := patch.Id()
-		base, found := baseResourceMap[id]
-		if !found {
+		matchedIds := baseResourceMap.FindByGVKN(id)
+		if len(matchedIds) == 0 {
 			return fmt.Errorf("failed to find an object with %#v to apply the patch", id.Gvk())
 		}
+		if len(matchedIds) > 1 {
+			return fmt.Errorf("Found multiple objects %#v that the patch %#v can apply", matchedIds, id)
+		}
+		id = matchedIds[0]
+		base := baseResourceMap[id]
 		merged := map[string]interface{}{}
 		versionedObj, err := scheme.Scheme.New(id.Gvk())
 		baseName := base.GetName()

pkg/transformers/prefixname.go

@@ -69,13 +69,17 @@ func (o *namePrefixTransformer) Transform(m resmap.ResMap) error {
 	mf := resmap.ResMap{}
 
 	for id := range m {
-		mf[id] = m[id]
+		found := false
 		for _, path := range o.skipPathConfigs {
 			if selectByGVK(id.Gvk(), path.GroupVersionKind) {
-				delete(mf, id)
+				found = true
 				break
 			}
 		}
+		if !found {
+			mf[id] = m[id]
+			delete(m, id)
+		}
 	}
 
 	for id := range mf {
@@ -88,6 +92,8 @@ func (o *namePrefixTransformer) Transform(m resmap.ResMap) error {
 			if err != nil {
 				return err
 			}
+			newId := id.CopyWithNewPrefix(o.prefix)
+			m[newId] = mf[id]
 		}
 	}
 	return nil

pkg/transformers/prefixname_test.go

@@ -53,7 +53,7 @@ func TestPrefixNameRun(t *testing.T) {
 			}),
 	}
 	expected := resmap.ResMap{
-		resource.NewResId(cmap, "cm1"): resource.NewResourceFromMap(
+		resource.NewResIdWithPrefix(cmap, "cm1", "someprefix-"): resource.NewResourceFromMap(
 			map[string]interface{}{
 				"apiVersion": "v1",
 				"kind":       "ConfigMap",
@@ -61,7 +61,7 @@ func TestPrefixNameRun(t *testing.T) {
 					"name": "someprefix-cm1",
 				},
 			}),
-		resource.NewResId(cmap, "cm2"): resource.NewResourceFromMap(
+		resource.NewResIdWithPrefix(cmap, "cm2", "someprefix-"): resource.NewResourceFromMap(
 			map[string]interface{}{
 				"apiVersion": "v1",
 				"kind":       "ConfigMap",