From 6d324d70c4c50e91a3c0e85fb6e515c56e020642 Mon Sep 17 00:00:00 2001
From: Donny Xia <donnyxia@google.com>
Date: Wed, 1 Jul 2020 12:46:16 -0700
Subject: [PATCH] Add benchmark for containerized KRM function in kustomize

---
 hack/krmFunctionBenchmark/benchmark.sh        | 67 +++++++++++++++++++
 hack/krmFunctionBenchmark/cleanup.sh          |  8 +++
 .../example_tshirt/containerfn/data.yaml      | 20 ++++++
 .../containerfn/kustomization.yaml            |  4 ++
 .../example_tshirt/containerfn/transf.yaml    |  8 +++
 .../example_tshirt/execfn/Dockerfile          | 16 +++++
 .../example_tshirt/execfn/build.sh            |  8 +++
 .../example_tshirt/execfn/data.yaml           | 20 ++++++
 .../example_tshirt/execfn/kustomization.yaml  |  4 ++
 .../example_tshirt/execfn/transf.yaml         |  8 +++
 .../containerfn/containerfn.yaml              | 11 +++
 .../label_namespace/containerfn/data1.yaml    |  4 ++
 .../label_namespace/containerfn/data2.yaml    |  4 ++
 .../containerfn/kustomization.yaml            |  5 ++
 .../label_namespace/execfn/Dockerfile         | 15 +++++
 .../label_namespace/execfn/build.sh           |  8 +++
 .../label_namespace/execfn/data1.yaml         |  4 ++
 .../label_namespace/execfn/data2.yaml         |  4 ++
 .../label_namespace/execfn/execfn.yaml        | 11 +++
 .../label_namespace/execfn/fn.sh              |  3 +
 .../label_namespace/execfn/kustomization.yaml |  5 ++
 hack/krmFunctionBenchmark/readme.md           | 19 ++++++
 22 files changed, 256 insertions(+)
 create mode 100755 hack/krmFunctionBenchmark/benchmark.sh
 create mode 100755 hack/krmFunctionBenchmark/cleanup.sh
 create mode 100644 hack/krmFunctionBenchmark/example_tshirt/containerfn/data.yaml
 create mode 100644 hack/krmFunctionBenchmark/example_tshirt/containerfn/kustomization.yaml
 create mode 100644 hack/krmFunctionBenchmark/example_tshirt/containerfn/transf.yaml
 create mode 100644 hack/krmFunctionBenchmark/example_tshirt/execfn/Dockerfile
 create mode 100755 hack/krmFunctionBenchmark/example_tshirt/execfn/build.sh
 create mode 100644 hack/krmFunctionBenchmark/example_tshirt/execfn/data.yaml
 create mode 100644 hack/krmFunctionBenchmark/example_tshirt/execfn/kustomization.yaml
 create mode 100644 hack/krmFunctionBenchmark/example_tshirt/execfn/transf.yaml
 create mode 100644 hack/krmFunctionBenchmark/label_namespace/containerfn/containerfn.yaml
 create mode 100644 hack/krmFunctionBenchmark/label_namespace/containerfn/data1.yaml
 create mode 100644 hack/krmFunctionBenchmark/label_namespace/containerfn/data2.yaml
 create mode 100644 hack/krmFunctionBenchmark/label_namespace/containerfn/kustomization.yaml
 create mode 100644 hack/krmFunctionBenchmark/label_namespace/execfn/Dockerfile
 create mode 100755 hack/krmFunctionBenchmark/label_namespace/execfn/build.sh
 create mode 100644 hack/krmFunctionBenchmark/label_namespace/execfn/data1.yaml
 create mode 100644 hack/krmFunctionBenchmark/label_namespace/execfn/data2.yaml
 create mode 100644 hack/krmFunctionBenchmark/label_namespace/execfn/execfn.yaml
 create mode 100755 hack/krmFunctionBenchmark/label_namespace/execfn/fn.sh
 create mode 100644 hack/krmFunctionBenchmark/label_namespace/execfn/kustomization.yaml
 create mode 100644 hack/krmFunctionBenchmark/readme.md

diff --git a/hack/krmFunctionBenchmark/benchmark.sh b/hack/krmFunctionBenchmark/benchmark.sh
new file mode 100755
index 000000000..0e59440b7
--- /dev/null
+++ b/hack/krmFunctionBenchmark/benchmark.sh
@@ -0,0 +1,67 @@
+#! /bin/bash
+
+set -e
+
+KUSTOMIZE_EXEC=kustomize
+KUSTOMIZE_FLAGS="build --enable_alpha_plugins --enable-exec"
+
+function build_label_namespace_exec {
+    cd label_namespace/execfn
+    . build.sh
+    cd -
+}
+
+function build_tshirt_exec {
+    cd example_tshirt/execfn
+    . build.sh
+    cd -
+}
+
+function build_exec {
+    echo "Building exec functions..."
+    build_tshirt_exec
+    build_label_namespace_exec
+    echo "Done. Start running benchmark."
+}
+
+function run_fn {
+    local loop=$1
+    local type=$2
+    local dir=$3
+    echo -e "=== Running ${type} ${loop} times ==="
+    cd $dir
+    local begin_time=$(date +%s%N)
+    for ((i = 0; i < $loop; i++))
+    do
+        $KUSTOMIZE_EXEC $KUSTOMIZE_FLAGS > /dev/null
+        echo -en "\r$i/$loop"
+    done
+    local end_time=$(date +%s%N)
+    local time_diff=$(($end_time - $begin_time))
+    local time_diff_s=$(echo "${time_diff} / 1000 / 1000 / 1000" | bc -l)
+    echo -e "\n=== Time used: ==="
+    echo "${time_diff_s}s"
+    cd -
+}
+
+function run_label_namespace_benchmark {
+    local loop=$1
+    run_fn $loop "Label Namespace Transformer Exec Function" "label_namespace/execfn"
+    run_fn $loop "Label Namespace Transformer Container Function" "label_namespace/containerfn"
+}
+
+function run_tshirt_benchmark {
+    local loop=$1
+    run_fn $loop "T-shirt Example Exec Function" "example_tshirt/execfn"
+    run_fn $loop "T-shirt Example Container Function" "example_tshirt/containerfn"
+}
+
+loops=(10 100 1000)
+
+build_exec
+
+for l in "${loops[@]}"
+do
+    run_label_namespace_benchmark $l
+    run_tshirt_benchmark $l
+done
\ No newline at end of file
diff --git a/hack/krmFunctionBenchmark/cleanup.sh b/hack/krmFunctionBenchmark/cleanup.sh
new file mode 100755
index 000000000..40d06fd31
--- /dev/null
+++ b/hack/krmFunctionBenchmark/cleanup.sh
@@ -0,0 +1,8 @@
+#! /bin/bash
+
+sudo rm -rf example_tshirt/execfn/tshirt label_namespace/execfn/dist label_namespace/execfn/node_modules
+
+if [ "$1" == "--image" ]; then
+    docker image rm label_namespace_build:latest
+    docker image rm tshirt_example_build:latest
+fi
\ No newline at end of file
diff --git a/hack/krmFunctionBenchmark/example_tshirt/containerfn/data.yaml b/hack/krmFunctionBenchmark/example_tshirt/containerfn/data.yaml
new file mode 100644
index 000000000..a3ac98f20
--- /dev/null
+++ b/hack/krmFunctionBenchmark/example_tshirt/containerfn/data.yaml
@@ -0,0 +1,20 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: nginx
+  labels:
+    app: nginx
+  annotations:
+    tshirt-size: small # this injects the resource reservations
+spec:
+  selector:
+    matchLabels:
+      app: nginx
+  template:
+    metadata:
+      labels:
+        app: nginx
+    spec:
+      containers:
+      - name: nginx
+        image: nginx
\ No newline at end of file
diff --git a/hack/krmFunctionBenchmark/example_tshirt/containerfn/kustomization.yaml b/hack/krmFunctionBenchmark/example_tshirt/containerfn/kustomization.yaml
new file mode 100644
index 000000000..a509be0ae
--- /dev/null
+++ b/hack/krmFunctionBenchmark/example_tshirt/containerfn/kustomization.yaml
@@ -0,0 +1,4 @@
+resources:
+- data.yaml
+transformers:
+- transf.yaml
\ No newline at end of file
diff --git a/hack/krmFunctionBenchmark/example_tshirt/containerfn/transf.yaml b/hack/krmFunctionBenchmark/example_tshirt/containerfn/transf.yaml
new file mode 100644
index 000000000..a144d7b09
--- /dev/null
+++ b/hack/krmFunctionBenchmark/example_tshirt/containerfn/transf.yaml
@@ -0,0 +1,8 @@
+apiVersion: examples.config.kubernetes.io/v1beta1
+kind: tshirt
+metadata:
+  name: tshirt
+  annotations:
+    config.kubernetes.io/function: |-
+      container:
+        image: gcr.io/kustomize-functions/example-tshirt:v0.2.0
\ No newline at end of file
diff --git a/hack/krmFunctionBenchmark/example_tshirt/execfn/Dockerfile b/hack/krmFunctionBenchmark/example_tshirt/execfn/Dockerfile
new file mode 100644
index 000000000..1887cbfc9
--- /dev/null
+++ b/hack/krmFunctionBenchmark/example_tshirt/execfn/Dockerfile
@@ -0,0 +1,16 @@
+FROM alpine:latest
+
+ENV BUILD_HOME=/usr/local/build
+RUN apk update && apk add --no-cache git go
+
+RUN mkdir -p $BUILD_HOME
+
+WORKDIR $BUILD_HOME
+
+RUN git clone https://github.com/kubernetes-sigs/kustomize.git .
+RUN git checkout tags/kustomize/v3.6.1
+WORKDIR $BUILD_HOME/functions/examples/injection-tshirt-sizes/image/
+
+ENV CGO_ENABLED=0
+RUN go mod download
+RUN go build -o tshirt .
\ No newline at end of file
diff --git a/hack/krmFunctionBenchmark/example_tshirt/execfn/build.sh b/hack/krmFunctionBenchmark/example_tshirt/execfn/build.sh
new file mode 100755
index 000000000..0070ab8a3
--- /dev/null
+++ b/hack/krmFunctionBenchmark/example_tshirt/execfn/build.sh
@@ -0,0 +1,8 @@
+#! /bin/bash
+
+IMAGE_LABEL="tshirt_example_build:latest"
+BUILD_HOME=/usr/local/build
+
+docker build -t $IMAGE_LABEL .
+
+docker run --rm -v $(pwd):/out $IMAGE_LABEL cp -r $BUILD_HOME/functions/examples/injection-tshirt-sizes/image/tshirt /out
\ No newline at end of file
diff --git a/hack/krmFunctionBenchmark/example_tshirt/execfn/data.yaml b/hack/krmFunctionBenchmark/example_tshirt/execfn/data.yaml
new file mode 100644
index 000000000..a3ac98f20
--- /dev/null
+++ b/hack/krmFunctionBenchmark/example_tshirt/execfn/data.yaml
@@ -0,0 +1,20 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: nginx
+  labels:
+    app: nginx
+  annotations:
+    tshirt-size: small # this injects the resource reservations
+spec:
+  selector:
+    matchLabels:
+      app: nginx
+  template:
+    metadata:
+      labels:
+        app: nginx
+    spec:
+      containers:
+      - name: nginx
+        image: nginx
\ No newline at end of file
diff --git a/hack/krmFunctionBenchmark/example_tshirt/execfn/kustomization.yaml b/hack/krmFunctionBenchmark/example_tshirt/execfn/kustomization.yaml
new file mode 100644
index 000000000..a509be0ae
--- /dev/null
+++ b/hack/krmFunctionBenchmark/example_tshirt/execfn/kustomization.yaml
@@ -0,0 +1,4 @@
+resources:
+- data.yaml
+transformers:
+- transf.yaml
\ No newline at end of file
diff --git a/hack/krmFunctionBenchmark/example_tshirt/execfn/transf.yaml b/hack/krmFunctionBenchmark/example_tshirt/execfn/transf.yaml
new file mode 100644
index 000000000..a619f3de5
--- /dev/null
+++ b/hack/krmFunctionBenchmark/example_tshirt/execfn/transf.yaml
@@ -0,0 +1,8 @@
+apiVersion: examples.config.kubernetes.io/v1beta1
+kind: tshirt
+metadata:
+  name: tshirt
+  annotations:
+    config.kubernetes.io/function: |-
+      exec:
+        path: ./tshirt
\ No newline at end of file
diff --git a/hack/krmFunctionBenchmark/label_namespace/containerfn/containerfn.yaml b/hack/krmFunctionBenchmark/label_namespace/containerfn/containerfn.yaml
new file mode 100644
index 000000000..e47703d98
--- /dev/null
+++ b/hack/krmFunctionBenchmark/label_namespace/containerfn/containerfn.yaml
@@ -0,0 +1,11 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: label_namespace
+  annotations:
+    config.kubernetes.io/function: |-
+      container:
+        image: gcr.io/kpt-functions/label-namespace@sha256:4f030738d6d25a207641ca517916431517578bd0eb8d98a8bde04e3bb9315dcd
+data:
+  label_name: my-ns-name
+  label_value: function-test
\ No newline at end of file
diff --git a/hack/krmFunctionBenchmark/label_namespace/containerfn/data1.yaml b/hack/krmFunctionBenchmark/label_namespace/containerfn/data1.yaml
new file mode 100644
index 000000000..97fc87977
--- /dev/null
+++ b/hack/krmFunctionBenchmark/label_namespace/containerfn/data1.yaml
@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: my-namespace
\ No newline at end of file
diff --git a/hack/krmFunctionBenchmark/label_namespace/containerfn/data2.yaml b/hack/krmFunctionBenchmark/label_namespace/containerfn/data2.yaml
new file mode 100644
index 000000000..167e67489
--- /dev/null
+++ b/hack/krmFunctionBenchmark/label_namespace/containerfn/data2.yaml
@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: another-namespace
\ No newline at end of file
diff --git a/hack/krmFunctionBenchmark/label_namespace/containerfn/kustomization.yaml b/hack/krmFunctionBenchmark/label_namespace/containerfn/kustomization.yaml
new file mode 100644
index 000000000..e06c3e97f
--- /dev/null
+++ b/hack/krmFunctionBenchmark/label_namespace/containerfn/kustomization.yaml
@@ -0,0 +1,5 @@
+resources:
+- data1.yaml
+- data2.yaml
+transformers:
+- containerfn.yaml
\ No newline at end of file
diff --git a/hack/krmFunctionBenchmark/label_namespace/execfn/Dockerfile b/hack/krmFunctionBenchmark/label_namespace/execfn/Dockerfile
new file mode 100644
index 000000000..6fd6b84d5
--- /dev/null
+++ b/hack/krmFunctionBenchmark/label_namespace/execfn/Dockerfile
@@ -0,0 +1,15 @@
+FROM alpine:latest
+
+ENV BUILD_HOME=/usr/local/build
+RUN apk update && apk add --no-cache git nodejs npm
+RUN npm install -g typescript
+
+RUN mkdir -p $BUILD_HOME
+
+WORKDIR $BUILD_HOME
+
+RUN git clone https://github.com/GoogleContainerTools/kpt-functions-sdk.git .
+RUN git checkout tags/release-kpt-functions-v0.14.2
+WORKDIR $BUILD_HOME/ts/hello-world/
+RUN npm install
+RUN npm run build
\ No newline at end of file
diff --git a/hack/krmFunctionBenchmark/label_namespace/execfn/build.sh b/hack/krmFunctionBenchmark/label_namespace/execfn/build.sh
new file mode 100755
index 000000000..49768780e
--- /dev/null
+++ b/hack/krmFunctionBenchmark/label_namespace/execfn/build.sh
@@ -0,0 +1,8 @@
+#! /bin/bash
+
+IMAGE_LABEL="label_namespace_build:latest"
+BUILD_HOME=/usr/local/build
+
+docker build -t $IMAGE_LABEL .
+
+docker run --rm -v $(pwd):/out $IMAGE_LABEL cp -r $BUILD_HOME/ts/hello-world/dist $BUILD_HOME/ts/hello-world/node_modules /out
\ No newline at end of file
diff --git a/hack/krmFunctionBenchmark/label_namespace/execfn/data1.yaml b/hack/krmFunctionBenchmark/label_namespace/execfn/data1.yaml
new file mode 100644
index 000000000..97fc87977
--- /dev/null
+++ b/hack/krmFunctionBenchmark/label_namespace/execfn/data1.yaml
@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: my-namespace
\ No newline at end of file
diff --git a/hack/krmFunctionBenchmark/label_namespace/execfn/data2.yaml b/hack/krmFunctionBenchmark/label_namespace/execfn/data2.yaml
new file mode 100644
index 000000000..167e67489
--- /dev/null
+++ b/hack/krmFunctionBenchmark/label_namespace/execfn/data2.yaml
@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: another-namespace
\ No newline at end of file
diff --git a/hack/krmFunctionBenchmark/label_namespace/execfn/execfn.yaml b/hack/krmFunctionBenchmark/label_namespace/execfn/execfn.yaml
new file mode 100644
index 000000000..c07242d4b
--- /dev/null
+++ b/hack/krmFunctionBenchmark/label_namespace/execfn/execfn.yaml
@@ -0,0 +1,11 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: label_namespace
+  annotations:
+    config.kubernetes.io/function: |-
+      exec:
+        path: ./fn.sh
+data:
+  label_name: my-ns-name
+  label_value: function-test
\ No newline at end of file
diff --git a/hack/krmFunctionBenchmark/label_namespace/execfn/fn.sh b/hack/krmFunctionBenchmark/label_namespace/execfn/fn.sh
new file mode 100755
index 000000000..d1e1d8ddd
--- /dev/null
+++ b/hack/krmFunctionBenchmark/label_namespace/execfn/fn.sh
@@ -0,0 +1,3 @@
+#! /bin/bash
+
+node ./dist/label_namespace_run.js
\ No newline at end of file
diff --git a/hack/krmFunctionBenchmark/label_namespace/execfn/kustomization.yaml b/hack/krmFunctionBenchmark/label_namespace/execfn/kustomization.yaml
new file mode 100644
index 000000000..6cb89b1df
--- /dev/null
+++ b/hack/krmFunctionBenchmark/label_namespace/execfn/kustomization.yaml
@@ -0,0 +1,5 @@
+resources:
+- data1.yaml
+- data2.yaml
+transformers:
+- execfn.yaml
\ No newline at end of file
diff --git a/hack/krmFunctionBenchmark/readme.md b/hack/krmFunctionBenchmark/readme.md
new file mode 100644
index 000000000..98cb33286
--- /dev/null
+++ b/hack/krmFunctionBenchmark/readme.md
@@ -0,0 +1,19 @@
+# Benchmark for KRM functions in Kustomize
+
+## Pre-request
+
+ - You have to use Kustomize version with KRM function supported.
+ - You need to have Docker running on your machine.
+
+## How to run
+
+```bash
+./benchmark.sh
+```
+
+The script will build the exec version of function via container and then run 10, 100 and 1000 times of exec version and container version and then print out the time used by both versions.
+
+```bash
+./cleanup.sh
+```
+Will remove the built exec version of the function. Add flag `--image` to remove the images that used to build exec function.
\ No newline at end of file