From 77cf6d6b888f93585406ebe6b2e51f4f691b31a1 Mon Sep 17 00:00:00 2001 From: koba1t Date: Thu, 20 Nov 2025 01:40:27 +0900 Subject: [PATCH] CI: parallelizing tests --- .github/workflows/go.yml | 101 ++++++++++++++++++++++----------------- Makefile | 8 +++- hack/for-each-module.sh | 12 ++++- 3 files changed, 74 insertions(+), 47 deletions(-) diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 58b7cab50..2b8c4471f 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -2,14 +2,15 @@ name: Go on: push: - branches: [ master ] + branches: [master] pull_request: - branches: [ master ] + branches: [master] permissions: contents: read jobs: + ## TODO: conditional-changes checker is not working conditional-changes: runs-on: ubuntu-latest permissions: @@ -27,7 +28,7 @@ jobs: check-modules: name: check-synced-go-modules - needs: conditional-changes + # needs: conditional-changes # if: needs.conditional-changes.outputs.doc == 'false' runs-on: [ubuntu-latest] steps: @@ -39,6 +40,9 @@ jobs: uses: actions/setup-go@v6 with: go-version-file: go.work + cache: true + cache-dependency-path: | + **/go.sum id: go - name: sync go modules run: make workspace-sync @@ -47,7 +51,7 @@ jobs: lint: name: Lint - needs: conditional-changes + # needs: conditional-changes # if: needs.conditional-changes.outputs.doc == 'false' runs-on: [ubuntu-latest] steps: @@ -59,15 +63,19 @@ jobs: uses: actions/setup-go@v6 with: go-version-file: go.work + cache: true + cache-dependency-path: | + **/go.sum id: go - name: Lint run: make lint - name: Verify boilerplate run: make check-license - test-linux: + ## Test all modules without plugins and released modules + test-non-released-modules: name: Test Linux - needs: conditional-changes + # needs: conditional-changes # if: needs.conditional-changes.outputs.doc == 'false' runs-on: [ubuntu-latest] steps: @@ -77,17 +85,46 @@ jobs: uses: actions/setup-go@v6 with: go-version-file: go.work + cache: true + cache-dependency-path: | + **/go.sum id: go - - name: Test all modules - run: make test-unit-non-plugin + - name: Test all modules without plugins and released modules + run: make test-unit-non-plugin-and-non-released env: KUSTOMIZE_DOCKER_E2E: true - test-macos: - name: Test MacOS - needs: conditional-changes + test-modules: + name: Test ${{ matrix.os }} - ${{ matrix.module }} + # needs: conditional-changes # if: needs.conditional-changes.outputs.doc == 'false' - runs-on: [macos-latest] + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + module: + - kyaml + - cmd/config + - api + - kustomize + include: + - module: kyaml + test-cmd: go test -race -v -cover ./... + - module: cmd/config + test-cmd: go test -v -cover ./... + - module: api + test-cmd: go test -v -cover ./... -ldflags "-X sigs.k8s.io/kustomize/api/provenance.buildDate=2023-01-31T23:38:41Z -X sigs.k8s.io/kustomize/api/provenance.version=(test)" + - module: kustomize + test-cmd: go test -v -cover ./... + - os: ubuntu-latest + docker-e2e: true + - os: macos-latest + docker-e2e: false + - os: windows-latest + docker-e2e: false + env: + KUSTOMIZE_DOCKER_E2E: ${{ matrix.docker-e2e }} steps: - name: Check out code into the Go module directory uses: actions/checkout@v6 @@ -95,36 +132,12 @@ jobs: uses: actions/setup-go@v6 with: go-version-file: go.work + cache: true + cache-dependency-path: | + **/go.sum id: go - - name: Test all modules - run: make test-unit-non-plugin - env: - KUSTOMIZE_DOCKER_E2E: false # docker not installed on mac - - test-windows: - name: Test Windows - needs: conditional-changes - # if: needs.conditional-changes.outputs.doc == 'false' - runs-on: [windows-latest] - steps: - - name: Check out code into the Go module directory - uses: actions/checkout@v6 - - name: Set up Go 1.x - uses: actions/setup-go@v6 - with: - go-version-file: go.work - id: go - - name: Test kyaml - run: go test -cover ./... - working-directory: ./kyaml - - name: Test cmd/config - run: go test -cover ./... - working-directory: ./cmd/config - env: - KUSTOMIZE_DOCKER_E2E: false # docker on windows not working well yet - - # TODO (#4001): replace specific modules above with this once Windows tests are passing. - #- name: Test all modules - # run: make test-unit-non-plugin - # env: - # KUSTOMIZE_DOCKER_E2E: false # docker on windows not working well yet + - name: Test ${{ matrix.module }} + run: ${{ matrix.test-cmd }} + # TODO (#4001): replace specific modules above with this once Windows tests are passing. + if: ${{ !(matrix.os == 'windows-latest' && (matrix.module == 'api' || matrix.module == 'kustomize')) }} + working-directory: ./${{ matrix.module }} diff --git a/Makefile b/Makefile index 6030267ab..2b9dfd3c1 100644 --- a/Makefile +++ b/Makefile @@ -136,11 +136,15 @@ test-unit-all: \ test-unit-non-plugin \ test-unit-kustomize-plugins -# This target is used by our Github Actions CI to run unit tests for all non-plugin modules in multiple GOOS environments. .PHONY: test-unit-non-plugin test-unit-non-plugin: ./hack/for-each-module.sh "make test" "./plugin/*" 20 +# This target is used by our Github Actions CI to run unit tests for all non-plugin and non-released modules in multiple GOOS environments. +.PHONY: test-unit-non-plugin-and-non-released +test-unit-non-plugin-and-non-released: + ./hack/for-each-module.sh "make test" "./plugin/*|./kyaml/go.mod|./cmd/config/go.mod|./api/go.mod|./kustomize/go.mod" 16 + .PHONY: build-non-plugin-all build-non-plugin-all: ./hack/for-each-module.sh "make build" "./plugin/*" 20 @@ -183,7 +187,7 @@ test-examples-kustomize-against-latest-release: $(MYGOBIN)/mdrip workspace-sync: go work sync ./hack/doGoMod.sh tidy - + # --- Cleanup targets --- .PHONY: clean clean: clean-kustomize-external-go-plugin uninstall-tools diff --git a/hack/for-each-module.sh b/hack/for-each-module.sh index 5981a68e2..3f15289ff 100755 --- a/hack/for-each-module.sh +++ b/hack/for-each-module.sh @@ -22,8 +22,18 @@ seen=() KUSTOMIZE_ROOT=$(pwd) export KUSTOMIZE_ROOT +# Build find command with multiple -not -path options +find_cmd="find . -name go.mod -not -path \"./site/*\"" +if [[ -n "$skip_pattern" ]]; then + # Split skip_pattern by | and add -not -path for each + IFS='|' read -ra PATTERNS <<< "$skip_pattern" + for pattern in "${PATTERNS[@]}"; do + find_cmd+=" -not -path \"$pattern\"" + done +fi + # verify all modules pass validation -for i in $(find . -name go.mod -not -path "./site/*" -not -path "$skip_pattern"); do +for i in $(eval "$find_cmd"); do pushd . cd $(dirname "$i");