From 4cefb62d419bee04ee08f9595288823b249d08f4 Mon Sep 17 00:00:00 2001 From: Sam Wronski Date: Wed, 14 Oct 2020 15:34:52 -0700 Subject: [PATCH 1/2] Add multi-module check to `prow-presubmit-check` --- Makefile | 13 +++++++++++++ travis/module-span/multi-module-span.go | 23 +++++++++++++++++++---- 2 files changed, 32 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index e036bb925..2300cd126 100644 --- a/Makefile +++ b/Makefile @@ -6,6 +6,7 @@ MYGOBIN := $(shell go env GOPATH)/bin SHELL := /bin/bash export PATH := $(MYGOBIN):$(PATH) +MODULES := "cmd/config" "api/" "kustomize/" "kyaml/" .PHONY: all all: verify-kustomize @@ -24,6 +25,7 @@ verify-kustomize: \ # https://github.com/kubernetes/test-infra/tree/master/config/jobs/kubernetes-sigs/kustomize .PHONY: prow-presubmit-check prow-presubmit-check: \ + test-multi-module \ lint-kustomize \ test-unit-kustomize-all \ test-examples-kustomize-against-HEAD \ @@ -220,6 +222,17 @@ test-unit-cmd-all: test-go-mod: ./travis/check-go-mod.sh +# Environment variables are defined at +# https://github.com/kubernetes/test-infra/blob/master/prow/jobs.md#job-environment-variables +.PHONY: test-multi-module +test-multi-module: + go install github.com/google/go-github/github + go run ./travis/module-span/multi-module-span.go \ + -owner=$(REPO_OWNER) \ + -repo=$(REPO_NAME) \ + -pr=$(PR_NUM) \ + $(MODULES) + .PHONY: test-examples-e2e-kustomize: $(MYGOBIN)/mdrip $(MYGOBIN)/kind ( \ diff --git a/travis/module-span/multi-module-span.go b/travis/module-span/multi-module-span.go index 1bb92c436..95a7d20a6 100644 --- a/travis/module-span/multi-module-span.go +++ b/travis/module-span/multi-module-span.go @@ -57,15 +57,30 @@ func main() { // ListAllPullRequestFiles retrieves as many files as possible for the // target pull request. // -// NOTE: GitHub API limits ListFiles to a maximum of 3000 files. Very large +// Note: GitHub API limits ListFiles to a maximum of 3000 files. Very large // changes which exceed this limit may pass this check even if they // do contain spanning changes. // see: https://developer.github.com/v3/pulls/#list-pull-requests-files func ListAllPullRequestFiles(client *github.Client, owner *string, pullrequest *int, repo *string) ([]*github.CommitFile, error) { + // foundFiles across all pages from github api + var foundFiles []*github.CommitFile // GitHub returns the first 30 files by default, increase this value - options := &github.ListOptions{PerPage: 3000} - files, _, err := client.PullRequests.ListFiles(context.Background(), *owner, *repo, *pullrequest, options) - return files, err + // Note: Page 1 is the first page of results. Page 0 is an end of list mark. + // Github only returns (max) 100 results per page and PR's may exceed this + // so loop until all pages have been enumerated. + options := &github.ListOptions{PerPage: 100, Page: 1} + for options.Page != 0 { + files, response, err := client.PullRequests.ListFiles(context.Background(), *owner, *repo, *pullrequest, options) + + // If an error has occurred while querying api exit early, report error + if err != nil { + return nil, err + } + foundFiles = append(foundFiles, files...) + // setup next page to continue loop + options = &github.ListOptions{PerPage: 100, Page: response.NextPage} + } + return foundFiles, nil } // CountModifiedRestrictedDirectories Accepts a map of paths and the number of From 6c63bb2727da2f85c648b44f64b21adb44efaba4 Mon Sep 17 00:00:00 2001 From: Sam Wronski Date: Thu, 15 Oct 2020 10:29:38 -0700 Subject: [PATCH 2/2] Revert Makefile changes --- Makefile | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/Makefile b/Makefile index 2300cd126..e036bb925 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,6 @@ MYGOBIN := $(shell go env GOPATH)/bin SHELL := /bin/bash export PATH := $(MYGOBIN):$(PATH) -MODULES := "cmd/config" "api/" "kustomize/" "kyaml/" .PHONY: all all: verify-kustomize @@ -25,7 +24,6 @@ verify-kustomize: \ # https://github.com/kubernetes/test-infra/tree/master/config/jobs/kubernetes-sigs/kustomize .PHONY: prow-presubmit-check prow-presubmit-check: \ - test-multi-module \ lint-kustomize \ test-unit-kustomize-all \ test-examples-kustomize-against-HEAD \ @@ -222,17 +220,6 @@ test-unit-cmd-all: test-go-mod: ./travis/check-go-mod.sh -# Environment variables are defined at -# https://github.com/kubernetes/test-infra/blob/master/prow/jobs.md#job-environment-variables -.PHONY: test-multi-module -test-multi-module: - go install github.com/google/go-github/github - go run ./travis/module-span/multi-module-span.go \ - -owner=$(REPO_OWNER) \ - -repo=$(REPO_NAME) \ - -pr=$(PR_NUM) \ - $(MODULES) - .PHONY: test-examples-e2e-kustomize: $(MYGOBIN)/mdrip $(MYGOBIN)/kind ( \