diff --git a/internal/crawl/ROADMAP.md b/internal/crawl/ROADMAP.md index 8fdcb0b81..eeec05451 100644 --- a/internal/crawl/ROADMAP.md +++ b/internal/crawl/ROADMAP.md @@ -82,13 +82,13 @@ following: "docID45", ... ], - } + }, { "identifier": "another:k8s:feature", "documents": [ ... ], - } + }, ... ] diff --git a/internal/crawl/crawler/crawler.go b/internal/crawl/crawler/crawler.go index 7a3b68b32..a5706e318 100644 --- a/internal/crawl/crawler/crawler.go +++ b/internal/crawl/crawler/crawler.go @@ -27,7 +27,7 @@ type Crawler interface { // Crawl returns when it is done processing. This method does not take // ownership of the channel. The channel is write only, and it // designates where the crawler should forward the documents. - Crawl(ctx context.Context, output chan<- CrawlerDocument) error + Crawl(ctx context.Context, output chan<- CrawledDocument) error // Get the document data given the FilePath, Repo, and Ref/Tag/Branch. FetchDocument(context.Context, *doc.Document) error @@ -37,21 +37,21 @@ type Crawler interface { Match(*doc.Document) bool } -type CrawlerDocument interface { +type CrawledDocument interface { ID() string GetDocument() *doc.Document GetResources() ([]*doc.Document, error) WasCached() bool } -type CrawlerSeed []*doc.Document +type CrawlSeed []*doc.Document -type IndexFunc func(CrawlerDocument, Crawler) error -type Converter func(*doc.Document) (CrawlerDocument, error) +type IndexFunc func(CrawledDocument, Crawler) error +type Converter func(*doc.Document) (CrawledDocument, error) // Cleaner, more efficient, and more extensible crawler implementation. // The seed must include the ids of each document in the index. -func CrawlFromSeed(ctx context.Context, seed CrawlerSeed, +func CrawlFromSeed(ctx context.Context, seed CrawlSeed, crawlers []Crawler, conv Converter, indx IndexFunc) { seen := make(map[string]struct{}) @@ -63,7 +63,7 @@ func CrawlFromSeed(ctx context.Context, seed CrawlerSeed, logger.Println("error: ", err) } - stack := make(CrawlerSeed, 0) + stack := make(CrawlSeed, 0) findMatch := func(d *doc.Document) Crawler { for _, crawl := range crawlers { @@ -75,7 +75,7 @@ func CrawlFromSeed(ctx context.Context, seed CrawlerSeed, return nil } - addBranches := func(cdoc CrawlerDocument, match Crawler) { + addBranches := func(cdoc CrawledDocument, match Crawler) { if _, ok := seen[cdoc.ID()]; ok { return } @@ -101,7 +101,7 @@ func CrawlFromSeed(ctx context.Context, seed CrawlerSeed, } } - doCrawl := func(docsPtr *CrawlerSeed) { + doCrawl := func(docsPtr *CrawlSeed) { for len(*docsPtr) > 0 { back := len(*docsPtr) - 1 next := (*docsPtr)[back] @@ -139,7 +139,7 @@ func CrawlFromSeed(ctx context.Context, seed CrawlerSeed, logger.Printf("crawling %d new documents found in the seed\n", len(stack)) doCrawl(&stack) - ch := make(chan CrawlerDocument, 1<<10) + ch := make(chan CrawledDocument, 1<<10) wg := sync.WaitGroup{} wg.Add(1) @@ -160,7 +160,7 @@ func CrawlFromSeed(ctx context.Context, seed CrawlerSeed, }() // Exploration through APIs. - errs := CrawlerRunner(ctx, ch, crawlers) + errs := CRunner(ctx, ch, crawlers) if errs != nil { for _, err := range errs { logIfErr(err) @@ -175,7 +175,7 @@ func CrawlFromSeed(ctx context.Context, seed CrawlerSeed, doCrawl(&stack) } -// CrawlerRunner is a blocking function and only returns once all of the +// CRunner is a blocking function and only returns once all of the // crawlers are finished with execution. // // This function uses the output channel to forward kustomization documents @@ -186,14 +186,14 @@ func CrawlFromSeed(ctx context.Context, seed CrawlerSeed, // index of the crawler that emitted the error. Although the errors themselves // can be nil, the array will always be exactly the size of the crawlers array. // -// Crawler Runner takes in a seed, which represents the documents stored in an +// CRunner takes in a seed, which represents the documents stored in an // index somewhere. The document data is not required to be populated. If there // are many documents, this is preferable. The order of iteration over the seed -// is not garanteed, but the CrawlerRunner does guarantee that every element +// is not garanteed, but the CRunner does guarantee that every element // from the seed will be processed before any other documents from the // crawlers. -func CrawlerRunner(ctx context.Context, - output chan<- CrawlerDocument, crawlers []Crawler) []error { +func CRunner(ctx context.Context, + output chan<- CrawledDocument, crawlers []Crawler) []error { errs := make([]error, len(crawlers)) wg := sync.WaitGroup{} @@ -201,12 +201,12 @@ func CrawlerRunner(ctx context.Context, for i, crawler := range crawlers { // Crawler implementations get their own channels to prevent a // crawler from closing the main output channel. - docs := make(chan CrawlerDocument) + docs := make(chan CrawledDocument) wg.Add(2) // Forward all of the documents from this crawler's channel to // the main output channel. - go func(docs <-chan CrawlerDocument) { + go func(docs <-chan CrawledDocument) { defer wg.Done() for doc := range docs { output <- doc @@ -215,7 +215,7 @@ func CrawlerRunner(ctx context.Context, // Run this crawler and capture its returned error. go func(idx int, crawler Crawler, - docs chan<- CrawlerDocument) { + docs chan<- CrawledDocument) { defer func() { wg.Done() diff --git a/internal/crawl/crawler/crawler_test.go b/internal/crawl/crawler/crawler_test.go index 100a1248c..3dbc6f3be 100644 --- a/internal/crawl/crawler/crawler_test.go +++ b/internal/crawl/crawler/crawler_test.go @@ -47,7 +47,7 @@ func (c testCrawler) FetchDocument(ctx context.Context, d *doc.Document) error { d.DocumentData = c.docs[i].DocumentData return nil } - return fmt.Errorf("Document %v does not exist for matcher: %s", + return fmt.Errorf("document %v does not exist for matcher: %s", d, c.matchPrefix) } @@ -72,7 +72,7 @@ func newCrawler(matchPrefix string, err error, // Crawl implements the Crawler interface for testing. func (c testCrawler) Crawl(ctx context.Context, - output chan<- CrawlerDocument) error { + output chan<- CrawledDocument) error { for i, d := range c.docs { isResource := true @@ -107,7 +107,7 @@ func (s sortableDocs) Len() int { } func TestCrawlerRunner(t *testing.T) { - fmt.Println("testing CrawlerRunner") + fmt.Println("testing CRunner") tests := []struct { tc []Crawler errs []error @@ -169,7 +169,7 @@ func TestCrawlerRunner(t *testing.T) { } for _, test := range tests { - output := make(chan CrawlerDocument) + output := make(chan CrawledDocument) wg := sync.WaitGroup{} wg.Add(1) @@ -178,7 +178,7 @@ func TestCrawlerRunner(t *testing.T) { defer close(output) defer wg.Done() - errs := CrawlerRunner(context.Background(), + errs := CRunner(context.Background(), output, test.tc) // Check that errors are returned as they should be. @@ -215,12 +215,12 @@ func TestCrawlFromSeed(t *testing.T) { fmt.Println("testing CrawlFromSeed") tests := []struct { - seed CrawlerSeed + seed CrawlSeed matcher string corpus []doc.KustomizationDocument }{ { - seed: CrawlerSeed{ + seed: CrawlSeed{ { RepositoryURL: kustomizeRepo, FilePath: "examples/helloWorld/kustomization.yaml", @@ -333,12 +333,12 @@ resources: cr := newCrawler(tc.matcher, nil, tc.corpus) visited := make(map[string]int) CrawlFromSeed(context.Background(), tc.seed, []Crawler{cr}, - func(d *doc.Document) (CrawlerDocument, error) { + func(d *doc.Document) (CrawledDocument, error) { return &doc.KustomizationDocument{ Document: *d, }, nil }, - func(d CrawlerDocument, cr Crawler) error { + func(d CrawledDocument, cr Crawler) error { visited[d.ID()]++ return nil }, diff --git a/internal/crawl/crawler/github/crawler.go b/internal/crawl/crawler/github/crawler.go index 81960b3e3..bbdc10aff 100644 --- a/internal/crawl/crawler/github/crawler.go +++ b/internal/crawl/crawler/github/crawler.go @@ -28,32 +28,22 @@ var logger = log.New(os.Stdout, "Github Crawler: ", // Implements crawler.Crawler. type githubCrawler struct { - client GitHubClient + client GhClient query Query } -type GitHubClient struct { +type GhClient struct { RequestConfig retryCount uint64 client *http.Client } -func NewClient(accessToken string, retryCount uint64, client *http.Client) GitHubClient { - return GitHubClient{ - retryCount: retryCount, - client: client, - RequestConfig: RequestConfig{ - perPage: githubMaxPageSize, - accessToken: accessToken, - }, - } -} - +/* func NewCrawler(accessToken string, retryCount uint64, client *http.Client, query Query) githubCrawler { return githubCrawler{ - client: GitHubClient{ + client: GhClient{ retryCount: retryCount, client: client, RequestConfig: RequestConfig{ @@ -64,12 +54,13 @@ func NewCrawler(accessToken string, retryCount uint64, client *http.Client, query: query, } } +*/ // Implements crawler.Crawler. func (gc githubCrawler) Crawl( - ctx context.Context, output chan<- crawler.CrawlerDocument) error { + ctx context.Context, output chan<- crawler.CrawledDocument) error { - noETagClient := GitHubClient{ + noETagClient := GhClient{ RequestConfig: gc.client.RequestConfig, client: &http.Client{Timeout: gc.client.client.Timeout}, retryCount: gc.client.retryCount, @@ -138,11 +129,11 @@ func (gc githubCrawler) FetchDocument(ctx context.Context, d *doc.Document) erro continue } } - return fmt.Errorf("File Not Found: %s", url) + return fmt.Errorf("file not found: %s", url) } func (gc githubCrawler) SetCreated(ctx context.Context, d *doc.Document) error { - fs := GithubFileSpec{} + fs := GhFileSpec{} fs.Repository.FullName = d.RepositoryURL + "/" + d.FilePath creationTime, err := gc.client.GetFileCreationTime(fs) if err != nil { @@ -165,10 +156,10 @@ func (gc githubCrawler) Match(d *doc.Document) bool { // processQuery follows all of the pages in a query, and updates/adds the // documents from the crawl to the datastore/index. -func processQuery(ctx context.Context, gcl GitHubClient, query string, - output chan<- crawler.CrawlerDocument) error { +func processQuery(ctx context.Context, gcl GhClient, query string, + output chan<- crawler.CrawledDocument) error { - queryPages := make(chan GithubResponseInfo) + queryPages := make(chan GhResponseInfo) go func() { // Forward the document metadata to the retrieval channel. @@ -209,8 +200,8 @@ func processQuery(ctx context.Context, gcl GitHubClient, query string, return errs } -func kustomizationResultAdapter(gcl GitHubClient, k GithubFileSpec) ( - crawler.CrawlerDocument, error) { +func kustomizationResultAdapter(gcl GhClient, k GhFileSpec) ( + crawler.CrawledDocument, error) { data, err := gcl.GetFileData(k) if err != nil { @@ -245,8 +236,8 @@ func kustomizationResultAdapter(gcl GitHubClient, k GithubFileSpec) ( // ForwardPaginatedQuery follows the links to the next pages and performs all of // the queries for a given search query, relaying the data from each request // back to an output channel. -func (gcl GitHubClient) ForwardPaginatedQuery(ctx context.Context, query string, - output chan<- GithubResponseInfo) error { +func (gcl GhClient) ForwardPaginatedQuery(ctx context.Context, query string, + output chan<- GhResponseInfo) error { logger.Println("querying: ", query) response := gcl.parseGithubResponse(query) @@ -275,7 +266,7 @@ func (gcl GitHubClient) ForwardPaginatedQuery(ctx context.Context, query string, } // GetFileData gets the bytes from a file. -func (gcl GitHubClient) GetFileData(k GithubFileSpec) ([]byte, error) { +func (gcl GhClient) GetFileData(k GhFileSpec) ([]byte, error) { url := gcl.ContentsRequest(k.Repository.FullName, k.Path) @@ -314,7 +305,7 @@ func (gcl GitHubClient) GetFileData(k GithubFileSpec) ([]byte, error) { return data, err } -func (gcl GitHubClient) GetDefaultBranch(url string) (string, error) { +func (gcl GhClient) GetDefaultBranch(url string) (string, error) { resp, err := gcl.GetReposData(url) if err != nil { return "", fmt.Errorf( @@ -341,8 +332,8 @@ func (gcl GitHubClient) GetDefaultBranch(url string) (string, error) { } // GetFileCreationTime gets the earliest date of a file. -func (gcl GitHubClient) GetFileCreationTime( - k GithubFileSpec) (time.Time, error) { +func (gcl GhClient) GetFileCreationTime( + k GhFileSpec) (time.Time, error) { url := gcl.CommitsRequest(k.Repository.FullName, k.Path) @@ -378,7 +369,7 @@ func (gcl GitHubClient) GetFileCreationTime( return defaultTime, fmt.Errorf( "%+v: failed to read metadata: %v", k, err) } - earliestDate := []DateSpec{} + var earliestDate []DateSpec err = json.Unmarshal(data, &earliestDate) size := len(earliestDate) if err != nil || size == 0 { @@ -413,18 +404,18 @@ func throttleRepoAPI() { type multiError []error -func (me multiError) Error() string { - size := len(me) + 2 +func (e multiError) Error() string { + size := len(e) + 2 strs := make([]string, size) strs[0] = "Errors [" - for i, err := range me { + for i, err := range e { strs[i+1] = "\t" + err.Error() } strs[size-1] = "]" return strings.Join(strs, "\n") } -type GithubFileSpec struct { +type GhFileSpec struct { Path string `json:"path,omitempty"` Repository struct { API string `json:"url,omitempty"` @@ -439,10 +430,10 @@ type githubResponse struct { TotalCount uint64 `json:"total_count,omitempty"` // Github representation of a file. - Items []GithubFileSpec `json:"items,omitempty"` + Items []GhFileSpec `json:"items,omitempty"` } -type GithubResponseInfo struct { +type GhResponseInfo struct { *http.Response Parsed *githubResponse Error error @@ -480,9 +471,9 @@ func parseGithubLinkFormat(links string) (string, string) { return next, last } -func (gcl GitHubClient) parseGithubResponse(getRequest string) GithubResponseInfo { +func (gcl GhClient) parseGithubResponse(getRequest string) GhResponseInfo { resp, err := gcl.SearchGithubAPI(getRequest) - requestInfo := GithubResponseInfo{ + requestInfo := GhResponseInfo{ Response: resp, Error: err, Parsed: nil, @@ -529,7 +520,7 @@ func (gcl GitHubClient) parseGithubResponse(getRequest string) GithubResponseInf // SearchGithubAPI performs a search query and handles rate limitting for // the 'code/search?' endpoint as well as timed retries in the case of abuse // prevention. -func (gcl GitHubClient) SearchGithubAPI(query string) (*http.Response, error) { +func (gcl GhClient) SearchGithubAPI(query string) (*http.Response, error) { throttleSearchAPI() return gcl.getWithRetry(query) } @@ -537,18 +528,18 @@ func (gcl GitHubClient) SearchGithubAPI(query string) (*http.Response, error) { // GetReposData performs a search query and handles rate limitting for // the '/repos' endpoint as well as timed retries in the case of abuse // prevention. -func (gcl GitHubClient) GetReposData(query string) (*http.Response, error) { +func (gcl GhClient) GetReposData(query string) (*http.Response, error) { throttleRepoAPI() return gcl.getWithRetry(query) } // User content (file contents) is not API rate limited, so there's no use in // throttling this call. -func (gcl GitHubClient) GetRawUserContent(query string) (*http.Response, error) { +func (gcl GhClient) GetRawUserContent(query string) (*http.Response, error) { return gcl.getWithRetry(query) } -func (gcl GitHubClient) getWithRetry( +func (gcl GhClient) getWithRetry( query string) (resp *http.Response, err error) { resp, err = gcl.client.Get(query) diff --git a/internal/crawl/crawler/github/queries.go b/internal/crawl/crawler/github/queries.go index a01d612fa..0a2d4c440 100644 --- a/internal/crawl/crawler/github/queries.go +++ b/internal/crawl/crawler/github/queries.go @@ -51,7 +51,7 @@ func (qf queryField) String() string { type Query []queryField func QueryWith(qfs ...queryField) Query { - return Query(qfs) + return qfs } func (q Query) String() string { diff --git a/internal/crawl/crawler/github/split_search_ranges.go b/internal/crawl/crawler/github/split_search_ranges.go index e62ea655a..919d3135d 100644 --- a/internal/crawl/crawler/github/split_search_ranges.go +++ b/internal/crawl/crawler/github/split_search_ranges.go @@ -147,11 +147,11 @@ type cachedSearch interface { // cache's performance. type githubCachedSearch struct { cache map[uint64]uint64 - gcl GitHubClient + gcl GhClient baseRequest request } -func newCache(client GitHubClient, query Query) githubCachedSearch { +func newCache(client GhClient, query Query) githubCachedSearch { return githubCachedSearch{ cache: map[uint64]uint64{ 0: 0, diff --git a/internal/crawl/index/elasticsearch.go b/internal/crawl/index/elasticsearch.go index 1e75c86bb..0d5dc6993 100644 --- a/internal/crawl/index/elasticsearch.go +++ b/internal/crawl/index/elasticsearch.go @@ -35,7 +35,7 @@ func newIndex(ctx context.Context, name string) (*index, error) { type readerFunc func(io.Reader) error -func ignoreResponseBody(reader io.Reader) error { +func ignoreResponseBody(_ io.Reader) error { return nil } diff --git a/pkg/target/chartinflatorplugin_test.go b/pkg/target/chartinflatorplugin_test.go index be0251a38..3c4d64fb1 100644 --- a/pkg/target/chartinflatorplugin_test.go +++ b/pkg/target/chartinflatorplugin_test.go @@ -61,7 +61,7 @@ kind: Secret metadata: labels: app: release-name-minecraft - chart: minecraft-1.1.1 + chart: minecraft-1.1.2 heritage: Tiller release: release-name name: LOOOOOOOONG-release-name-minecraft @@ -74,7 +74,7 @@ metadata: volume.alpha.kubernetes.io/storage-class: default labels: app: release-name-minecraft - chart: minecraft-1.1.1 + chart: minecraft-1.1.2 heritage: Tiller release: release-name name: LOOOOOOOONG-release-name-minecraft-datadir @@ -90,7 +90,7 @@ kind: Service metadata: labels: app: release-name-minecraft - chart: minecraft-1.1.1 + chart: minecraft-1.1.2 heritage: Tiller release: release-name name: LOOOOOOOONG-release-name-minecraft diff --git a/pkg/target/complexcomposition_test.go b/pkg/target/complexcomposition_test.go index 02d7557b3..669827842 100644 --- a/pkg/target/complexcomposition_test.go +++ b/pkg/target/complexcomposition_test.go @@ -7,7 +7,7 @@ import ( "strings" "testing" - kusttest_test "sigs.k8s.io/kustomize/v3/pkg/kusttest" + "sigs.k8s.io/kustomize/v3/pkg/kusttest" ) const httpsService = ` diff --git a/pkg/target/variableref_test.go b/pkg/target/variableref_test.go index 6c5101573..22048e89f 100644 --- a/pkg/target/variableref_test.go +++ b/pkg/target/variableref_test.go @@ -1,18 +1,5 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ +// Copyright 2019 The Kubernetes Authors. +// SPDX-License-Identifier: Apache-2.0 package target_test @@ -343,31 +330,33 @@ resources: - ../o2 `) - const presumablyDesired = ` -apiVersion: v1 -kind: Pod -metadata: - name: p1-base-myServerPod -spec: - containers: - - env: - - name: POD_NAME - value: p1-base-myServerPod - image: whatever - name: myServer ---- -apiVersion: v1 -kind: Pod -metadata: - name: p2-base-myServerPod -spec: - containers: - - env: - - name: POD_NAME - value: p2-base-myServerPod - image: whatever - name: myServer -` + /* + const presumablyDesired = ` + apiVersion: v1 + kind: Pod + metadata: + name: p1-base-myServerPod + spec: + containers: + - env: + - name: POD_NAME + value: p1-base-myServerPod + image: whatever + name: myServer + --- + apiVersion: v1 + kind: Pod + metadata: + name: p2-base-myServerPod + spec: + containers: + - env: + - name: POD_NAME + value: p2-base-myServerPod + image: whatever + name: myServer + ` + */ _, err := th.MakeKustTarget().MakeCustomizedResMap() if err == nil { t.Fatalf("should have an error") diff --git a/plugin/someteam.example.com/v1/chartinflator/ChartInflator_test.go b/plugin/someteam.example.com/v1/chartinflator/ChartInflator_test.go index 0bd1575fd..c5117f259 100644 --- a/plugin/someteam.example.com/v1/chartinflator/ChartInflator_test.go +++ b/plugin/someteam.example.com/v1/chartinflator/ChartInflator_test.go @@ -42,7 +42,7 @@ kind: Secret metadata: labels: app: release-name-minecraft - chart: minecraft-1.1.1 + chart: minecraft-1.1.2 heritage: Tiller release: release-name name: release-name-minecraft @@ -55,7 +55,7 @@ metadata: volume.alpha.kubernetes.io/storage-class: default labels: app: release-name-minecraft - chart: minecraft-1.1.1 + chart: minecraft-1.1.2 heritage: Tiller release: release-name name: release-name-minecraft-datadir @@ -71,7 +71,7 @@ kind: Service metadata: labels: app: release-name-minecraft - chart: minecraft-1.1.1 + chart: minecraft-1.1.2 heritage: Tiller release: release-name name: release-name-minecraft