Fix some nits

internal/crawl/ROADMAP.md

@@ -82,13 +82,13 @@ following:
         "docID45",
         ...
       ],
-    }
+    },
     {
       "identifier": "another:k8s:feature",
       "documents": [
         ...
       ],
-    }
+    },
     ...
   ]
 

internal/crawl/crawler/crawler.go

@@ -27,7 +27,7 @@ type Crawler interface {
 	// Crawl returns when it is done processing. This method does not take
 	// ownership of the channel. The channel is write only, and it
 	// designates where the crawler should forward the documents.
-	Crawl(ctx context.Context, output chan<- CrawlerDocument) error
+	Crawl(ctx context.Context, output chan<- CrawledDocument) error
 
 	// Get the document data given the FilePath, Repo, and Ref/Tag/Branch.
 	FetchDocument(context.Context, *doc.Document) error
@@ -37,21 +37,21 @@ type Crawler interface {
 	Match(*doc.Document) bool
 }
 
-type CrawlerDocument interface {
+type CrawledDocument interface {
 	ID() string
 	GetDocument() *doc.Document
 	GetResources() ([]*doc.Document, error)
 	WasCached() bool
 }
 
-type CrawlerSeed []*doc.Document
+type CrawlSeed []*doc.Document
 
-type IndexFunc func(CrawlerDocument, Crawler) error
-type Converter func(*doc.Document) (CrawlerDocument, error)
+type IndexFunc func(CrawledDocument, Crawler) error
+type Converter func(*doc.Document) (CrawledDocument, error)
 
 // Cleaner, more efficient, and more extensible crawler implementation.
 // The seed must include the ids of each document in the index.
-func CrawlFromSeed(ctx context.Context, seed CrawlerSeed,
+func CrawlFromSeed(ctx context.Context, seed CrawlSeed,
 	crawlers []Crawler, conv Converter, indx IndexFunc) {
 
 	seen := make(map[string]struct{})
@@ -63,7 +63,7 @@ func CrawlFromSeed(ctx context.Context, seed CrawlerSeed,
 		logger.Println("error: ", err)
 	}
 
-	stack := make(CrawlerSeed, 0)
+	stack := make(CrawlSeed, 0)
 
 	findMatch := func(d *doc.Document) Crawler {
 		for _, crawl := range crawlers {
@@ -75,7 +75,7 @@ func CrawlFromSeed(ctx context.Context, seed CrawlerSeed,
 		return nil
 	}
 
-	addBranches := func(cdoc CrawlerDocument, match Crawler) {
+	addBranches := func(cdoc CrawledDocument, match Crawler) {
 		if _, ok := seen[cdoc.ID()]; ok {
 			return
 		}
@@ -101,7 +101,7 @@ func CrawlFromSeed(ctx context.Context, seed CrawlerSeed,
 		}
 	}
 
-	doCrawl := func(docsPtr *CrawlerSeed) {
+	doCrawl := func(docsPtr *CrawlSeed) {
 		for len(*docsPtr) > 0 {
 			back := len(*docsPtr) - 1
 			next := (*docsPtr)[back]
@@ -139,7 +139,7 @@ func CrawlFromSeed(ctx context.Context, seed CrawlerSeed,
 	logger.Printf("crawling %d new documents found in the seed\n", len(stack))
 	doCrawl(&stack)
 
-	ch := make(chan CrawlerDocument, 1<<10)
+	ch := make(chan CrawledDocument, 1<<10)
 	wg := sync.WaitGroup{}
 
 	wg.Add(1)
@@ -160,7 +160,7 @@ func CrawlFromSeed(ctx context.Context, seed CrawlerSeed,
 	}()
 
 	// Exploration through APIs.
-	errs := CrawlerRunner(ctx, ch, crawlers)
+	errs := CRunner(ctx, ch, crawlers)
 	if errs != nil {
 		for _, err := range errs {
 			logIfErr(err)
@@ -175,7 +175,7 @@ func CrawlFromSeed(ctx context.Context, seed CrawlerSeed,
 	doCrawl(&stack)
 }
 
-// CrawlerRunner is a blocking function and only returns once all of the
+// CRunner is a blocking function and only returns once all of the
 // crawlers are finished with execution.
 //
 // This function uses the output channel to forward kustomization documents
@@ -186,14 +186,14 @@ func CrawlFromSeed(ctx context.Context, seed CrawlerSeed,
 // index of the crawler that emitted the error. Although the errors themselves
 // can be nil, the array will always be exactly the size of the crawlers array.
 //
-// Crawler Runner takes in a seed, which represents the documents stored in an
+// CRunner takes in a seed, which represents the documents stored in an
 // index somewhere. The document data is not required to be populated. If there
 // are many documents, this is preferable. The order of iteration over the seed
-// is not garanteed, but the CrawlerRunner does guarantee that every element
+// is not garanteed, but the CRunner does guarantee that every element
 // from the seed will be processed before any other documents from the
 // crawlers.
-func CrawlerRunner(ctx context.Context,
-	output chan<- CrawlerDocument, crawlers []Crawler) []error {
+func CRunner(ctx context.Context,
+	output chan<- CrawledDocument, crawlers []Crawler) []error {
 
 	errs := make([]error, len(crawlers))
 	wg := sync.WaitGroup{}
@@ -201,12 +201,12 @@ func CrawlerRunner(ctx context.Context,
 	for i, crawler := range crawlers {
 		// Crawler implementations get their own channels to prevent a
 		// crawler from closing the main output channel.
-		docs := make(chan CrawlerDocument)
+		docs := make(chan CrawledDocument)
 		wg.Add(2)
 
 		// Forward all of the documents from this crawler's channel to
 		// the main output channel.
-		go func(docs <-chan CrawlerDocument) {
+		go func(docs <-chan CrawledDocument) {
 			defer wg.Done()
 			for doc := range docs {
 				output <- doc
@@ -215,7 +215,7 @@ func CrawlerRunner(ctx context.Context,
 
 		// Run this crawler and capture its returned error.
 		go func(idx int, crawler Crawler,
-			docs chan<- CrawlerDocument) {
+			docs chan<- CrawledDocument) {
 
 			defer func() {
 				wg.Done()

internal/crawl/crawler/crawler_test.go

@@ -47,7 +47,7 @@ func (c testCrawler) FetchDocument(ctx context.Context, d *doc.Document) error {
 		d.DocumentData = c.docs[i].DocumentData
 		return nil
 	}
-	return fmt.Errorf("Document %v does not exist for matcher: %s",
+	return fmt.Errorf("document %v does not exist for matcher: %s",
 		d, c.matchPrefix)
 }
 
@@ -72,7 +72,7 @@ func newCrawler(matchPrefix string, err error,
 
 // Crawl implements the Crawler interface for testing.
 func (c testCrawler) Crawl(ctx context.Context,
-	output chan<- CrawlerDocument) error {
+	output chan<- CrawledDocument) error {
 
 	for i, d := range c.docs {
 		isResource := true
@@ -107,7 +107,7 @@ func (s sortableDocs) Len() int {
 }
 
 func TestCrawlerRunner(t *testing.T) {
-	fmt.Println("testing CrawlerRunner")
+	fmt.Println("testing CRunner")
 	tests := []struct {
 		tc   []Crawler
 		errs []error
@@ -169,7 +169,7 @@ func TestCrawlerRunner(t *testing.T) {
 	}
 
 	for _, test := range tests {
-		output := make(chan CrawlerDocument)
+		output := make(chan CrawledDocument)
 		wg := sync.WaitGroup{}
 		wg.Add(1)
 
@@ -178,7 +178,7 @@ func TestCrawlerRunner(t *testing.T) {
 			defer close(output)
 			defer wg.Done()
 
-			errs := CrawlerRunner(context.Background(),
+			errs := CRunner(context.Background(),
 				output, test.tc)
 
 			// Check that errors are returned as they should be.
@@ -215,12 +215,12 @@ func TestCrawlFromSeed(t *testing.T) {
 	fmt.Println("testing CrawlFromSeed")
 
 	tests := []struct {
-		seed    CrawlerSeed
+		seed    CrawlSeed
 		matcher string
 		corpus  []doc.KustomizationDocument
 	}{
 		{
-			seed: CrawlerSeed{
+			seed: CrawlSeed{
 				{
 					RepositoryURL: kustomizeRepo,
 					FilePath:      "examples/helloWorld/kustomization.yaml",
@@ -333,12 +333,12 @@ resources:
 		cr := newCrawler(tc.matcher, nil, tc.corpus)
 		visited := make(map[string]int)
 		CrawlFromSeed(context.Background(), tc.seed, []Crawler{cr},
-			func(d *doc.Document) (CrawlerDocument, error) {
+			func(d *doc.Document) (CrawledDocument, error) {
 				return &doc.KustomizationDocument{
 					Document: *d,
 				}, nil
 			},
-			func(d CrawlerDocument, cr Crawler) error {
+			func(d CrawledDocument, cr Crawler) error {
 				visited[d.ID()]++
 				return nil
 			},

internal/crawl/crawler/github/crawler.go

@@ -28,32 +28,22 @@ var logger = log.New(os.Stdout, "Github Crawler: ",
 
 // Implements crawler.Crawler.
 type githubCrawler struct {
-	client GitHubClient
+	client GhClient
 	query  Query
 }
 
-type GitHubClient struct {
+type GhClient struct {
 	RequestConfig
 	retryCount uint64
 	client     *http.Client
 }
 
-func NewClient(accessToken string, retryCount uint64, client *http.Client) GitHubClient {
-	return GitHubClient{
-		retryCount: retryCount,
-		client:     client,
-		RequestConfig: RequestConfig{
-			perPage:     githubMaxPageSize,
-			accessToken: accessToken,
-		},
-	}
-}
-
+/*
 func NewCrawler(accessToken string, retryCount uint64, client *http.Client,
 	query Query) githubCrawler {
 
 	return githubCrawler{
-		client: GitHubClient{
+		client: GhClient{
 			retryCount: retryCount,
 			client:     client,
 			RequestConfig: RequestConfig{
@@ -64,12 +54,13 @@ func NewCrawler(accessToken string, retryCount uint64, client *http.Client,
 		query: query,
 	}
 }
+*/
 
 // Implements crawler.Crawler.
 func (gc githubCrawler) Crawl(
-	ctx context.Context, output chan<- crawler.CrawlerDocument) error {
+	ctx context.Context, output chan<- crawler.CrawledDocument) error {
 
-	noETagClient := GitHubClient{
+	noETagClient := GhClient{
 		RequestConfig: gc.client.RequestConfig,
 		client:        &http.Client{Timeout: gc.client.client.Timeout},
 		retryCount:    gc.client.retryCount,
@@ -138,11 +129,11 @@ func (gc githubCrawler) FetchDocument(ctx context.Context, d *doc.Document) erro
 			continue
 		}
 	}
-	return fmt.Errorf("File Not Found: %s", url)
+	return fmt.Errorf("file not found: %s", url)
 }
 
 func (gc githubCrawler) SetCreated(ctx context.Context, d *doc.Document) error {
-	fs := GithubFileSpec{}
+	fs := GhFileSpec{}
 	fs.Repository.FullName = d.RepositoryURL + "/" + d.FilePath
 	creationTime, err := gc.client.GetFileCreationTime(fs)
 	if err != nil {
@@ -165,10 +156,10 @@ func (gc githubCrawler) Match(d *doc.Document) bool {
 
 // processQuery follows all of the pages in a query, and updates/adds the
 // documents from the crawl to the datastore/index.
-func processQuery(ctx context.Context, gcl GitHubClient, query string,
-	output chan<- crawler.CrawlerDocument) error {
+func processQuery(ctx context.Context, gcl GhClient, query string,
+	output chan<- crawler.CrawledDocument) error {
 
-	queryPages := make(chan GithubResponseInfo)
+	queryPages := make(chan GhResponseInfo)
 
 	go func() {
 		// Forward the document metadata to the retrieval channel.
@@ -209,8 +200,8 @@ func processQuery(ctx context.Context, gcl GitHubClient, query string,
 	return errs
 }
 
-func kustomizationResultAdapter(gcl GitHubClient, k GithubFileSpec) (
-	crawler.CrawlerDocument, error) {
+func kustomizationResultAdapter(gcl GhClient, k GhFileSpec) (
+	crawler.CrawledDocument, error) {
 
 	data, err := gcl.GetFileData(k)
 	if err != nil {
@@ -245,8 +236,8 @@ func kustomizationResultAdapter(gcl GitHubClient, k GithubFileSpec) (
 // ForwardPaginatedQuery follows the links to the next pages and performs all of
 // the queries for a given search query, relaying the data from each request
 // back to an output channel.
-func (gcl GitHubClient) ForwardPaginatedQuery(ctx context.Context, query string,
-	output chan<- GithubResponseInfo) error {
+func (gcl GhClient) ForwardPaginatedQuery(ctx context.Context, query string,
+	output chan<- GhResponseInfo) error {
 
 	logger.Println("querying: ", query)
 	response := gcl.parseGithubResponse(query)
@@ -275,7 +266,7 @@ func (gcl GitHubClient) ForwardPaginatedQuery(ctx context.Context, query string,
 }
 
 // GetFileData gets the bytes from a file.
-func (gcl GitHubClient) GetFileData(k GithubFileSpec) ([]byte, error) {
+func (gcl GhClient) GetFileData(k GhFileSpec) ([]byte, error) {
 
 	url := gcl.ContentsRequest(k.Repository.FullName, k.Path)
 
@@ -314,7 +305,7 @@ func (gcl GitHubClient) GetFileData(k GithubFileSpec) ([]byte, error) {
 	return data, err
 }
 
-func (gcl GitHubClient) GetDefaultBranch(url string) (string, error) {
+func (gcl GhClient) GetDefaultBranch(url string) (string, error) {
 	resp, err := gcl.GetReposData(url)
 	if err != nil {
 		return "", fmt.Errorf(
@@ -341,8 +332,8 @@ func (gcl GitHubClient) GetDefaultBranch(url string) (string, error) {
 }
 
 // GetFileCreationTime gets the earliest date of a file.
-func (gcl GitHubClient) GetFileCreationTime(
-	k GithubFileSpec) (time.Time, error) {
+func (gcl GhClient) GetFileCreationTime(
+	k GhFileSpec) (time.Time, error) {
 
 	url := gcl.CommitsRequest(k.Repository.FullName, k.Path)
 
@@ -378,7 +369,7 @@ func (gcl GitHubClient) GetFileCreationTime(
 		return defaultTime, fmt.Errorf(
 			"%+v: failed to read metadata: %v", k, err)
 	}
-	earliestDate := []DateSpec{}
+	var earliestDate []DateSpec
 	err = json.Unmarshal(data, &earliestDate)
 	size := len(earliestDate)
 	if err != nil || size == 0 {
@@ -413,18 +404,18 @@ func throttleRepoAPI() {
 
 type multiError []error
 
-func (me multiError) Error() string {
-	size := len(me) + 2
+func (e multiError) Error() string {
+	size := len(e) + 2
 	strs := make([]string, size)
 	strs[0] = "Errors ["
-	for i, err := range me {
+	for i, err := range e {
 		strs[i+1] = "\t" + err.Error()
 	}
 	strs[size-1] = "]"
 	return strings.Join(strs, "\n")
 }
 
-type GithubFileSpec struct {
+type GhFileSpec struct {
 	Path       string `json:"path,omitempty"`
 	Repository struct {
 		API      string `json:"url,omitempty"`
@@ -439,10 +430,10 @@ type githubResponse struct {
 	TotalCount uint64 `json:"total_count,omitempty"`
 
 	// Github representation of a file.
-	Items []GithubFileSpec `json:"items,omitempty"`
+	Items []GhFileSpec `json:"items,omitempty"`
 }
 
-type GithubResponseInfo struct {
+type GhResponseInfo struct {
 	*http.Response
 	Parsed  *githubResponse
 	Error   error
@@ -480,9 +471,9 @@ func parseGithubLinkFormat(links string) (string, string) {
 	return next, last
 }
 
-func (gcl GitHubClient) parseGithubResponse(getRequest string) GithubResponseInfo {
+func (gcl GhClient) parseGithubResponse(getRequest string) GhResponseInfo {
 	resp, err := gcl.SearchGithubAPI(getRequest)
-	requestInfo := GithubResponseInfo{
+	requestInfo := GhResponseInfo{
 		Response: resp,
 		Error:    err,
 		Parsed:   nil,
@@ -529,7 +520,7 @@ func (gcl GitHubClient) parseGithubResponse(getRequest string) GithubResponseInf
 // SearchGithubAPI performs a search query and handles rate limitting for
 // the 'code/search?' endpoint as well as timed retries in the case of abuse
 // prevention.
-func (gcl GitHubClient) SearchGithubAPI(query string) (*http.Response, error) {
+func (gcl GhClient) SearchGithubAPI(query string) (*http.Response, error) {
 	throttleSearchAPI()
 	return gcl.getWithRetry(query)
 }
@@ -537,18 +528,18 @@ func (gcl GitHubClient) SearchGithubAPI(query string) (*http.Response, error) {
 // GetReposData performs a search query and handles rate limitting for
 // the '/repos' endpoint as well as timed retries in the case of abuse
 // prevention.
-func (gcl GitHubClient) GetReposData(query string) (*http.Response, error) {
+func (gcl GhClient) GetReposData(query string) (*http.Response, error) {
 	throttleRepoAPI()
 	return gcl.getWithRetry(query)
 }
 
 // User content (file contents) is not API rate limited, so there's no use in
 // throttling this call.
-func (gcl GitHubClient) GetRawUserContent(query string) (*http.Response, error) {
+func (gcl GhClient) GetRawUserContent(query string) (*http.Response, error) {
 	return gcl.getWithRetry(query)
 }
 
-func (gcl GitHubClient) getWithRetry(
+func (gcl GhClient) getWithRetry(
 	query string) (resp *http.Response, err error) {
 
 	resp, err = gcl.client.Get(query)

internal/crawl/crawler/github/queries.go

@@ -51,7 +51,7 @@ func (qf queryField) String() string {
 type Query []queryField
 
 func QueryWith(qfs ...queryField) Query {
-	return Query(qfs)
+	return qfs
 }
 
 func (q Query) String() string {

internal/crawl/crawler/github/split_search_ranges.go

@@ -147,11 +147,11 @@ type cachedSearch interface {
 //    cache's performance.
 type githubCachedSearch struct {
 	cache       map[uint64]uint64
-	gcl         GitHubClient
+	gcl         GhClient
 	baseRequest request
 }
 
-func newCache(client GitHubClient, query Query) githubCachedSearch {
+func newCache(client GhClient, query Query) githubCachedSearch {
 	return githubCachedSearch{
 		cache: map[uint64]uint64{
 			0: 0,

internal/crawl/index/elasticsearch.go

@@ -35,7 +35,7 @@ func newIndex(ctx context.Context, name string) (*index, error) {
 
 type readerFunc func(io.Reader) error
 
-func ignoreResponseBody(reader io.Reader) error {
+func ignoreResponseBody(_ io.Reader) error {
 	return nil
 }