Improve the efficiency of crawling github by skipping the documents

already in the index

api/internal/crawl/cmd/crawler/crawler.go

@@ -187,6 +187,12 @@ func main() {
 		crawler.CrawlFromSeed(ctx, seedDocs, crawlers, docConverter, indexFunc, seen)
 	case CrawlGithub:
 		crawlers := []crawler.Crawler{ghCrawlerConstructor("", "")}
+		// add all the documents in the index into seen.
+		// this greatly reduces the time overhead of CrawlGithub.
+		getSeedDocsFunc()
+		for _, d := range seedDocs {
+			seen[d.ID()] = d.FileType
+		}
 		crawler.CrawlGithub(ctx, crawlers, docConverter, indexFunc, seen)
 	case CrawlUser:
 		if *githubUserPtr == "" {