diff --git a/api/internal/crawl/cmd/kustomize_stats/main.go b/api/internal/crawl/cmd/kustomize_stats/main.go index b2c3d5a04..803e5dbac 100644 --- a/api/internal/crawl/cmd/kustomize_stats/main.go +++ b/api/internal/crawl/cmd/kustomize_stats/main.go @@ -14,11 +14,6 @@ import ( "sigs.k8s.io/kustomize/api/internal/crawl/index" ) -const ( - githubAccessTokenVar = "GITHUB_ACCESS_TOKEN" - retryCount = 3 -) - // iterateArr adds each item in arr into countMap. func iterateArr(arr []string, countMap map[string]int) { for _, item := range arr { diff --git a/api/internal/crawl/cmd/log-parser/kustomize-stats-cmd b/api/internal/crawl/cmd/log-parser/kustomize-stats-cmd new file mode 100644 index 000000000..4f2cec9b7 --- /dev/null +++ b/api/internal/crawl/cmd/log-parser/kustomize-stats-cmd @@ -0,0 +1,7 @@ +wget -O log +go build . +./log-parser log >out +cat out | grep "kind \`" | cut -d\` -f2 | tail -n 50 +cat out | grep "kind \`" | awk '{print $6}' | tail -n 50 +cat out | grep "feature \`" | grep -v "\`resources\`" | grep -v -e "\`apiVersion\`" | grep -v -e "\`apiversion\`" | cut -d\` -f2 +cat out | grep "feature \`" | grep -v "\`resources\`" | grep -v -e "\`apiVersion\`" | grep -v -e "\`apiversion\`" | awk '{print $6}' diff --git a/api/internal/crawl/config/crawler/base/.gitignore b/api/internal/crawl/config/crawler/base/.gitignore new file mode 100644 index 000000000..cea109352 --- /dev/null +++ b/api/internal/crawl/config/crawler/base/.gitignore @@ -0,0 +1 @@ +github_api_secret.txt diff --git a/api/internal/crawl/config/crawler/kustomize_stats/job.yaml b/api/internal/crawl/config/crawler/kustomize_stats/job.yaml index 9e2457983..b78cc28ce 100644 --- a/api/internal/crawl/config/crawler/kustomize_stats/job.yaml +++ b/api/internal/crawl/config/crawler/kustomize_stats/job.yaml @@ -11,13 +11,8 @@ spec: image: gcr.io/haiyanmeng-gke-dev/kustomize_stats:v1 imagePullPolicy: Always command: ["/kustomize_stats"] - args: ["--index=kustomize", "--kinds=50", "--identifiers=50", "--kustomize-features=50"] + args: ["--index=kustomize", "--kinds=51", "--identifiers=50", "--kustomize-features=50"] env: - - name: GITHUB_ACCESS_TOKEN - valueFrom: - secretKeyRef: - name: github-access-token - key: token - name: ELASTICSEARCH_URL valueFrom: configMapKeyRef: diff --git a/api/internal/crawl/config/elastic/esbackup.yaml b/api/internal/crawl/config/elastic/esbackup.yaml index c509ba206..a3b5e6da8 100644 --- a/api/internal/crawl/config/elastic/esbackup.yaml +++ b/api/internal/crawl/config/elastic/esbackup.yaml @@ -1,6 +1,8 @@ # Creating `esbackup/kustomize-backbup` will create the `kustomize-backup` snapshot repository. -# Deleting `esbackup/kustomize-backbup` will delete the `kustomize-backup` snapshot repository and all the snapshots in the repository. +# Deleting `esbackup/kustomize-backbup` will delete the `kustomize-backup` snapshot repository. +# Deleting `esbackup/kustomize-backbup` will NOT delete the snapshots in the `kustomize-backup` snapshot repository, instead makes all the snapshots in the repository inaccessible. # Deleting `esbackup/kustomize-backbup` will NOT delete essnapshot and esrestore objects. +# If you delete the `kustomize-backup` snapshot repository directly without deleting `esbackup/kustomize-backbup`, the ESBackup object will not recreate the snapshot repository. apiVersion: elasticsearch.cloud.google.com/v1alpha1 kind: ESBackup metadata: diff --git a/api/internal/crawl/config/elastic/esrestore.yaml b/api/internal/crawl/config/elastic/esrestore.yaml index 0d77bed93..4490d8248 100644 --- a/api/internal/crawl/config/elastic/esrestore.yaml +++ b/api/internal/crawl/config/elastic/esrestore.yaml @@ -1,6 +1,7 @@ # Creating `esrestore/kustomize-restore` will restore the `kuostmize` index in the `kustomize-snapshot` snapshot to a new index named `kusotmize-restore`. # Deleting `esrestore/kustomize-restore` will not delete the restored index. # Deleting `esrestore/kustomize-restore` should happen before deleting `essnapshot/kustomize-snapshot`. +# If the `kusotmize-restore` index is deleted manually, the ESRestore object will restore the `kustomize` index to it again. apiVersion: elasticsearch.cloud.google.com/v1alpha1 kind: ESRestore metadata: diff --git a/api/internal/crawl/config/elastic/essnaptshot.yaml b/api/internal/crawl/config/elastic/essnaptshot.yaml index 2a9468960..46b7c95eb 100644 --- a/api/internal/crawl/config/elastic/essnaptshot.yaml +++ b/api/internal/crawl/config/elastic/essnaptshot.yaml @@ -1,6 +1,7 @@ # Creating `essnapshot/kustomize-snapshot` will create a snapshot named `kustomize-snapshot` in the `kustomize-backup` snapshot repository. # Deleting `essnapshot/kustomize-snapshot` will delete the snapshot. # Deleting `essnapshot/kustomize-snapshot` should happen before deleting `esbackup/kustomize-backup`. +# If the `kustomize-snapshot` snapshot is deleted directly without deleting `essnapshot/kustomize-snapshot`, the ESSnapshot object will recreate the snapshot. apiVersion: elasticsearch.cloud.google.com/v1alpha1 kind: ESSnapshot metadata: diff --git a/api/internal/crawl/search_cmds/creationTime.md b/api/internal/crawl/search_cmds/creationTime.md index 08b6b3e1e..a0a11f412 100644 --- a/api/internal/crawl/search_cmds/creationTime.md +++ b/api/internal/crawl/search_cmds/creationTime.md @@ -1,6 +1,6 @@ Find out the largest value of the `creationTime` field: ``` -curl -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' +curl -s -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' { "aggs" : { "max_creationTime" : { "max" : { "field" : "creationTime" } } @@ -11,7 +11,7 @@ curl -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'C Find out the smallest value of the `creationTime` field: ``` -curl -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' +curl -s -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' { "aggs" : { "min_creationTime" : { "min" : { "field" : "creationTime" } } @@ -22,7 +22,7 @@ curl -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'C Find out the smallest value of the `creationTime` field of all the kustomization files: ``` -curl -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' +curl -s -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' { "query": { "bool": { @@ -40,7 +40,7 @@ curl -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'C Find out the smallest value of the `creationTime` field of all kustomize resource files: ``` -curl -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' +curl -s -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' { "query": { "bool": { @@ -61,7 +61,7 @@ curl -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'C Find out the smallest value of the `creationTime` field of all kustomize generator files: ``` -curl -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' +curl -s -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' { "query": { "bool": { @@ -82,7 +82,7 @@ curl -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'C Find out the smallest value of the `creationTime` field of all kustomize transformer files: ``` -curl -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' +curl -s -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' { "query": { "bool": { @@ -103,7 +103,7 @@ curl -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'C Query all the documents whose `creationTime` <= `2016-07-29T17:38:26.000Z`: ``` -curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d' +curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d' { "query": { "range": { @@ -118,7 +118,7 @@ curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-T Query all the documents whose `creationTime` falls within the specific range: ``` -curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d' +curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d' { "query": { "range": { @@ -134,7 +134,7 @@ curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-T Query all the kustomization files whose `creationTime` falls within the specific range: ``` -curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d' +curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d' { "size": 20, "query": { @@ -158,7 +158,7 @@ curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-T Aggregate how many new kustomization files were added into Github each month: ``` -curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' +curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' { "query": { "bool": { @@ -181,7 +181,7 @@ curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Co Aggregate how many new kustomize resource files were added into Github each month: ``` -curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' +curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' { "query": { "bool": { @@ -207,7 +207,7 @@ curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Co Aggregate how many new kustomize generator files were added into Github each month: ``` -curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' +curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' { "query": { "bool": { @@ -233,7 +233,7 @@ curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Co Aggregate how many new kustomize transformer files were added into Github each month: ``` -curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' +curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' { "query": { "bool": { @@ -259,7 +259,7 @@ curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Co Aggregate how many new kustomization files were added into Github each year: ``` -curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' +curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' { "query": { "bool": { @@ -282,7 +282,7 @@ curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Co Aggregate how many new kustomize resource files were added into Github each year: ``` -curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' +curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' { "query": { "bool": { @@ -308,7 +308,7 @@ curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Co Aggregate how many new kustomize generator files were added into Github each year: ``` -curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' +curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' { "query": { "bool": { @@ -334,7 +334,7 @@ curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Co Aggregate how many new kustomize transformer files were added into Github each year: ``` -curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' +curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' { "query": { "bool": { @@ -360,7 +360,7 @@ curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Co Find the generator files created within the given time range: ``` -curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d' +curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d' { "query": { "bool": { @@ -386,7 +386,7 @@ curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-T Find the transformer files created within the given time range: ``` -curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d' +curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d' { "query": { "bool": { diff --git a/api/internal/crawl/search_cmds/defaultBranch.md b/api/internal/crawl/search_cmds/defaultBranch.md index d4cab8e6b..b4e1643a2 100644 --- a/api/internal/crawl/search_cmds/defaultBranch.md +++ b/api/internal/crawl/search_cmds/defaultBranch.md @@ -1,6 +1,6 @@ Count distinct values of the `defaultBranch` field: ``` -curl -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' +curl -s -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' { "aggs" : { "defaultBranch_count" : { @@ -17,7 +17,7 @@ curl -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'C List all the github branches where kustomization files and kustomize resource files live, and how many kustomization files and kustomize resource files live in each branch: ``` -curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d' +curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d' { "aggs" : { "defaultBranch" : { diff --git a/api/internal/crawl/search_cmds/fieldExistence.md b/api/internal/crawl/search_cmds/fieldExistence.md index f804c784a..9f8fdb135 100644 --- a/api/internal/crawl/search_cmds/fieldExistence.md +++ b/api/internal/crawl/search_cmds/fieldExistence.md @@ -1,7 +1,7 @@ Count the documents whose `document` field is empty (The reason why the `document` field of a document is empty is because of empty documents): ``` -curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d' +curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d' { "size": 10000, "query": { @@ -19,7 +19,7 @@ curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-T Find all the documents having the `creationTime` field set: ``` -curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d' +curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d' { "query": { "exists": { @@ -32,7 +32,7 @@ curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-T Find all the documents whose `creationTime` field is not set: ``` -curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d' +curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d' { "size": 10000, "query": { diff --git a/api/internal/crawl/search_cmds/fileType.md b/api/internal/crawl/search_cmds/fileType.md index 3be7b41e1..c533ed34b 100644 --- a/api/internal/crawl/search_cmds/fileType.md +++ b/api/internal/crawl/search_cmds/fileType.md @@ -1,6 +1,6 @@ Find all the documents having the `fileType` field set: ``` -curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d' +curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d' { "query": { "exists": { @@ -13,7 +13,7 @@ curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-T Find all the documents whose `fileType` field is not set: ``` -curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d' +curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d' { "size": 10000, "query": { @@ -31,7 +31,7 @@ curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-T Search for all the documents whose `fileType` field is `resource`: ``` -curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d' +curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d' { "query": { "bool": { @@ -46,7 +46,7 @@ curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-T Search for all the kustomization files whose `fileType` field is `resource`: ``` -curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d' +curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d' { "query": { "bool": { @@ -62,7 +62,7 @@ curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-T Search for all the kustomize resource files: ``` -curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d' +curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d' { "query": { "bool": { @@ -80,7 +80,7 @@ curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-T Search all the kustomization files including a `generators` field: ``` -curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d' +curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d' { "size": 10000, "query": { @@ -103,7 +103,7 @@ curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-T Search for all the documents whose `fileType` field is `generator`: ``` -curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d' +curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d' { "query": { "bool": { @@ -118,7 +118,7 @@ curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-T Search for all the kustomization files whose `fileType` field is `generator`: ``` -curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d' +curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d' { "query": { "bool": { @@ -134,7 +134,7 @@ curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-T Search for all the kustomize generator files: ``` -curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d' +curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d' { "query": { "bool": { @@ -152,7 +152,7 @@ curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-T Search all the kustomization files including a `transformers` field: ``` -curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d' +curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d' { "size": 10000, "query": { @@ -175,7 +175,7 @@ curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-T Search for all the documents whose `fileType` field is `transformer`: ``` -curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d' +curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d' { "query": { "bool": { @@ -190,7 +190,7 @@ curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-T Search for all the kustomization files whose `fileType` field is `transformer`: ``` -curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d' +curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d' { "query": { "bool": { @@ -206,7 +206,7 @@ curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-T Search for all the kustomize transformer files: ``` -curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d' +curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d' { "query": { "bool": { @@ -224,7 +224,7 @@ curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-T Count distinct values of the `fileType` field: ``` -curl -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' +curl -s -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' { "aggs" : { "fileType_count" : { @@ -240,7 +240,7 @@ curl -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'C List all the values of the `fileType` field and the frequency of each value: ``` -curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' +curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' { "aggs" : { "fileType" : { @@ -257,7 +257,7 @@ curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Co For all the kustomization files in the index, list all the values of the `fileType` field and the frequency of each value: ``` -curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' +curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' { "query": { "bool": { @@ -280,7 +280,7 @@ curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Co For all the non-kustomization files in the index, list all the values of the `fileType` field and the frequency of each value: ``` -curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' +curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' { "query": { "bool": { diff --git a/api/internal/crawl/search_cmds/generator.md b/api/internal/crawl/search_cmds/generator.md index 0b324600f..9a7a7e842 100644 --- a/api/internal/crawl/search_cmds/generator.md +++ b/api/internal/crawl/search_cmds/generator.md @@ -1,7 +1,7 @@ Find all the generator files whose `kinds` field includes `ChartRenderer`, and only output certain fields of each document: ``` -curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d' +curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d' { "size": 200, "_source": { diff --git a/api/internal/crawl/search_cmds/id.md b/api/internal/crawl/search_cmds/id.md index 66d767eae..f15702065 100644 --- a/api/internal/crawl/search_cmds/id.md +++ b/api/internal/crawl/search_cmds/id.md @@ -1,6 +1,6 @@ Find the document with the given `_id`: ``` -curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d' +curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d' { "query": { "terms": { diff --git a/api/internal/crawl/search_cmds/keyword_search.md b/api/internal/crawl/search_cmds/keyword_search.md index 99a677813..51c79b948 100644 --- a/api/internal/crawl/search_cmds/keyword_search.md +++ b/api/internal/crawl/search_cmds/keyword_search.md @@ -1,7 +1,7 @@ Count the documents in the index whose `repositoryUrl` field starts with `https://github.com/`: ``` -curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d' +curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d' { "query": { "bool": { @@ -17,7 +17,7 @@ curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-T Count the documents in the index whose `repositoryUrl` field does not start with `https://github.com/`: ``` -curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d' +curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d' { "query": { "bool": { @@ -33,7 +33,7 @@ curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-T Search all the documents matching the given `repositoryUrl` and `filePath`, and return a version for each search hit: ``` -curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d' +curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d' { "size": 10000, "version": true, @@ -52,7 +52,7 @@ curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-T Search all the documents whose filePath ends with one of these following three filenames: `kustomization.yaml`, `kustomization.yml`, `kustomization`: ``` -curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d' +curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d' { "query": { "bool": { @@ -68,7 +68,7 @@ curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-T Search all the documents whose filePath does not end with any of these following three filenames: `kustomization.yaml`, `kustomization.yml`, `kustomization`: ``` -curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d' +curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d' { "query": { "bool": { diff --git a/api/internal/crawl/search_cmds/misc.md b/api/internal/crawl/search_cmds/misc.md index d68736ea2..c1fca9368 100644 --- a/api/internal/crawl/search_cmds/misc.md +++ b/api/internal/crawl/search_cmds/misc.md @@ -1,26 +1,26 @@ Check the health status of an ElasticSearch cluster: ``` -curl -X GET "${ElasticSearchURL}:9200/_cat/health?v&pretty" +curl -s -X GET "${ElasticSearchURL}:9200/_cat/health?v&pretty" ``` Check the indices in an ElasticSearch cluster: ``` -curl "${ElasticSearchURL}:9200/_cat/indices?v" +curl -s "${ElasticSearchURL}:9200/_cat/indices?v" ``` Get the mapping of the index: ``` -curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_mapping?pretty" +curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_mapping?pretty" ``` Delete the kustomize index from the ElasticSearch cluster (**Use this command with caution**): ``` -curl -X DELETE "${ElasticSearchURL}:9200/${INDEXNAME}?pretty" +curl -s -X DELETE "${ElasticSearchURL}:9200/${INDEXNAME}?pretty" ``` Add a new field into an existing index. ``` -curl -X PUT "${ElasticSearchURL}:9200/${INDEXNAME}/_mapping/_doc?pretty" -H 'Content-Type: application/json' -d' +curl -s -X PUT "${ElasticSearchURL}:9200/${INDEXNAME}/_mapping/_doc?pretty" -H 'Content-Type: application/json' -d' { "properties": { "fileType": { diff --git a/api/internal/crawl/search_cmds/repositoryUrl.md b/api/internal/crawl/search_cmds/repositoryUrl.md index 37e99d088..a4acb5cb0 100644 --- a/api/internal/crawl/search_cmds/repositoryUrl.md +++ b/api/internal/crawl/search_cmds/repositoryUrl.md @@ -1,6 +1,6 @@ Count distinct values of the `repositoryUrl` field: ``` -curl -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' +curl -s -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' { "aggs" : { "repositoryUrl_count" : { @@ -16,7 +16,7 @@ curl -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'C Count how many Github repositories include kustomization files: ``` -curl -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' +curl -s -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' { "query": { "bool": { @@ -39,7 +39,7 @@ curl -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'C Count distinct values of the `repositoryUrl` field for all the kustomize resource files in the index: ``` -curl -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' +curl -s -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' { "query": { "bool": { @@ -65,7 +65,7 @@ curl -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'C Count distinct values of the `repositoryUrl` field for all the kustomize generator files in the index: ``` -curl -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' +curl -s -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' { "query": { "bool": { @@ -91,7 +91,7 @@ curl -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'C Count distinct values of the `repositoryUrl` field for all the kustomize transformer files in the index: ``` -curl -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' +curl -s -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' { "query": { "bool": { @@ -117,7 +117,7 @@ curl -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'C Count distinct values of the `repositoryUrl` field for all the kustomize resource dirs in the index: ``` -curl -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' +curl -s -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' { "query": { "bool": { @@ -141,7 +141,7 @@ curl -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'C Count distinct values of the `repositoryUrl` field for all the kustomize generator dirs in the index: ``` -curl -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' +curl -s -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' { "query": { "bool": { @@ -165,7 +165,7 @@ curl -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'C Count distinct values of the `repositoryUrl` field for all the kustomize transformer dirs in the index: ``` -curl -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' +curl -s -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' { "query": { "bool": { @@ -191,7 +191,7 @@ List all the github repositories including kustomization files and kustomize res and how many kustomization files and kustomize resource files each github repository includes (the github repository including the most kustomization files is listed first): ``` -curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' +curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' { "aggs" : { "repositoryUrl" : { @@ -207,7 +207,7 @@ curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Co List the top 20 Github repositories including the most amount of kustomization files: ``` -curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' +curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' { "query": { "bool": { @@ -230,7 +230,7 @@ curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Co List the top 20 Github repositories including the most amount of kustomize resource files: ``` -curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' +curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' { "query": { "bool": { diff --git a/api/internal/crawl/search_cmds/snapshot.md b/api/internal/crawl/search_cmds/snapshot.md index fd7cc5301..7bdbba639 100644 --- a/api/internal/crawl/search_cmds/snapshot.md +++ b/api/internal/crawl/search_cmds/snapshot.md @@ -1,29 +1,29 @@ Retrieve information about all registered snapshot repositories: ``` -curl -X GET "${ElasticSearchURL}:9200/_snapshot?pretty" +curl -s -X GET "${ElasticSearchURL}:9200/_snapshot?pretty" ``` Retrieve information about a given snapshot repository, `kustomize-backup`: ``` -curl -X GET "${ElasticSearchURL}:9200/_snapshot/kustomize-backup?pretty" +curl -s -X GET "${ElasticSearchURL}:9200/_snapshot/kustomize-backup?pretty" ``` Verify a snapshot repository, `kustomize-backup`, manually: ``` -curl -X POST "${ElasticSearchURL}:9200/_snapshot/kustomize-backup/_verify?pretty" +curl -s -X POST "${ElasticSearchURL}:9200/_snapshot/kustomize-backup/_verify?pretty" ``` List all the snapshots in a given snapshot repository: ``` -curl -X GET "${ElasticSearchURL}:9200/_cat/snapshots/kustomize-backup?v&s=id&pretty" +curl -s -X GET "${ElasticSearchURL}:9200/_cat/snapshots/kustomize-backup?v&s=id&pretty" ``` Retrieve a summary information about a given snapshot: ``` -curl -X GET "${ElasticSearchURL}:9200/_snapshot/kustomize-backup/kustomize-snapshot?pretty" +curl -s -X GET "${ElasticSearchURL}:9200/_snapshot/kustomize-backup/kustomize-snapshot?pretty" ``` Retrieve a detailed information about a given snapshot: ``` -curl -X GET "${ElasticSearchURL}:9200/_snapshot/kustomize-backup/kustomize-snapshot/_status?pretty" +curl -s -X GET "${ElasticSearchURL}:9200/_snapshot/kustomize-backup/kustomize-snapshot/_status?pretty" ``` diff --git a/api/internal/crawl/search_cmds/stats.md b/api/internal/crawl/search_cmds/stats.md new file mode 100644 index 000000000..3047a97b1 --- /dev/null +++ b/api/internal/crawl/search_cmds/stats.md @@ -0,0 +1,1090 @@ +- [How many kustomization files on Github?](#how-many-kustomization-files-on-github) +- [How many new kustomization files were created each year on Github?](#how-many-new-kustomization-files-were-created-each-year-on-github) +- [How many kustomization files were created each month on Github?](#how-many-kustomization-files-were-created-each-month-on-github) +- [How many Github repositories include kustomization files?](#how-many-github-repositories-include-kustomization-files) +- [How many Github users include kustomization files?](#how-many-github-users-include-kustomization-files) +- [List the top 20 Github users including the most amount of kustomization files](#list-the-top-20-github-users-including-the-most-amount-of-kustomization-files) +- [How many kustomize resource files on Github?](#how-many-kustomize-resource-files-on-github) +- [How many kustomize generator files on Github?](#how-many-kustomize-generator-files-on-github) +- [How many new kustomize resource files were created each year on Github?](#how-many-new-kustomize-resource-files-were-created-each-year-on-github) +- [How many kustomize resource files were created each month on Github?](#how-many-kustomize-resource-files-were-created-each-month-on-github) +- [How many Github repositories include kustomize resource files?](#how-many-github-repositories-include-kustomize-resource-files) +- [How many Github users include kustomize resource files?](#how-many-github-users-include-kustomize-resource-files) +- [List the top 20 Github users including the most amount of kustomize resource files](#list-the-top-20-github-users-including-the-most-amount-of-kustomize-resource-files) +- [How many kustomize generator files were created each month on Github?](#how-many-kustomize-generator-files-were-created-each-month-on-github) +- [How many Github repositories including generator files?](#how-many-github-repositories-including-generator-files) +- [How many Github users including generator files?](#how-many-github-users-including-generator-files) +- [List the top 20 Github users including the most generator files](#list-the-top-20-github-users-including-the-most-generator-files) +- [How many kustomization files have a generators field?](#how-many-kustomization-files-have-a-generators-field) +- [How many kustomization roots are referred to in all the generators fields?](#how-many-kustomization-roots-are-referred-to-in-all-the-generators-fields) +- [How many Github repositories including generator directories?](#how-many-github-repositories-including-generator-directories) +- [How many Github users including generator directories?](#how-many-github-users-including-generator-directories) +- [List the top 20 Github users including the most generator directories](#list-the-top-20-github-users-including-the-most-generator-directories) +- [How many kustomize transformer files on github?](#how-many-kustomize-transformer-files-on-github) +- [How many kustomize transformer files were created each month on Github?](#how-many-kustomize-transformer-files-were-created-each-month-on-github) +- [How many Github repositories including transformer files?](#how-many-github-repositories-including-transformer-files) +- [How many Github users including transformer files?](#how-many-github-users-including-transformer-files) +- [List the top 20 Github users including the most transformer files](#list-the-top-20-github-users-including-the-most-transformer-files) +- [How many kustomization files have a transformers field?](#how-many-kustomization-files-have-a-transformers-field) +- [How many kustomization roots are referred to in all the transformers fields?](#how-many-kustomization-roots-are-referred-to-in-all-the-transformers-fields) +- [How many Github repositories including transformer directories?](#how-many-github-repositories-including-transformer-directories) +- [How many Github users including transformer directories?](#how-many-github-users-including-transformer-directories) +- [List the top 20 Github users including the most transformer directories](#list-the-top-20-github-users-including-the-most-transformer-directories) + + +## How many kustomization files on Github? +``` +curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d' +{ + "query": { + "bool": { + "filter": [ + { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" }} + ] + } + } +} +' | grep "\"hits\" : {" -A1 | grep total | awk '{print $3}' | awk -F, '{print $1}' +``` + + + +## How many new kustomization files were created each year on Github? +Column `Year`: +``` +curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' +{ + "query": { + "bool": { + "filter": [ + { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" }} + ] + } + }, + "aggs" : { + "newFiles_over_time" : { + "date_histogram" : { + "field" : "creationTime", + "interval" : "year" + } + } + } +} +' | grep key_as_string | awk -F\" '{print $4}' | awk -F\- '{print $1}' +``` + +Column `New Kustomization Files`: +``` +curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' +{ + "query": { + "bool": { + "filter": [ + { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" }} + ] + } + }, + "aggs" : { + "newFiles_over_time" : { + "date_histogram" : { + "field" : "creationTime", + "interval" : "year" + } + } + } +} +' | grep doc_count | awk '{print $3}' +``` + +## How many kustomization files were created each month on Github? +Column `Month`: +``` +curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' +{ + "query": { + "bool": { + "filter": [ + { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" }} + ] + } + }, + "aggs" : { + "newFiles_over_time" : { + "date_histogram" : { + "field" : "creationTime", + "interval" : "month" + } + } + } +} +' | grep key_as_string | awk -F\" '{print $4}' | cut -dT -f1 +``` + +Column `New Kustomization Files`: +``` +curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' +{ + "query": { + "bool": { + "filter": [ + { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" }} + ] + } + }, + "aggs" : { + "newFiles_over_time" : { + "date_histogram" : { + "field" : "creationTime", + "interval" : "month" + } + } + } +} +' | grep doc_count | awk '{print $3}' +``` + +## How many Github repositories include kustomization files? +``` +curl -s -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' +{ + "query": { + "bool": { + "filter": [ + { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" }} + ] + } + }, + "aggs" : { + "repositoryUrl_count" : { + "cardinality" : { + "field" : "repositoryUrl", + "precision_threshold": 40000 + } + } + } +} +' | grep "\"value\"" | awk '{print $3}' +``` + +## How many Github users include kustomization files? +``` +curl -s -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' +{ + "query": { + "bool": { + "filter": [ + { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" }} + ] + } + }, + "aggs" : { + "user_count" : { + "cardinality" : { + "field" : "user", + "precision_threshold": 40000 + } + } + } +} +' | grep "\"value\"" | awk '{print $3}' +``` + +## List the top 20 Github users including the most amount of kustomization files +Column `Github User`: +``` +curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' +{ + "query": { + "bool": { + "filter": [ + { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" }} + ] + } + }, + "aggs" : { + "user" : { + "terms" : { + "field" : "user", + "size": 20 + } + } + } +} +' | grep "\"key\"" | awk -F\" '{print $4}' +``` + +Column `Kustomization File Count`: +``` +curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' +{ + "query": { + "bool": { + "filter": [ + { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" }} + ] + } + }, + "aggs" : { + "user" : { + "terms" : { + "field" : "user", + "size": 20 + } + } + } +} +' | grep "\"doc_count\"" | awk '{print $3}' +``` + +## How many kustomize resource files on Github? +``` +curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d' +{ + "query": { + "bool": { + "filter": [ + { "regexp": { "fileType": "resource" }} + ], + "must_not": { + "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" } + } + } + } +} +' | grep "\"hits\" : {" -A1 | grep total | awk '{print $3}' | awk -F, '{print $1}' +``` + +## How many new kustomize resource files were created each year on Github? +Column `Year`: +``` +curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' +{ + "query": { + "bool": { + "must_not": [ + { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" }} + ], + "filter": [ + { "regexp": { "fileType": "resource" }} + ] + } + }, + "aggs" : { + "newFiles_over_time" : { + "date_histogram" : { + "field" : "creationTime", + "interval" : "year" + } + } + } +} +' | grep key_as_string | awk -F\" '{print $4}' | awk -F\- '{print $1}' +``` + +Column `New Kustomize Resource Files`: +``` +curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' +{ + "query": { + "bool": { + "must_not": [ + { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" }} + ], + "filter": [ + { "regexp": { "fileType": "resource" }} + ] + } + }, + "aggs" : { + "newFiles_over_time" : { + "date_histogram" : { + "field" : "creationTime", + "interval" : "year" + } + } + } +} +' | grep "\"doc_count\"" | awk '{print $3}' +``` + +## How many kustomize resource files were created each month on Github? +Column `Month`: +``` +curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' +{ + "query": { + "bool": { + "must_not": [ + { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" }} + ], + "filter": [ + { "regexp": { "fileType": "resource" }} + ] + } + }, + "aggs" : { + "newFiles_over_time" : { + "date_histogram" : { + "field" : "creationTime", + "interval" : "month" + } + } + } +} +' | grep key_as_string | awk -F\" '{print $4}' | awk -FT '{print $1}' +``` + +Column `New Kustomize Resource Files`: +``` +curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' +{ + "query": { + "bool": { + "must_not": [ + { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" }} + ], + "filter": [ + { "regexp": { "fileType": "resource" }} + ] + } + }, + "aggs" : { + "newFiles_over_time" : { + "date_histogram" : { + "field" : "creationTime", + "interval" : "month" + } + } + } +} +' | grep "\"doc_count\"" | awk '{print $3}' +``` + +## How many Github repositories include kustomize resource files? +``` +curl -s -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' +{ + "query": { + "bool": { + "must_not": { + "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" } + }, + "filter": [ + { "regexp": { "fileType": "resource" }} + ] + } + }, + "aggs" : { + "repositoryUrl_count" : { + "cardinality" : { + "field" : "repositoryUrl", + "precision_threshold": 40000 + } + } + } +} +' | grep "\"value\"" | awk '{print $3}' +``` + +## How many Github users include kustomize resource files? +``` +curl -s -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' +{ + "query": { + "bool": { + "must_not": { + "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" } + }, + "filter": [ + { "regexp": { "fileType": "resource" }} + ] + } + }, + "aggs" : { + "user_count" : { + "cardinality" : { + "field" : "user", + "precision_threshold": 40000 + } + } + } +} +' | grep "\"value\"" | awk '{print $3}' +``` + +## List the top 20 Github users including the most amount of kustomize resource files +Column `Github User`: +``` +curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' +{ + "query": { + "bool": { + "must_not": { + "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" } + }, + "filter": [ + { "regexp": { "fileType": "resource" }} + ] + } + }, + "aggs" : { + "user" : { + "terms" : { + "field" : "user", + "size": 20 + } + } + } +} +' | grep "\"key\"" | awk -F\" '{print $4}' +``` + +Column `Resource File Count`: +``` +curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' +{ + "query": { + "bool": { + "must_not": { + "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" } + }, + "filter": [ + { "regexp": { "fileType": "resource" }} + ] + } + }, + "aggs" : { + "user" : { + "terms" : { + "field" : "user", + "size": 20 + } + } + } +} +' | grep "\"doc_count\"" | awk '{print $3}' +``` + +## How many kustomize generator files on Github? +``` +curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d' +{ + "query": { + "bool": { + "filter": [ + { "regexp": { "fileType": "generator" }} + ], + "must_not": { + "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" } + } + } + } +} +' | grep "\"hits\" : {" -A1 | grep total | awk '{print $3}' | awk -F, '{print $1}' +``` + +## How many kustomize generator files were created each month on Github? +Column `Month`: +``` +curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' +{ + "query": { + "bool": { + "must_not": [ + { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" }} + ], + "filter": [ + { "regexp": { "fileType": "generator" }} + ] + } + }, + "aggs" : { + "newFiles_over_time" : { + "date_histogram" : { + "field" : "creationTime", + "interval" : "month" + } + } + } +} +' | grep key_as_string | awk -F\" '{print $4}' | awk -FT '{print $1}' +``` + +Column `New Kustomize Transformer Files`: +``` +curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' +{ + "query": { + "bool": { + "must_not": [ + { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" }} + ], + "filter": [ + { "regexp": { "fileType": "generator" }} + ] + } + }, + "aggs" : { + "newFiles_over_time" : { + "date_histogram" : { + "field" : "creationTime", + "interval" : "month" + } + } + } +} +' | grep "\"doc_count\"" | awk '{print $3}' +``` + +## How many Github repositories including generator files? +``` +curl -s -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' +{ + "query": { + "bool": { + "must_not": { + "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" } + }, + "filter": [ + { "regexp": { "fileType": "generator" }} + ] + } + }, + "aggs" : { + "repositoryUrl_count" : { + "cardinality" : { + "field" : "repositoryUrl", + "precision_threshold": 40000 + } + } + } +} +' | grep "\"value\"" | awk '{print $3}' +``` + +## How many Github users including generator files? +``` +curl -s -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' +{ + "query": { + "bool": { + "must_not": { + "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" } + }, + "filter": [ + { "regexp": { "fileType": "generator" }} + ] + } + }, + "aggs" : { + "user_count" : { + "cardinality" : { + "field" : "user", + "precision_threshold": 40000 + } + } + } +} +' | grep "\"value\"" | awk '{print $3}' +``` + +## List the top 20 Github users including the most generator files +Column `Github User`: +``` +curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' +{ + "query": { + "bool": { + "must_not": { + "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" } + }, + "filter": [ + { "regexp": { "fileType": "generator" }} + ] + } + }, + "aggs" : { + "user" : { + "terms" : { + "field" : "user", + "size": 20 + } + } + } +} +' | grep "\"key\"" | awk -F\" '{print $4}' +``` + +Column `Generator File Count`: +``` +curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' +{ + "query": { + "bool": { + "must_not": { + "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" } + }, + "filter": [ + { "regexp": { "fileType": "generator" }} + ] + } + }, + "aggs" : { + "user" : { + "terms" : { + "field" : "user", + "size": 20 + } + } + } +} +' | grep "\"doc_count\"" | awk '{print $3}' +``` + +## How many kustomization files have a generators field? +``` +curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d' +{ + "size": 10000, + "query": { + "bool": { + "must": { + "match" : { + "identifiers" : { + "query" : "generators" + } + } + }, + "filter": { + "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" } + } + } + } +} +' | grep "\"hits\" : {" -A1 | grep total | awk '{print $3}' | awk -F, '{print $1}' +``` + +## How many kustomization roots are referred to in all the generators fields? +``` +curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d' +{ + "query": { + "bool": { + "filter": [ + { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" }}, + { "regexp": { "fileType": "generator" }} + ] + } + } +} +' | grep "\"hits\" : {" -A1 | grep total | awk '{print $3}' | awk -F, '{print $1}' +``` + +## How many Github repositories including generator directories? +``` +curl -s -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' +{ + "query": { + "bool": { + "filter": [ + { "regexp": { "fileType": "generator" }}, + { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" }} + ] + } + }, + "aggs" : { + "repositoryUrl_count" : { + "cardinality" : { + "field" : "repositoryUrl", + "precision_threshold": 40000 + } + } + } +} +' | grep "\"value\"" | awk '{print $3}' +``` + +## How many Github users including generator directories? +``` +curl -s -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' +{ + "query": { + "bool": { + "filter": [ + { "regexp": { "fileType": "generator" }}, + { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" }} + ] + } + }, + "aggs" : { + "user_count" : { + "cardinality" : { + "field" : "user", + "precision_threshold": 40000 + } + } + } +} +' | grep "\"value\"" | awk '{print $3}' +``` + +## List the top 20 Github users including the most generator directories +Column `Github User`: +``` +curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' +{ + "query": { + "bool": { + "filter": [ + { "regexp": { "fileType": "generator" }}, + { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" }} + ] + } + }, + "aggs" : { + "user" : { + "terms" : { + "field" : "user", + "size": 20 + } + } + } +} +' | grep "\"key\"" | awk -F\" '{print $4}' +``` + +Column `Generator Dir Count`: +``` +curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' +{ + "query": { + "bool": { + "filter": [ + { "regexp": { "fileType": "generator" }}, + { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" }} + ] + } + }, + "aggs" : { + "user" : { + "terms" : { + "field" : "user", + "size": 20 + } + } + } +} +' | grep "\"doc_count\"" | awk '{print $3}' +``` + +## How many kustomize transformer files on github? +``` +curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d' +{ + "query": { + "bool": { + "filter": [ + { "regexp": { "fileType": "transformer" }} + ], + "must_not": { + "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" } + } + } + } +} +' | grep "\"hits\" : {" -A1 | grep total | awk '{print $3}' | awk -F, '{print $1}' +``` + +## How many kustomize transformer files were created each month on Github? +Column `Month`: +``` +curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' +{ + "query": { + "bool": { + "must_not": [ + { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" }} + ], + "filter": [ + { "regexp": { "fileType": "transformer" }} + ] + } + }, + "aggs" : { + "newFiles_over_time" : { + "date_histogram" : { + "field" : "creationTime", + "interval" : "month" + } + } + } +} +' | grep key_as_string | awk -F\" '{print $4}' | awk -FT '{print $1}' +``` + +Column `New Kustomize Transformer Files`: +``` +curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' +{ + "query": { + "bool": { + "must_not": [ + { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" }} + ], + "filter": [ + { "regexp": { "fileType": "transformer" }} + ] + } + }, + "aggs" : { + "newFiles_over_time" : { + "date_histogram" : { + "field" : "creationTime", + "interval" : "month" + } + } + } +} +' | grep "\"doc_count\"" | awk '{print $3}' +``` + +## How many Github repositories including transformer files? +``` +curl -s -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' +{ + "query": { + "bool": { + "must_not": { + "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" } + }, + "filter": [ + { "regexp": { "fileType": "transformer" }} + ] + } + }, + "aggs" : { + "repositoryUrl_count" : { + "cardinality" : { + "field" : "repositoryUrl", + "precision_threshold": 40000 + } + } + } +} +' | grep "\"value\"" | awk '{print $3}' +``` + +## How many Github users including transformer files? +``` +curl -s -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' +{ + "query": { + "bool": { + "must_not": { + "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" } + }, + "filter": [ + { "regexp": { "fileType": "transformer" }} + ] + } + }, + "aggs" : { + "user_count" : { + "cardinality" : { + "field" : "user", + "precision_threshold": 40000 + } + } + } +} +' | grep "\"value\"" | awk '{print $3}' +``` + +## List the top 20 Github users including the most transformer files +Column `Github User`: +``` +curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' +{ + "query": { + "bool": { + "must_not": { + "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" } + }, + "filter": [ + { "regexp": { "fileType": "transformer" }} + ] + } + }, + "aggs" : { + "user" : { + "terms" : { + "field" : "user", + "size": 20 + } + } + } +} +' | grep "\"key\"" | awk -F\" '{print $4}' +``` + +Column `Transformer File Count`: +``` +curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' +{ + "query": { + "bool": { + "must_not": { + "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" } + }, + "filter": [ + { "regexp": { "fileType": "transformer" }} + ] + } + }, + "aggs" : { + "user" : { + "terms" : { + "field" : "user", + "size": 20 + } + } + } +} +' | grep "\"doc_count\"" | awk '{print $3}' +``` + +## How many kustomization files have a transformers field? +``` +curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d' +{ + "size": 10000, + "query": { + "bool": { + "must": { + "match" : { + "identifiers" : { + "query" : "transformers" + } + } + }, + "filter": { + "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" } + } + } + } +} +' | grep "\"hits\" : {" -A1 | grep total | awk '{print $3}' | awk -F, '{print $1}' +``` + +## How many kustomization roots are referred to in all the transformers fields? +``` +curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d' +{ + "query": { + "bool": { + "filter": [ + { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" }}, + { "regexp": { "fileType": "transformer" }} + ] + } + } +} +' | grep "\"hits\" : {" -A1 | grep total | awk '{print $3}' | awk -F, '{print $1}' +``` + +## How many Github repositories including transformer directories? +``` +curl -s -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' +{ + "query": { + "bool": { + "filter": [ + { "regexp": { "fileType": "transformer" }}, + { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" }} + ] + } + }, + "aggs" : { + "repositoryUrl_count" : { + "cardinality" : { + "field" : "repositoryUrl", + "precision_threshold": 40000 + } + } + } +} +' | grep "\"value\"" | awk '{print $3}' +``` + +## How many Github users including transformer directories? +``` +curl -s -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' +{ + "query": { + "bool": { + "filter": [ + { "regexp": { "fileType": "transformer" }}, + { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" }} + ] + } + }, + "aggs" : { + "user_count" : { + "cardinality" : { + "field" : "user", + "precision_threshold": 40000 + } + } + } +} +' | grep "\"value\"" | awk '{print $3}' +``` + +## List the top 20 Github users including the most transformer directories +Column `Github User`: +``` +curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' +{ + "query": { + "bool": { + "filter": [ + { "regexp": { "fileType": "transformer" }}, + { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" }} + ] + } + }, + "aggs" : { + "user" : { + "terms" : { + "field" : "user", + "size": 20 + } + } + } +} +' | grep "\"key\"" | awk -F\" '{print $4}' +``` + +Column `Transformer Dir Count`: +``` +curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' +{ + "query": { + "bool": { + "filter": [ + { "regexp": { "fileType": "transformer" }}, + { "regexp": { "filePath": "(.*/)?kustomization((.yaml)?|(.yml)?)(/)*" }} + ] + } + }, + "aggs" : { + "user" : { + "terms" : { + "field" : "user", + "size": 20 + } + } + } +} +' | grep "\"doc_count\"" | awk '{print $3}' +``` \ No newline at end of file diff --git a/api/internal/crawl/search_cmds/text_search.md b/api/internal/crawl/search_cmds/text_search.md index dcc033fb8..beaa07f20 100644 --- a/api/internal/crawl/search_cmds/text_search.md +++ b/api/internal/crawl/search_cmds/text_search.md @@ -1,6 +1,6 @@ Search for all the kustomize resource files including a Deployment object: ``` -curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d' +curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d' { "query": { "match" : { @@ -16,7 +16,7 @@ curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-T Search for all the kustomize resource files including a Deployment object, but only including the `kinds` field in the result: ``` -curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d' +curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d' { "_source": { "includes": ["kinds"] @@ -35,7 +35,7 @@ curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-T Search for all the kustomize resource files including both a Deployment object and a Service object: ``` -curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d' +curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d' { "query": { "match" : { @@ -52,7 +52,7 @@ curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-T Count the number of documents including Deployment and the number of documents including Service: ``` -curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d' +curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d' { "size": 0, "aggs" : { @@ -71,7 +71,7 @@ curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-T Search for all the kustomization files involving CRDs: ``` -curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d' +curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d' { "size": 10000, "query": { @@ -87,7 +87,7 @@ curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-T Search for all the kustomization files defining configMapGenerator: ``` -curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d' +curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d' { "size": 10000, "query": { @@ -103,7 +103,7 @@ curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-T Search for all the documents having a `kind` field: ``` -curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d' +curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d' { "query": { "bool": { @@ -118,7 +118,7 @@ curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-T Search for all the kuostmization files having a `kind` field: ``` -curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d' +curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d' { "query": { "bool": { @@ -134,7 +134,7 @@ curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-T Search for all the kustomization files defining the `generatorOptions:disableNameSuffixHash` feature: ``` -curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d' +curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d' { "query": { "match" : { diff --git a/api/internal/crawl/search_cmds/transformer.md b/api/internal/crawl/search_cmds/transformer.md index 0b35aeb6e..ca4f3ce4c 100644 --- a/api/internal/crawl/search_cmds/transformer.md +++ b/api/internal/crawl/search_cmds/transformer.md @@ -1,7 +1,7 @@ Find all the trasnformer files whose `kinds` field includes `HelmValues`, and only output certain fields of each document: ``` -curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d' +curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d' { "size": 200, "_source": { diff --git a/api/internal/crawl/search_cmds/user.md b/api/internal/crawl/search_cmds/user.md index 83f4cd2e5..ab2efd2fe 100644 --- a/api/internal/crawl/search_cmds/user.md +++ b/api/internal/crawl/search_cmds/user.md @@ -1,6 +1,6 @@ Find all the documents having the `user` field set: ``` -curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d' +curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d' { "query": { "exists": { @@ -13,7 +13,7 @@ curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-T Find all the documents whose `user` field is not set: ``` -curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d' +curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d' { "size": 10000, "query": { @@ -31,7 +31,7 @@ curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-T Search for all the documents whose `user` field is `kubernetes-sigs`: ``` -curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d' +curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-Type: application/json' -d' { "query": { "bool": { @@ -46,7 +46,7 @@ curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?pretty" -H 'Content-T Count distinct values of the `user` field: ``` -curl -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' +curl -s -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' { "aggs" : { "user_count" : { @@ -62,7 +62,7 @@ curl -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'C List all the values of the `user` field and the frequency of each value: ``` -curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' +curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' { "aggs" : { "user" : { @@ -78,7 +78,7 @@ curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Co Count distinct values of the `user` field for all the kustomization files in the index: ``` -curl -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' +curl -s -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' { "query": { "bool": { @@ -102,7 +102,7 @@ curl -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'C For all the kustomization files in the index, list all the values of the `user` field and the frequency of each value: ``` -curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' +curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' { "query": { "bool": { @@ -125,7 +125,7 @@ curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Co Count distinct values of the `user` field for all the kustomize resource files in the index: ``` -curl -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' +curl -s -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' { "query": { "bool": { @@ -152,7 +152,7 @@ curl -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'C For all the kustomize resource files in the index, list all the values of the `user` field and the frequency of each value: ``` -curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' +curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' { "query": { "bool": { @@ -178,7 +178,7 @@ curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Co Count distinct values of the `user` field for all the kustomize generator files in the index: ``` -curl -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' +curl -s -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' { "query": { "bool": { @@ -205,7 +205,7 @@ curl -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'C For all the kustomize generator files in the index, list all the values of the `user` field and the frequency of each value: ``` -curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' +curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' { "query": { "bool": { @@ -231,7 +231,7 @@ curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Co Count distinct values of the `user` field for all the kustomize transformer files in the index: ``` -curl -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' +curl -s -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' { "query": { "bool": { @@ -258,7 +258,7 @@ curl -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'C For all the kustomize transformer files in the index, list all the values of the `user` field and the frequency of each value: ``` -curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' +curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' { "query": { "bool": { @@ -284,7 +284,7 @@ curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Co Count distinct values of the `user` field for all the kustomize generator dirs in the index: ``` -curl -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' +curl -s -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' { "query": { "bool": { @@ -309,7 +309,7 @@ curl -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'C For all the kustomize generator dirs in the index, list all the values of the `user` field and the frequency of each value: ``` -curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' +curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' { "query": { "bool": { @@ -333,7 +333,7 @@ curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Co Count distinct values of the `user` field for all the kustomize transformer dirs in the index: ``` -curl -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' +curl -s -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' { "query": { "bool": { @@ -358,7 +358,7 @@ curl -X POST "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'C For all the kustomize transformer dirs in the index, list all the values of the `user` field and the frequency of each value: ``` -curl -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' +curl -s -X GET "${ElasticSearchURL}:9200/${INDEXNAME}/_search?size=0&pretty" -H 'Content-Type: application/json' -d' { "query": { "bool": {