diff --git a/api/internal/crawl/cmd/log-parser/README.md b/api/internal/crawl/cmd/log-parser/README.md new file mode 100644 index 000000000..8142a155a --- /dev/null +++ b/api/internal/crawl/cmd/log-parser/README.md @@ -0,0 +1,8 @@ +This binary takes as its input a json file including GKE logs (which can be +[exported](https://cloud.google.com/logging/docs/export/configure_export_v2) into +[Cloud Storage](https://cloud.google.com/storage/docs/)), +and extracts the `textPayload` field of each log entry. + +Here is an log entry example: + +{"insertId":"1sxuh4jg5lw6w10","labels":{"compute.googleapis.com/resource_name":"gke-crawler2-default-pool-5e55ea05-gzgv","container.googleapis.com/namespace_name":"default","container.googleapis.com/pod_name":"kustomize-stats-5bczg","container.googleapis.com/stream":"stdout"},"logName":"projects/haiyanmeng-gke-dev/logs/kustomize-stats","receiveTimestamp":"2020-01-06T23:33:07.012831742Z","resource":{"labels":{"cluster_name":"crawler2","container_name":"kustomize-stats","instance_id":"8183086081854184383","namespace_id":"default","pod_id":"kustomize-stats-5bczg","project_id":"haiyanmeng-gke-dev","zone":"us-central1-a"},"type":"container"},"severity":"INFO","textPayload":"The kustomize index already exists\n","timestamp":"2020-01-06T23:32:46.628930547Z"} diff --git a/api/internal/crawl/cmd/log-parser/main.go b/api/internal/crawl/cmd/log-parser/main.go new file mode 100644 index 000000000..878650f0a --- /dev/null +++ b/api/internal/crawl/cmd/log-parser/main.go @@ -0,0 +1,48 @@ +package main + +import ( + "bufio" + "encoding/json" + "fmt" + "log" + "os" +) + +func main() { + if len(os.Args) != 2 { + log.Fatalf("The usage of the command is: \n\t%s ", os.Args[0]) + } + + file, err := os.Open(os.Args[1]) + if err != nil { + log.Fatal(err) + } + + closeFile := func(file *os.File) { + if err := file.Close(); err != nil { + log.Fatal(err) + } + } + defer closeFile(file) + + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := scanner.Text() + + var entry interface{} + if err := json.Unmarshal([]byte(line), &entry); err != nil { + log.Printf("failed to unmarshal a log entry: %s\n", line) + } + + m := entry.(map[string]interface{}) + if payload, ok := m["textPayload"]; ok { + fmt.Printf("%s", payload) + } else { + log.Printf("the log entry does not have the `textPayload` field: %s\n", line) + } + } + + if err := scanner.Err(); err != nil { + log.Fatal(err) + } +} \ No newline at end of file