Move hack/crawl under api/internal

api/internal/crawl/ui/src/app/app.component.html

@@ -0,0 +1,2 @@
+<h1>{{ title }}</h1>
+<router-outlet></router-outlet>

api/internal/crawl/ui/src/app/app.component.spec.ts

@@ -0,0 +1,31 @@
+import { TestBed, async } from '@angular/core/testing';
+import { AppComponent } from './app.component';
+
+describe('AppComponent', () => {
+  beforeEach(async(() => {
+    TestBed.configureTestingModule({
+      declarations: [
+        AppComponent
+      ],
+    }).compileComponents();
+  }));
+
+  it('should create the app', () => {
+    const fixture = TestBed.createComponent(AppComponent);
+    const app = fixture.debugElement.componentInstance;
+    expect(app).toBeTruthy();
+  });
+
+  it(`should have as title 'kustomize-search'`, () => {
+    const fixture = TestBed.createComponent(AppComponent);
+    const app = fixture.debugElement.componentInstance;
+    expect(app.title).toEqual('kustomize-search');
+  });
+
+  it('should render title in a h1 tag', () => {
+    const fixture = TestBed.createComponent(AppComponent);
+    fixture.detectChanges();
+    const compiled = fixture.debugElement.nativeElement;
+    expect(compiled.querySelector('h1').textContent).toContain('Welcome to kustomize-search!');
+  });
+});

api/internal/crawl/ui/src/app/app.component.ts

@@ -0,0 +1,10 @@
+import { Component } from '@angular/core';
+
+@Component({
+  selector: 'app-root',
+  templateUrl: './app.component.html',
+  styleUrls: ['./app.component.css']
+})
+export class AppComponent {
+  title = 'k8s Search';
+}

api/internal/crawl/ui/src/app/app.module.ts

@@ -0,0 +1,58 @@
+import { BrowserModule } from '@angular/platform-browser';
+import { Routes, RouterModule } from '@angular/router';
+import { NgModule } from '@angular/core';
+import { FormsModule } from '@angular/forms';
+import { HttpClientModule } from '@angular/common/http';
+
+import { MatExpansionModule } from '@angular/material/expansion';
+import { MatInputModule } from '@angular/material/input';
+import { MatListModule } from '@angular/material/list';
+import { MatButtonModule } from '@angular/material/button';
+
+import { AppComponent } from './app.component';
+import { SearchComponent } from './search/search.component';
+import { BrowserAnimationsModule } from '@angular/platform-browser/animations';
+import { HistogramComponent } from './histogram/histogram.component';
+import { TimeseriesComponent } from './timeseries/timeseries.component';
+
+const appRoutes: Routes = [
+  {
+    path: 'search',
+    component: SearchComponent,
+    runGuardsAndResolvers: 'always'
+  },
+  // Always ridirect to the search endpoint for now.
+  {
+    path: '',
+    redirectTo: 'search',
+    pathMatch: 'full',
+  },
+];
+
+@NgModule({
+  declarations: [
+    AppComponent,
+    SearchComponent,
+    HistogramComponent,
+    TimeseriesComponent,
+  ],
+  imports: [
+    BrowserModule,
+    BrowserAnimationsModule,
+    HttpClientModule,
+    MatExpansionModule,
+    MatInputModule,
+    MatListModule,
+    MatButtonModule,
+    FormsModule,
+    RouterModule.forRoot(
+      appRoutes,
+      { onSameUrlNavigation: 'reload', }
+    )
+  ],
+  providers: [
+    {provide: HttpClientModule}
+  ],
+  bootstrap: [AppComponent]
+})
+export class AppModule {}

api/internal/crawl/ui/src/app/documents.ts

@@ -0,0 +1,41 @@
+export interface SearchResults {
+  hits: SearchResults.Hits;
+  aggregations?: SearchResults.Aggregations;
+};
+
+export namespace SearchResults {
+  export class Hits {
+    total: number;
+    hits: SearchResults.InnerHits[];
+  };
+
+  export class InnerHits {
+    id: string;
+    result: SearchResults.Result;
+  };
+
+  export class Result {
+    repositoryUrl: string;
+    filePath: string;
+    defaultBranch: string;
+    document: string;
+    creationTime: Date;
+    values: string;
+    kinds: string;
+  };
+
+  export interface Aggregations {
+    timeseries?: SearchResults.BucketAggregation;
+    kinds?: SearchResults.BucketAggregation;
+  };
+
+  export interface BucketAggregation {
+    otherResults?: number;
+    buckets: SearchResults.Bucket[];
+  };
+
+  export class Bucket {
+    key: string;
+    count: number;
+  };
+};

api/internal/crawl/ui/src/app/histogram/histogram.component.html

@@ -0,0 +1 @@
+<div><canvas id="histogram">{{hist}}</canvas></div>

api/internal/crawl/ui/src/app/histogram/histogram.component.spec.ts

@@ -0,0 +1,25 @@
+import { async, ComponentFixture, TestBed } from '@angular/core/testing';
+
+import { HistogramComponent } from './histogram.component';
+
+describe('HistogramComponent', () => {
+  let component: HistogramComponent;
+  let fixture: ComponentFixture<HistogramComponent>;
+
+  beforeEach(async(() => {
+    TestBed.configureTestingModule({
+      declarations: [ HistogramComponent ]
+    })
+    .compileComponents();
+  }));
+
+  beforeEach(() => {
+    fixture = TestBed.createComponent(HistogramComponent);
+    component = fixture.componentInstance;
+    fixture.detectChanges();
+  });
+
+  it('should create', () => {
+    expect(component).toBeTruthy();
+  });
+});

api/internal/crawl/ui/src/app/histogram/histogram.component.ts

@@ -0,0 +1,61 @@
+import { Chart } from 'chart.js';
+import { SearchResults } from '../documents';
+
+import { Component, OnInit } from '@angular/core';
+import { Subject, Observable } from 'rxjs';
+
+const otherLabel = 'Other Kinds';
+
+// Draws a histogram from SearchResults.BucketAggregation data.
+@Component({
+  selector: 'app-histogram',
+  templateUrl: './histogram.component.html',
+  styleUrls: ['./histogram.component.css']
+})
+export class HistogramComponent implements OnInit {
+  hist;
+
+  constructor() {}
+  ngOnInit() {}
+
+  public update(agg: SearchResults.BucketAggregation): Observable<string> {
+    if (this.hist) {
+      this.hist.destroy();
+    }
+
+    let labels = agg.buckets.map(bucket => bucket.key);
+    let counts = agg.buckets.map(bucket => bucket.count);
+    if (agg.otherResults && agg.otherResults > 0) {
+      labels.push(otherLabel)
+      counts.push(agg.otherResults)
+    }
+
+    let selectedLabel = new Subject<string>();
+
+    this.hist = new Chart('histogram', {
+      type: 'bar',
+      data: {
+        datasets: [ { data: counts } ],
+        labels: labels,
+      },
+      options: {
+        legend: { display: false },
+        'onClick' : function(e, it) {
+          if (!(it && it[0] && it[0]._model && it[0]._model.label)) {
+            return
+          }
+          let label = it[0]._model.label;
+          if (label != otherLabel) {
+            selectedLabel.next(label);
+          }
+        }.bind(selectedLabel),
+        scales: {
+          // no floating point
+          yAxes: [ { ticks: { precision: 0, beginAtZero: true } } ],
+        },
+      },
+    })
+
+    return selectedLabel;
+  }
+}

api/internal/crawl/ui/src/app/search/search.component.css

@@ -0,0 +1,8 @@
+.json_query > * {
+  width: 100%;
+  font-family: monospace;
+}
+
+mat-expansion-panel-header {
+  padding: 20px;
+}

api/internal/crawl/ui/src/app/search/search.component.html

@@ -0,0 +1,36 @@
+<div class="json_query">
+  <mat-form-field>
+    <input matInput (keydown.enter)="search()" placeholder="Search" [(ngModel)]='inputQueryValue'>
+  </mat-form-field>
+</div>
+
+<br>
+<mat-expansion-panel
+  [disabled]="docs.hits.total == 0"
+  [expanded]="docs.hits.hits.length == 0 && docs.hits.total != 0">
+
+  <mat-expansion-panel-header>
+    {{docs.hits.total}} matching config files
+    <div *ngIf="docs.hits.total > 0">, expand to see a breakdown by kind.</div>
+  </mat-expansion-panel-header>
+  <app-histogram></app-histogram>
+  <app-timeseries></app-timeseries>
+</mat-expansion-panel>
+
+<br>
+<mat-expansion-panel class="result" *ngFor="let doc of docs.hits.hits">
+  <mat-expansion-panel-header class="result" [collapsedHeight]="'auto'" [expandedHeight]="'auto'">
+    {{ doc.result.repositoryUrl }}/{{ doc.result.filePath }}
+  </mat-expansion-panel-header>
+  <div mat-line>
+    <h3>File Contents</h3>
+    <pre><code>{{doc.result.document}}</code></pre>
+  </div>
+</mat-expansion-panel>
+
+<button mat-button [disabled]="first()" (click)="prev()">
+  Previous
+</button>
+<button mat-button [disabled]="last()" (click)="next()">
+  Next
+</button>

api/internal/crawl/ui/src/app/search/search.component.spec.ts

@@ -0,0 +1,25 @@
+import { async, ComponentFixture, TestBed } from '@angular/core/testing';
+
+import { SearchComponent } from './search.component';
+
+describe('SearchComponent', () => {
+  let component: SearchComponent;
+  let fixture: ComponentFixture<SearchComponent>;
+
+  beforeEach(async(() => {
+    TestBed.configureTestingModule({
+      declarations: [ SearchComponent ]
+    })
+    .compileComponents();
+  }));
+
+  beforeEach(() => {
+    fixture = TestBed.createComponent(SearchComponent);
+    component = fixture.componentInstance;
+    fixture.detectChanges();
+  });
+
+  it('should create', () => {
+    expect(component).toBeTruthy();
+  });
+});

api/internal/crawl/ui/src/app/search/search.component.ts

@@ -0,0 +1,111 @@
+import { Component, OnInit, NgModule } from '@angular/core';
+import { Router, ActivatedRoute } from '@angular/router';
+import { SearchResults } from '../documents';
+import { HistogramComponent } from '../histogram/histogram.component';
+import { TimeseriesComponent } from '../timeseries/timeseries.component';
+import { SearchService } from './search.service';
+
+const perPage = 10;
+
+@Component({
+  selector: 'app-search',
+  templateUrl: './search.component.html',
+  styleUrls: ['./search.component.css'],
+  providers: [SearchService]
+})
+export class SearchComponent implements OnInit {
+  inputQuery: string[] = [];
+  from: number = 0;
+  disableNav: boolean = false;
+
+  docs: SearchResults = {
+    hits: {
+      total: 0,
+      hits: [],
+    },
+  };
+
+  kindBreakdown = new HistogramComponent();
+  timeseries = new TimeseriesComponent();
+
+  constructor(
+    private searcher : SearchService,
+    private router: Router,
+    private route: ActivatedRoute
+  ) {}
+
+  ngOnInit() {
+    this.route.queryParams.subscribe(params => {
+      if (params.q instanceof Array) {
+        this.inputQuery = params.q || [""]
+      } else {
+        this.inputQuery = [params.q || "" ];
+      }
+
+      this.from = parseInt(params.from) || 0;
+      if (this.from < 0) {
+        this.from = Math.max(this.from, 0);
+        this.searchWithParams();
+      }
+
+      this.searcher.search(params).subscribe(sr => {
+        this.docs = sr;
+        this.kindBreakdown.update(sr.aggregations.kinds).subscribe(selectedKind => {
+          this.addToQuery('kind='+selectedKind)
+          this.search();
+        })
+        this.timeseries.update(sr.aggregations.timeseries);
+      });
+    });
+  }
+
+  public addToQuery(q: string) {
+    for (let v of this.inputQuery) {
+      if (v == q) {
+        return
+      }
+    }
+    this.inputQuery.push(q)
+  }
+
+  search(): void {
+    this.from = 0;
+    this.searchWithParams();
+  }
+
+  searchWithParams(): void {
+    let params = {
+      q: this.inputQuery,
+      from: this.from,
+    }
+    this.router.navigate([], {
+      relativeTo: this.route,
+      queryParams: params,
+    });
+  }
+
+  first(): boolean {
+    return this.from <= 0 || this.disableNav;
+  }
+
+  last(): boolean {
+    return this.from + perPage >= this.docs.hits.total || this.disableNav;
+  }
+
+  next (): void {
+    this.from += perPage;
+    this.searchWithParams();
+  }
+  prev (): void {
+    this.from -= perPage;
+    this.searchWithParams();
+  }
+
+  get inputQueryValue() : string {
+    return this.inputQuery.join(' ')
+  }
+
+  set inputQueryValue(input : string) {
+    this.inputQuery = [input]
+  }
+}

api/internal/crawl/ui/src/app/search/search.service.ts

@@ -0,0 +1,41 @@
+import { SearchResults } from '../documents';
+
+import { Injectable } from '@angular/core';
+import {
+  HttpClient,
+  HttpResponse,
+  HttpParams } from '@angular/common/http';
+import { Params, convertToParamMap } from '@angular/router';
+
+import { Observable } from 'rxjs';
+import { filter, map, catchError } from 'rxjs/operators';
+
+@Injectable()
+export class SearchService {
+  private serviceUrl = "https://www.example.com/";
+
+  constructor(private http: HttpClient) {}
+
+  public search(params: Params): Observable<SearchResults> {
+    let requestParams = new HttpParams();
+    let pmap = convertToParamMap(params);
+    let hasQuery = false;
+
+    for (var k of pmap.keys) {
+      for (var v of pmap.getAll(k)) {
+        if (k == "q" && v != "") {
+          hasQuery = true
+        }
+        requestParams.append(k, v)
+      }
+    }
+
+    let queryUrl = this.serviceUrl
+    if (hasQuery) {
+      queryUrl += "search"
+    } else {
+      queryUrl += "metrics"
+    }
+    return this.http.get<SearchResults>(queryUrl, {params: params});
+  }
+}

api/internal/crawl/ui/src/app/timeseries/timeseries.component.html

@@ -0,0 +1,9 @@
+<!--
+  TODO(someone who knows angular) Canvas is still populated when the chart
+  is empty. I'm not sure how to do this.
+-->
+<div>
+  <canvas max-height="100%" max-width="100%" id="timeseries">
+    {{timeseries}}
+  </canvas>
+</div>

api/internal/crawl/ui/src/app/timeseries/timeseries.component.spec.ts

@@ -0,0 +1,25 @@
+import { async, ComponentFixture, TestBed } from '@angular/core/testing';
+
+import { TimeseriesComponent } from './timeseries.component';
+
+describe('TimeseriesComponent', () => {
+  let component: TimeseriesComponent;
+  let fixture: ComponentFixture<TimeseriesComponent>;
+
+  beforeEach(async(() => {
+    TestBed.configureTestingModule({
+      declarations: [ TimeseriesComponent ]
+    })
+    .compileComponents();
+  }));
+
+  beforeEach(() => {
+    fixture = TestBed.createComponent(TimeseriesComponent);
+    component = fixture.componentInstance;
+    fixture.detectChanges();
+  });
+
+  it('should create', () => {
+    expect(component).toBeTruthy();
+  });
+});

api/internal/crawl/ui/src/app/timeseries/timeseries.component.ts

@@ -0,0 +1,65 @@
+import { Chart } from 'chart.js';
+import { SearchResults } from '../documents';
+
+import { Component, OnInit } from '@angular/core';
+import { Subject, Observable } from 'rxjs';
+
+@Component({
+  selector: 'app-timeseries',
+  templateUrl: './timeseries.component.html',
+  styleUrls: ['./timeseries.component.css']
+})
+export class TimeseriesComponent implements OnInit {
+  timeseries;
+
+  constructor() {}
+
+  ngOnInit() {}
+
+  update(agg: SearchResults.BucketAggregation) {
+    if (this.timeseries) {
+      this.timeseries.destroy();
+    }
+    if (!agg || agg.buckets.length == 0) {
+      this.timeseries = null;
+      return
+    }
+
+    let buckets = agg.buckets
+      .filter(bucket => new Date(bucket.key) > new Date(2017, 1));
+
+    let labels = buckets.map(bucket => new Date(bucket.key))
+    let counts = buckets.map(bucket => bucket.count);
+
+    let sum = 0;
+    for (let i = 0; i < counts.length; i++) {
+      sum += counts[i];
+      counts[i] = sum;
+    }
+
+    this.timeseries = new Chart('timeseries', {
+      type: 'line',
+      data: {
+        datasets: [{
+          label: 'Kustomizations Over time',
+          data: counts,
+          type: 'line',
+          pointRadius: 0,
+          lineTension: 0,
+        }],
+        labels: labels,
+      },
+      options: {
+        scales: {
+          xAxes: [{
+            type: 'time',
+            distribution: 'linear',
+            ticks: {
+              autoSkip: true,
+            },
+          }],
+        }
+      },
+    })
+  }
+}