Move hack/crawl under api/internal

api/internal/crawl/ui/src/app/app.component.html

@@ -0,0 +1,2 @@
+<h1>{{ title }}</h1>
+<router-outlet></router-outlet>

api/internal/crawl/ui/src/app/app.component.spec.ts

@@ -0,0 +1,31 @@
+import { TestBed, async } from '@angular/core/testing';
+import { AppComponent } from './app.component';
+
+describe('AppComponent', () => {
+  beforeEach(async(() => {
+    TestBed.configureTestingModule({
+      declarations: [
+        AppComponent
+      ],
+    }).compileComponents();
+  }));
+
+  it('should create the app', () => {
+    const fixture = TestBed.createComponent(AppComponent);
+    const app = fixture.debugElement.componentInstance;
+    expect(app).toBeTruthy();
+  });
+
+  it(`should have as title 'kustomize-search'`, () => {
+    const fixture = TestBed.createComponent(AppComponent);
+    const app = fixture.debugElement.componentInstance;
+    expect(app.title).toEqual('kustomize-search');
+  });
+
+  it('should render title in a h1 tag', () => {
+    const fixture = TestBed.createComponent(AppComponent);
+    fixture.detectChanges();
+    const compiled = fixture.debugElement.nativeElement;
+    expect(compiled.querySelector('h1').textContent).toContain('Welcome to kustomize-search!');
+  });
+});

api/internal/crawl/ui/src/app/app.component.ts

@@ -0,0 +1,10 @@
+import { Component } from '@angular/core';
+
+@Component({
+  selector: 'app-root',
+  templateUrl: './app.component.html',
+  styleUrls: ['./app.component.css']
+})
+export class AppComponent {
+  title = 'k8s Search';
+}

api/internal/crawl/ui/src/app/app.module.ts

@@ -0,0 +1,58 @@
+import { BrowserModule } from '@angular/platform-browser';
+import { Routes, RouterModule } from '@angular/router';
+import { NgModule } from '@angular/core';
+import { FormsModule } from '@angular/forms';
+import { HttpClientModule } from '@angular/common/http';
+
+import { MatExpansionModule } from '@angular/material/expansion';
+import { MatInputModule } from '@angular/material/input';
+import { MatListModule } from '@angular/material/list';
+import { MatButtonModule } from '@angular/material/button';
+
+import { AppComponent } from './app.component';
+import { SearchComponent } from './search/search.component';
+import { BrowserAnimationsModule } from '@angular/platform-browser/animations';
+import { HistogramComponent } from './histogram/histogram.component';
+import { TimeseriesComponent } from './timeseries/timeseries.component';
+
+const appRoutes: Routes = [
+  {
+    path: 'search',
+    component: SearchComponent,
+    runGuardsAndResolvers: 'always'
+  },
+  // Always ridirect to the search endpoint for now.
+  {
+    path: '',
+    redirectTo: 'search',
+    pathMatch: 'full',
+  },
+];
+
+@NgModule({
+  declarations: [
+    AppComponent,
+    SearchComponent,
+    HistogramComponent,
+    TimeseriesComponent,
+  ],
+  imports: [
+    BrowserModule,
+    BrowserAnimationsModule,
+    HttpClientModule,
+    MatExpansionModule,
+    MatInputModule,
+    MatListModule,
+    MatButtonModule,
+    FormsModule,
+    RouterModule.forRoot(
+      appRoutes,
+      { onSameUrlNavigation: 'reload', }
+    )
+  ],
+  providers: [
+    {provide: HttpClientModule}
+  ],
+  bootstrap: [AppComponent]
+})
+export class AppModule {}

api/internal/crawl/ui/src/app/documents.ts

@@ -0,0 +1,41 @@
+export interface SearchResults {
+  hits: SearchResults.Hits;
+  aggregations?: SearchResults.Aggregations;
+};
+
+export namespace SearchResults {
+  export class Hits {
+    total: number;
+    hits: SearchResults.InnerHits[];
+  };
+
+  export class InnerHits {
+    id: string;
+    result: SearchResults.Result;
+  };
+
+  export class Result {
+    repositoryUrl: string;
+    filePath: string;
+    defaultBranch: string;
+    document: string;
+    creationTime: Date;
+    values: string;
+    kinds: string;
+  };
+
+  export interface Aggregations {
+    timeseries?: SearchResults.BucketAggregation;
+    kinds?: SearchResults.BucketAggregation;
+  };
+
+  export interface BucketAggregation {
+    otherResults?: number;
+    buckets: SearchResults.Bucket[];
+  };
+
+  export class Bucket {
+    key: string;
+    count: number;
+  };
+};

api/internal/crawl/ui/src/app/histogram/histogram.component.html

@@ -0,0 +1 @@
+<div><canvas id="histogram">{{hist}}</canvas></div>

api/internal/crawl/ui/src/app/histogram/histogram.component.spec.ts

@@ -0,0 +1,25 @@
+import { async, ComponentFixture, TestBed } from '@angular/core/testing';
+
+import { HistogramComponent } from './histogram.component';
+
+describe('HistogramComponent', () => {
+  let component: HistogramComponent;
+  let fixture: ComponentFixture<HistogramComponent>;
+
+  beforeEach(async(() => {
+    TestBed.configureTestingModule({
+      declarations: [ HistogramComponent ]
+    })
+    .compileComponents();
+  }));
+
+  beforeEach(() => {
+    fixture = TestBed.createComponent(HistogramComponent);
+    component = fixture.componentInstance;
+    fixture.detectChanges();
+  });
+
+  it('should create', () => {
+    expect(component).toBeTruthy();
+  });
+});

api/internal/crawl/ui/src/app/histogram/histogram.component.ts

@@ -0,0 +1,61 @@
+import { Chart } from 'chart.js';
+import { SearchResults } from '../documents';
+
+import { Component, OnInit } from '@angular/core';
+import { Subject, Observable } from 'rxjs';
+
+const otherLabel = 'Other Kinds';
+
+// Draws a histogram from SearchResults.BucketAggregation data.
+@Component({
+  selector: 'app-histogram',
+  templateUrl: './histogram.component.html',
+  styleUrls: ['./histogram.component.css']
+})
+export class HistogramComponent implements OnInit {
+  hist;
+
+  constructor() {}
+  ngOnInit() {}
+
+  public update(agg: SearchResults.BucketAggregation): Observable<string> {
+    if (this.hist) {
+      this.hist.destroy();
+    }
+
+    let labels = agg.buckets.map(bucket => bucket.key);
+    let counts = agg.buckets.map(bucket => bucket.count);
+    if (agg.otherResults && agg.otherResults > 0) {
+      labels.push(otherLabel)
+      counts.push(agg.otherResults)
+    }
+
+    let selectedLabel = new Subject<string>();
+
+    this.hist = new Chart('histogram', {
+      type: 'bar',
+      data: {
+        datasets: [ { data: counts } ],
+        labels: labels,
+      },
+      options: {
+        legend: { display: false },
+        'onClick' : function(e, it) {
+          if (!(it && it[0] && it[0]._model && it[0]._model.label)) {
+            return
+          }
+          let label = it[0]._model.label;
+          if (label != otherLabel) {
+            selectedLabel.next(label);
+          }
+        }.bind(selectedLabel),
+        scales: {
+          // no floating point
+          yAxes: [ { ticks: { precision: 0, beginAtZero: true } } ],
+        },
+      },
+    })
+
+    return selectedLabel;
+  }
+}

api/internal/crawl/ui/src/app/search/search.component.css

@@ -0,0 +1,8 @@
+.json_query > * {
+  width: 100%;
+  font-family: monospace;
+}
+
+mat-expansion-panel-header {
+  padding: 20px;
+}

api/internal/crawl/ui/src/app/search/search.component.html

@@ -0,0 +1,36 @@
+<div class="json_query">
+  <mat-form-field>
+    <input matInput (keydown.enter)="search()" placeholder="Search" [(ngModel)]='inputQueryValue'>
+  </mat-form-field>
+</div>
+
+<br>
+<mat-expansion-panel
+  [disabled]="docs.hits.total == 0"
+  [expanded]="docs.hits.hits.length == 0 && docs.hits.total != 0">
+
+  <mat-expansion-panel-header>
+    {{docs.hits.total}} matching config files
+    <div *ngIf="docs.hits.total > 0">, expand to see a breakdown by kind.</div>
+  </mat-expansion-panel-header>
+  <app-histogram></app-histogram>
+  <app-timeseries></app-timeseries>
+</mat-expansion-panel>
+
+<br>
+<mat-expansion-panel class="result" *ngFor="let doc of docs.hits.hits">
+  <mat-expansion-panel-header class="result" [collapsedHeight]="'auto'" [expandedHeight]="'auto'">
+    {{ doc.result.repositoryUrl }}/{{ doc.result.filePath }}
+  </mat-expansion-panel-header>
+  <div mat-line>
+    <h3>File Contents</h3>
+    <pre><code>{{doc.result.document}}</code></pre>
+  </div>
+</mat-expansion-panel>
+
+<button mat-button [disabled]="first()" (click)="prev()">
+  Previous
+</button>
+<button mat-button [disabled]="last()" (click)="next()">
+  Next
+</button>

api/internal/crawl/ui/src/app/search/search.component.spec.ts

@@ -0,0 +1,25 @@
+import { async, ComponentFixture, TestBed } from '@angular/core/testing';
+
+import { SearchComponent } from './search.component';
+
+describe('SearchComponent', () => {
+  let component: SearchComponent;
+  let fixture: ComponentFixture<SearchComponent>;
+
+  beforeEach(async(() => {
+    TestBed.configureTestingModule({
+      declarations: [ SearchComponent ]
+    })
+    .compileComponents();
+  }));
+
+  beforeEach(() => {
+    fixture = TestBed.createComponent(SearchComponent);
+    component = fixture.componentInstance;
+    fixture.detectChanges();
+  });
+
+  it('should create', () => {
+    expect(component).toBeTruthy();
+  });
+});

api/internal/crawl/ui/src/app/search/search.component.ts

@@ -0,0 +1,111 @@
+import { Component, OnInit, NgModule } from '@angular/core';
+import { Router, ActivatedRoute } from '@angular/router';
+import { SearchResults } from '../documents';
+import { HistogramComponent } from '../histogram/histogram.component';
+import { TimeseriesComponent } from '../timeseries/timeseries.component';
+import { SearchService } from './search.service';
+
+const perPage = 10;
+
+@Component({
+  selector: 'app-search',
+  templateUrl: './search.component.html',
+  styleUrls: ['./search.component.css'],
+  providers: [SearchService]
+})
+export class SearchComponent implements OnInit {
+  inputQuery: string[] = [];
+  from: number = 0;
+  disableNav: boolean = false;
+
+  docs: SearchResults = {
+    hits: {
+      total: 0,
+      hits: [],
+    },
+  };
+
+  kindBreakdown = new HistogramComponent();
+  timeseries = new TimeseriesComponent();
+
+  constructor(
+    private searcher : SearchService,
+    private router: Router,
+    private route: ActivatedRoute
+  ) {}
+
+  ngOnInit() {
+    this.route.queryParams.subscribe(params => {
+      if (params.q instanceof Array) {
+        this.inputQuery = params.q || [""]
+      } else {
+        this.inputQuery = [params.q || "" ];
+      }
+
+      this.from = parseInt(params.from) || 0;
+      if (this.from < 0) {
+        this.from = Math.max(this.from, 0);
+        this.searchWithParams();
+      }
+
+      this.searcher.search(params).subscribe(sr => {
+        this.docs = sr;
+        this.kindBreakdown.update(sr.aggregations.kinds).subscribe(selectedKind => {
+          this.addToQuery('kind='+selectedKind)
+          this.search();
+        })
+        this.timeseries.update(sr.aggregations.timeseries);
+      });
+    });
+  }
+
+  public addToQuery(q: string) {
+    for (let v of this.inputQuery) {
+      if (v == q) {
+        return
+      }
+    }
+    this.inputQuery.push(q)
+  }
+
+  search(): void {
+    this.from = 0;
+    this.searchWithParams();
+  }
+
+  searchWithParams(): void {
+    let params = {
+      q: this.inputQuery,
+      from: this.from,
+    }
+    this.router.navigate([], {
+      relativeTo: this.route,
+      queryParams: params,
+    });
+  }
+
+  first(): boolean {
+    return this.from <= 0 || this.disableNav;
+  }
+
+  last(): boolean {
+    return this.from + perPage >= this.docs.hits.total || this.disableNav;
+  }
+
+  next (): void {
+    this.from += perPage;
+    this.searchWithParams();
+  }
+  prev (): void {
+    this.from -= perPage;
+    this.searchWithParams();
+  }
+
+  get inputQueryValue() : string {
+    return this.inputQuery.join(' ')
+  }
+
+  set inputQueryValue(input : string) {
+    this.inputQuery = [input]
+  }
+}

api/internal/crawl/ui/src/app/search/search.service.ts

@@ -0,0 +1,41 @@
+import { SearchResults } from '../documents';
+
+import { Injectable } from '@angular/core';
+import {
+  HttpClient,
+  HttpResponse,
+  HttpParams } from '@angular/common/http';
+import { Params, convertToParamMap } from '@angular/router';
+
+import { Observable } from 'rxjs';
+import { filter, map, catchError } from 'rxjs/operators';
+
+@Injectable()
+export class SearchService {
+  private serviceUrl = "https://www.example.com/";
+
+  constructor(private http: HttpClient) {}
+
+  public search(params: Params): Observable<SearchResults> {
+    let requestParams = new HttpParams();
+    let pmap = convertToParamMap(params);
+    let hasQuery = false;
+
+    for (var k of pmap.keys) {
+      for (var v of pmap.getAll(k)) {
+        if (k == "q" && v != "") {
+          hasQuery = true
+        }
+        requestParams.append(k, v)
+      }
+    }
+
+    let queryUrl = this.serviceUrl
+    if (hasQuery) {
+      queryUrl += "search"
+    } else {
+      queryUrl += "metrics"
+    }
+    return this.http.get<SearchResults>(queryUrl, {params: params});
+  }
+}

api/internal/crawl/ui/src/app/timeseries/timeseries.component.html

@@ -0,0 +1,9 @@
+<!--
+  TODO(someone who knows angular) Canvas is still populated when the chart
+  is empty. I'm not sure how to do this.
+-->
+<div>
+  <canvas max-height="100%" max-width="100%" id="timeseries">
+    {{timeseries}}
+  </canvas>
+</div>

api/internal/crawl/ui/src/app/timeseries/timeseries.component.spec.ts

@@ -0,0 +1,25 @@
+import { async, ComponentFixture, TestBed } from '@angular/core/testing';
+
+import { TimeseriesComponent } from './timeseries.component';
+
+describe('TimeseriesComponent', () => {
+  let component: TimeseriesComponent;
+  let fixture: ComponentFixture<TimeseriesComponent>;
+
+  beforeEach(async(() => {
+    TestBed.configureTestingModule({
+      declarations: [ TimeseriesComponent ]
+    })
+    .compileComponents();
+  }));
+
+  beforeEach(() => {
+    fixture = TestBed.createComponent(TimeseriesComponent);
+    component = fixture.componentInstance;
+    fixture.detectChanges();
+  });
+
+  it('should create', () => {
+    expect(component).toBeTruthy();
+  });
+});

api/internal/crawl/ui/src/app/timeseries/timeseries.component.ts

@@ -0,0 +1,65 @@
+import { Chart } from 'chart.js';
+import { SearchResults } from '../documents';
+
+import { Component, OnInit } from '@angular/core';
+import { Subject, Observable } from 'rxjs';
+
+@Component({
+  selector: 'app-timeseries',
+  templateUrl: './timeseries.component.html',
+  styleUrls: ['./timeseries.component.css']
+})
+export class TimeseriesComponent implements OnInit {
+  timeseries;
+
+  constructor() {}
+
+  ngOnInit() {}
+
+  update(agg: SearchResults.BucketAggregation) {
+    if (this.timeseries) {
+      this.timeseries.destroy();
+    }
+    if (!agg || agg.buckets.length == 0) {
+      this.timeseries = null;
+      return
+    }
+
+    let buckets = agg.buckets
+      .filter(bucket => new Date(bucket.key) > new Date(2017, 1));
+
+    let labels = buckets.map(bucket => new Date(bucket.key))
+    let counts = buckets.map(bucket => bucket.count);
+
+    let sum = 0;
+    for (let i = 0; i < counts.length; i++) {
+      sum += counts[i];
+      counts[i] = sum;
+    }
+
+    this.timeseries = new Chart('timeseries', {
+      type: 'line',
+      data: {
+        datasets: [{
+          label: 'Kustomizations Over time',
+          data: counts,
+          type: 'line',
+          pointRadius: 0,
+          lineTension: 0,
+        }],
+        labels: labels,
+      },
+      options: {
+        scales: {
+          xAxes: [{
+            type: 'time',
+            distribution: 'linear',
+            ticks: {
+              autoSkip: true,
+            },
+          }],
+        }
+      },
+    })
+  }
+}

api/internal/crawl/ui/src/environments/environment.prod.ts

@@ -0,0 +1,3 @@
+export const environment = {
+  production: true
+};

api/internal/crawl/ui/src/environments/environment.ts

@@ -0,0 +1,16 @@
+// This file can be replaced during build by using the `fileReplacements` array.
+// `ng build --prod` replaces `environment.ts` with `environment.prod.ts`.
+// The list of file replacements can be found in `angular.json`.
+
+export const environment = {
+  production: false
+};
+
+/*
+ * For easier debugging in development mode, you can import the following file
+ * to ignore zone related error stack frames such as `zone.run`, `zoneDelegate.invokeTask`.
+ *
+ * This import should be commented out in production mode because it will have a negative impact
+ * on performance if an error is thrown.
+ */
+// import 'zone.js/dist/zone-error';  // Included with Angular CLI.

api/internal/crawl/ui/src/index.html

@@ -0,0 +1,16 @@
+<!doctype html>
+<html lang="en">
+<head>
+  <meta charset="utf-8">
+  <title>Kustomize Search</title>
+  <base href="/">
+
+  <meta name="viewport" content="width=device-width, initial-scale=1">
+  <link rel="icon" type="image/x-icon" href="favicon.ico">
+  <link href="https://fonts.googleapis.com/css?family=Roboto:300,400,500" rel="stylesheet">
+  <link href="https://fonts.googleapis.com/icon?family=Material+Icons" rel="stylesheet">
+</head>
+<body>
+  <app-root></app-root>
+</body>
+</html>

api/internal/crawl/ui/src/main.ts

@@ -0,0 +1,13 @@
+import 'hammerjs';
+import { enableProdMode } from '@angular/core';
+import { platformBrowserDynamic } from '@angular/platform-browser-dynamic';
+
+import { AppModule } from './app/app.module';
+import { environment } from './environments/environment';
+
+if (environment.production) {
+  enableProdMode();
+}
+
+platformBrowserDynamic().bootstrapModule(AppModule)
+  .catch(err => console.error(err));

api/internal/crawl/ui/src/polyfills.ts

@@ -0,0 +1,63 @@
+/**
+ * This file includes polyfills needed by Angular and is loaded before the app.
+ * You can add your own extra polyfills to this file.
+ *
+ * This file is divided into 2 sections:
+ *   1. Browser polyfills. These are applied before loading ZoneJS and are sorted by browsers.
+ *   2. Application imports. Files imported after ZoneJS that should be loaded before your main
+ *      file.
+ *
+ * The current setup is for so-called "evergreen" browsers; the last versions of browsers that
+ * automatically update themselves. This includes Safari >= 10, Chrome >= 55 (including Opera),
+ * Edge >= 13 on the desktop, and iOS 10 and Chrome on mobile.
+ *
+ * Learn more in https://angular.io/guide/browser-support
+ */
+
+/***************************************************************************************************
+ * BROWSER POLYFILLS
+ */
+
+/** IE10 and IE11 requires the following for NgClass support on SVG elements */
+// import 'classlist.js';  // Run `npm install --save classlist.js`.
+
+/**
+ * Web Animations `@angular/platform-browser/animations`
+ * Only required if AnimationBuilder is used within the application and using IE/Edge or Safari.
+ * Standard animation support in Angular DOES NOT require any polyfills (as of Angular 6.0).
+ */
+// import 'web-animations-js';  // Run `npm install --save web-animations-js`.
+
+/**
+ * By default, zone.js will patch all possible macroTask and DomEvents
+ * user can disable parts of macroTask/DomEvents patch by setting following flags
+ * because those flags need to be set before `zone.js` being loaded, and webpack
+ * will put import in the top of bundle, so user need to create a separate file
+ * in this directory (for example: zone-flags.ts), and put the following flags
+ * into that file, and then add the following code before importing zone.js.
+ * import './zone-flags.ts';
+ *
+ * The flags allowed in zone-flags.ts are listed here.
+ *
+ * The following flags will work for all browsers.
+ *
+ * (window as any).__Zone_disable_requestAnimationFrame = true; // disable patch requestAnimationFrame
+ * (window as any).__Zone_disable_on_property = true; // disable patch onProperty such as onclick
+ * (window as any).__zone_symbol__UNPATCHED_EVENTS = ['scroll', 'mousemove']; // disable patch specified eventNames
+ *
+ *  in IE/Edge developer tools, the addEventListener will also be wrapped by zone.js
+ *  with the following flag, it will bypass `zone.js` patch for IE/Edge
+ *
+ *  (window as any).__Zone_enable_cross_context_check = true;
+ *
+ */
+
+/***************************************************************************************************
+ * Zone JS is required by default for Angular itself.
+ */
+import 'zone.js/dist/zone';  // Included with Angular CLI.
+
+
+/***************************************************************************************************
+ * APPLICATION IMPORTS
+ */

api/internal/crawl/ui/src/styles.css

@@ -0,0 +1,36 @@
+/* You can add global styles to this file, and also import other style files */
+@import '~@angular/material/prebuilt-themes/deeppurple-amber.css';
+html, body { height: 100%; }
+
+body {
+  font-family: Roboto, "Helvetica Neue", sans-serif;
+  font-style: normal;
+  max-width: 800px;
+  margin-right: auto;
+  margin-left: auto;
+  padding-left: 16px;
+  padding-right: 16px;
+}
+
+h1, .h1, h2, .h2, h3, .h3 {
+  font-style: normal;
+  font-weight: 400;
+  margin-top: 20px;
+  margin-bottom: 10px;
+}
+
+pre {
+  overflow: auto;
+  padding: 10px;
+  background-color: #f0f0f0;
+  border-radius: 4px;
+  border: 1px solid #ddd;
+}
+
+mat-expansion-panel.result {
+  margin-top: 8px;
+  margin-bottom: 8px;
+}
+.mat-expansion-panel-header {
+  padding: 20px;
+}

api/internal/crawl/ui/src/test.ts

@@ -0,0 +1,20 @@
+// This file is required by karma.conf.js and loads recursively all the .spec and framework files
+
+import 'zone.js/dist/zone-testing';
+import { getTestBed } from '@angular/core/testing';
+import {
+  BrowserDynamicTestingModule,
+  platformBrowserDynamicTesting
+} from '@angular/platform-browser-dynamic/testing';
+
+declare const require: any;
+
+// First, initialize the Angular testing environment.
+getTestBed().initTestEnvironment(
+  BrowserDynamicTestingModule,
+  platformBrowserDynamicTesting()
+);
+// Then we find all the tests.
+const context = require.context('./', true, /\.spec\.ts$/);
+// And load the modules.
+context.keys().map(context);