From 3f1b2bb744bc70d291e2a278e4b67ca23ba768ae Mon Sep 17 00:00:00 2001 From: Damien Robichaud Date: Fri, 23 Aug 2019 12:52:23 -0700 Subject: [PATCH] Add configs --- .../config/crawler/base/kustomization.yaml | 13 +++++++ .../tools/config/crawler/cronjob/cronjob.yaml | 30 +++++++++++++++ .../config/crawler/cronjob/kustomization.yaml | 3 ++ internal/tools/config/crawler/job/job.yaml | 32 ++++++++++++++++ .../config/crawler/job/kustomization.yaml | 3 ++ .../document_keystore/kustomization.yaml | 7 ++++ .../config/redis/document_keystore/redis.yaml | 37 +++++++++++++++++++ .../redis/document_keystore/service.yaml | 10 +++++ .../redis/http_cache/kustomization.yaml | 7 ++++ .../tools/config/redis/http_cache/redis.yaml | 16 ++++++++ .../config/redis/http_cache/service.yaml | 10 +++++ 11 files changed, 168 insertions(+) create mode 100644 internal/tools/config/crawler/base/kustomization.yaml create mode 100644 internal/tools/config/crawler/cronjob/cronjob.yaml create mode 100644 internal/tools/config/crawler/cronjob/kustomization.yaml create mode 100644 internal/tools/config/crawler/job/job.yaml create mode 100644 internal/tools/config/crawler/job/kustomization.yaml create mode 100644 internal/tools/config/redis/document_keystore/kustomization.yaml create mode 100644 internal/tools/config/redis/document_keystore/redis.yaml create mode 100644 internal/tools/config/redis/document_keystore/service.yaml create mode 100644 internal/tools/config/redis/http_cache/kustomization.yaml create mode 100644 internal/tools/config/redis/http_cache/redis.yaml create mode 100644 internal/tools/config/redis/http_cache/service.yaml diff --git a/internal/tools/config/crawler/base/kustomization.yaml b/internal/tools/config/crawler/base/kustomization.yaml new file mode 100644 index 000000000..f7cee507a --- /dev/null +++ b/internal/tools/config/crawler/base/kustomization.yaml @@ -0,0 +1,13 @@ +resources: +- ../../base + +configmapGenerator: +- name: crawler-http-cache + literals: + - redis-cache-url="redis://redis-http-cache:6379" + + +secretGenerator: +- name: github-access-token + files: + - token=github_api_secret.txt diff --git a/internal/tools/config/crawler/cronjob/cronjob.yaml b/internal/tools/config/crawler/cronjob/cronjob.yaml new file mode 100644 index 000000000..af4308d7b --- /dev/null +++ b/internal/tools/config/crawler/cronjob/cronjob.yaml @@ -0,0 +1,30 @@ +apiVersion: batch/v1beta1 +kind: CronJob +metadata: + name: crawler +spec: + schedule: "5 0 * * */1" + jobTemplate: + spec: + template: + spec: + restartPolicy: OnFailure + containers: + - name: crawler + image: gcr.io/kustomize-search/crawler:latest + env: + - name: GITHUB_ACCESS_TOKEN + valueFrom: + secretKeyRef: + name: github-access-token + key: token + - name: ELASTICSEARCH_URL + valueFrom: + configMapKeyRef: + name: elasticsearch-config + key: es-url + - name: REDIS_URL + valueFrom: + configMapKeyRef: + name: crawler-http-cache + key: redis-cache-url diff --git a/internal/tools/config/crawler/cronjob/kustomization.yaml b/internal/tools/config/crawler/cronjob/kustomization.yaml new file mode 100644 index 000000000..71007fdcc --- /dev/null +++ b/internal/tools/config/crawler/cronjob/kustomization.yaml @@ -0,0 +1,3 @@ +resources: +- ../base +- cronjob.yaml diff --git a/internal/tools/config/crawler/job/job.yaml b/internal/tools/config/crawler/job/job.yaml new file mode 100644 index 000000000..8c319ded4 --- /dev/null +++ b/internal/tools/config/crawler/job/job.yaml @@ -0,0 +1,32 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: crawler +spec: + template: + spec: + restartPolicy: OnFailure + containers: + - name: crawler + image: gcr.io/kustomize-search/crawler:latest + env: + - name: GITHUB_ACCESS_TOKEN + valueFrom: + secretKeyRef: + name: github-access-token + key: token + - name: ELASTICSEARCH_URL + valueFrom: + configMapKeyRef: + name: elasticsearch-config + key: es-url + - name: REDIS_CACHE_URL + valueFrom: + configMapKeyRef: + name: crawler-http-cache + key: redis-cache-url + - name: REDIS_KEY_URL + valueFrom: + configMapKeyRef: + name: redis-keystore + key: keystore-url diff --git a/internal/tools/config/crawler/job/kustomization.yaml b/internal/tools/config/crawler/job/kustomization.yaml new file mode 100644 index 000000000..a2933dd58 --- /dev/null +++ b/internal/tools/config/crawler/job/kustomization.yaml @@ -0,0 +1,3 @@ +resources: +- ../base +- job.yaml diff --git a/internal/tools/config/redis/document_keystore/kustomization.yaml b/internal/tools/config/redis/document_keystore/kustomization.yaml new file mode 100644 index 000000000..32fceef56 --- /dev/null +++ b/internal/tools/config/redis/document_keystore/kustomization.yaml @@ -0,0 +1,7 @@ +resources: +- redis.yaml +- service.yaml + +commonLabels: + app: redis + tier: document-keystore diff --git a/internal/tools/config/redis/document_keystore/redis.yaml b/internal/tools/config/redis/document_keystore/redis.yaml new file mode 100644 index 000000000..c896fa247 --- /dev/null +++ b/internal/tools/config/redis/document_keystore/redis.yaml @@ -0,0 +1,37 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: redis-docs-keystore +spec: + serviceName: "redis-docs-keystore" + template: + spec: + containers: + - name: redis + image: redis:5-alpine + imagePullPolicy: Always + args: + - "--save" + - "900" + - "1" + - "--save" + - "30" + - "100" + - "--appendonly" + - "yes" + ports: + - name: redis-docs-port + containerPort: 6379 + volumeMounts: + - mountPath: /data + name: redis-docs-keystore-data + restartPolicy: Always + volumeClaimTemplates: + - metadata: + name: redis-docs-keystore-data + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 4Gi diff --git a/internal/tools/config/redis/document_keystore/service.yaml b/internal/tools/config/redis/document_keystore/service.yaml new file mode 100644 index 000000000..f5e1f9b5d --- /dev/null +++ b/internal/tools/config/redis/document_keystore/service.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: Service +metadata: + name: redis-docs-keystore +spec: + clusterIP: None + ports: + - protocol: "TCP" + port: 6379 + targetPort: redis-docs-port diff --git a/internal/tools/config/redis/http_cache/kustomization.yaml b/internal/tools/config/redis/http_cache/kustomization.yaml new file mode 100644 index 000000000..92fca06a7 --- /dev/null +++ b/internal/tools/config/redis/http_cache/kustomization.yaml @@ -0,0 +1,7 @@ +resources: +- redis.yaml +- service.yaml + +commonLabels: + app: redis + tier: http-cache diff --git a/internal/tools/config/redis/http_cache/redis.yaml b/internal/tools/config/redis/http_cache/redis.yaml new file mode 100644 index 000000000..ee840804a --- /dev/null +++ b/internal/tools/config/redis/http_cache/redis.yaml @@ -0,0 +1,16 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: redis-http-cache +spec: + template: + spec: + containers: + - name: redis + image: redis:5-alpine + imagePullPolicy: Always + # see redis.io/topics/lru-cache for other policy options. + args: ["--maxmemory", "1gb", "--maxmemory-policy", "allkeys-lru"] + ports: + - name: http-cache-port + containerPort: 6379 diff --git a/internal/tools/config/redis/http_cache/service.yaml b/internal/tools/config/redis/http_cache/service.yaml new file mode 100644 index 000000000..0e8050abc --- /dev/null +++ b/internal/tools/config/redis/http_cache/service.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: Service +metadata: + name: redis-http-cache +spec: + clusterIP: None + ports: + - protocol: "TCP" + port: 6379 + targetPort: http-cache-port