From 54e0d7c69ee0960b022e496f9ae28536812146f3 Mon Sep 17 00:00:00 2001 From: Pablo Date: Tue, 13 Jul 2021 14:38:14 -0600 Subject: [PATCH 1/3] feat: add node selectors and namespace support To allow deployments to be on specific nodes and with specific namespaces given on the helm command --- templates/default/deployment.yaml | 8 ++++++-- templates/default/service.yaml | 2 +- templates/frontend/deployment.yaml | 7 ++++++- templates/frontend/service.yaml | 2 +- templates/hooks/database_pre_install.yaml | 8 +++++++- templates/hooks/database_pre_upgrade.yaml | 7 ++++++- templates/hooks/secret_gcr_db_migrations.yaml | 2 +- templates/ingress.yaml | 1 + templates/postgres/deployment.yaml | 8 ++++++-- templates/postgres/service.yaml | 1 + templates/secret-gcr.yaml | 2 +- templates/walrus/deployment.yaml | 8 ++++++-- templates/walrus/service.yaml | 2 +- values.yaml | 2 ++ 14 files changed, 46 insertions(+), 14 deletions(-) diff --git a/templates/default/deployment.yaml b/templates/default/deployment.yaml index 1906c6a..1f57cb7 100644 --- a/templates/default/deployment.yaml +++ b/templates/default/deployment.yaml @@ -4,7 +4,7 @@ metadata: labels: app: diffgram-default name: diffgram-default - namespace: default + namespace: {{ .Release.Namespace }} spec: replicas: {{ .Values.defaultService.numReplicas }} selector: @@ -15,6 +15,10 @@ spec: labels: app: diffgram-default spec: + {{ if .Values.nodeGroupLabel }} + nodeSelector: + poolName: {{ .Values.nodeGroupLabel }} + {{ end }} {{ if eq .Values.diffgramEdition "enterprise"}} imagePullSecrets: - name: diffgramsecret @@ -30,7 +34,7 @@ spec: {{ end }} initContainers: - name: check-db-ready - image: postgres:9.6.5 + image: postgres:11 {{ if eq .Values.dbSettings.dbProvider "local"}} command: ['sh', '-c', 'until pg_isready -h diffgram-postgres -p 5432; diff --git a/templates/default/service.yaml b/templates/default/service.yaml index b62b0d1..fc22f23 100644 --- a/templates/default/service.yaml +++ b/templates/default/service.yaml @@ -4,7 +4,7 @@ metadata: labels: app: diffgram-default name: diffgram-default - namespace: default + namespace: {{ .Release.Namespace }} spec: ports: - port: 8080 diff --git a/templates/frontend/deployment.yaml b/templates/frontend/deployment.yaml index 92ac2c5..615cae6 100644 --- a/templates/frontend/deployment.yaml +++ b/templates/frontend/deployment.yaml @@ -4,9 +4,10 @@ metadata: labels: app: frontend name: frontend - namespace: default + namespace: {{ .Release.Namespace }} spec: replicas: {{ .Values.frontendService.numReplicas }} + selector: matchLabels: app: frontend @@ -15,6 +16,10 @@ spec: labels: app: frontend spec: + {{ if .Values.nodeGroupLabel }} + nodeSelector: + poolName: {{ .Values.nodeGroupLabel }} + {{ end }} {{ if eq .Values.diffgramEdition "enterprise"}} imagePullSecrets: - name: diffgramsecret diff --git a/templates/frontend/service.yaml b/templates/frontend/service.yaml index 3031f45..6528548 100644 --- a/templates/frontend/service.yaml +++ b/templates/frontend/service.yaml @@ -4,7 +4,7 @@ metadata: labels: app: frontend name: frontend - namespace: default + namespace: {{ .Release.Namespace }} spec: ports: - port: 8080 diff --git a/templates/hooks/database_pre_install.yaml b/templates/hooks/database_pre_install.yaml index 066262e..88321d2 100644 --- a/templates/hooks/database_pre_install.yaml +++ b/templates/hooks/database_pre_install.yaml @@ -22,6 +22,10 @@ spec: app.kubernetes.io/instance: {{ .Release.Name | quote }} helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" spec: + {{ if .Values.nodeGroupLabel }} + nodeSelector: + poolName: {{ .Values.nodeGroupLabel }} + {{ end }} restartPolicy: Never {{ if eq .Values.diffgramEdition "enterprise"}} imagePullSecrets: @@ -36,9 +40,11 @@ spec: - key: sa_json path: sa_credentials.json {{ end }} + {{ if eq .Values.dbSettings.dbProvider "local"}} - name: postgres-storage persistentVolumeClaim: claimName: postgres-pv-claim + {{ end }} containers: {{ if eq .Values.diffgramEdition "enterprise"}} - image: gcr.io/diffgram-enterprise/default:{{ .Values.diffgramVersion }} @@ -50,7 +56,7 @@ spec: name: pre-upgrade-alembic-hook {{ if eq .Values.diffgramSettings.DIFFGRAM_STATIC_STORAGE_PROVIDER "gcp"}} volumeMounts: - - name: service-account-credentials-volume + - name: service-account-credentials-volume-hook mountPath: /etc/gcp readOnly: true {{ end }} diff --git a/templates/hooks/database_pre_upgrade.yaml b/templates/hooks/database_pre_upgrade.yaml index 6251f01..cd5efad 100644 --- a/templates/hooks/database_pre_upgrade.yaml +++ b/templates/hooks/database_pre_upgrade.yaml @@ -53,4 +53,9 @@ spec: - secretRef: name: db-migrations-secret # The actual migrations command - command: ["sh","-c", "cd shared; export PYTHONPATH=/app; pip install sqlalchemy-utils==0.36.6;python /app/play_and_scripts/scripts/create_database.py;"] \ No newline at end of file + command: ["sh","-c", "cd shared; export PYTHONPATH=/app; pip install sqlalchemy-utils==0.36.6;python /app/play_and_scripts/scripts/create_database.py;"] + + {{ if .Values.nodeGroupLabel }} + nodeSelector: + poolName: {{ .Values.nodeGroupLabel }} + {{ end }} \ No newline at end of file diff --git a/templates/hooks/secret_gcr_db_migrations.yaml b/templates/hooks/secret_gcr_db_migrations.yaml index 2d29772..4bae753 100644 --- a/templates/hooks/secret_gcr_db_migrations.yaml +++ b/templates/hooks/secret_gcr_db_migrations.yaml @@ -20,6 +20,6 @@ metadata: manager: kubectl operation: Update name: db-migrations-pull-secret - namespace: default + namespace: {{ .Release.Namespace }} type: kubernetes.io/dockerconfigjson {{ end }} \ No newline at end of file diff --git a/templates/ingress.yaml b/templates/ingress.yaml index 320ac7a..cf008ad 100644 --- a/templates/ingress.yaml +++ b/templates/ingress.yaml @@ -2,6 +2,7 @@ apiVersion: extensions/v1beta1 kind: Ingress metadata: name: diffgram-ingress + namespace: {{ .Release.Namespace }} annotations: kubernetes.io/ingress.class: "nginx" nginx.ingress.kubernetes.io/use-regex: "true" diff --git a/templates/postgres/deployment.yaml b/templates/postgres/deployment.yaml index 1de8932..e10fa54 100644 --- a/templates/postgres/deployment.yaml +++ b/templates/postgres/deployment.yaml @@ -5,7 +5,7 @@ apiVersion: "apps/v1" kind: "Deployment" metadata: name: "postgres" - namespace: "default" + namespace: {{ .Release.Namespace }}" labels: app: "postgres" annotations: @@ -26,7 +26,7 @@ spec: spec: containers: - name: "postgres" - image: "postgres:9.6.2" + image: "postgres:11" env: - name: "POSTGRES_DB" value: {{ .Values.dbSettings.dbName }} @@ -44,5 +44,9 @@ spec: - name: postgres-storage persistentVolumeClaim: claimName: postgres-pv-claim + {{ if eq .Values.nodeGroupLabel }} + nodeSelector: + poolName: {{ .Values.nodeGroupLabel }} + {{ end }} {{ end }} {{ end }} \ No newline at end of file diff --git a/templates/postgres/service.yaml b/templates/postgres/service.yaml index c5297e5..49c6cc8 100644 --- a/templates/postgres/service.yaml +++ b/templates/postgres/service.yaml @@ -3,6 +3,7 @@ apiVersion: v1 kind: Service metadata: name: diffgram-postgres + namespace: {{ .Release.Namespace }} spec: ports: - port: 5432 diff --git a/templates/secret-gcr.yaml b/templates/secret-gcr.yaml index cc3f4e3..7fc7e46 100644 --- a/templates/secret-gcr.yaml +++ b/templates/secret-gcr.yaml @@ -15,6 +15,6 @@ metadata: manager: kubectl operation: Update name: diffgramsecret - namespace: default + namespace: {{ .Release.Namespace }} type: kubernetes.io/dockerconfigjson {{ end }} \ No newline at end of file diff --git a/templates/walrus/deployment.yaml b/templates/walrus/deployment.yaml index 788659a..79a27b3 100644 --- a/templates/walrus/deployment.yaml +++ b/templates/walrus/deployment.yaml @@ -4,7 +4,7 @@ metadata: labels: app: diffgram-walrus name: diffgram-walrus - namespace: default + namespace: spec: replicas: {{ .Values.walrusService.numReplicas }} selector: @@ -15,6 +15,10 @@ spec: labels: app: diffgram-walrus spec: + {{ if .Values.nodeGroupLabel }} + nodeSelector: + poolName: {{ .Values.nodeGroupLabel }} + {{ end }} {{ if eq .Values.diffgramEdition "enterprise"}} imagePullSecrets: - name: diffgramsecret @@ -32,7 +36,7 @@ spec: emptyDir: {} initContainers: - name: check-db-ready - image: postgres:9.6.5 + image: postgres:11 {{ if eq .Values.dbSettings.dbProvider "local"}} command: ['sh', '-c', 'until pg_isready -h diffgram-postgres -p 5432; diff --git a/templates/walrus/service.yaml b/templates/walrus/service.yaml index 02766f4..bb89a5f 100644 --- a/templates/walrus/service.yaml +++ b/templates/walrus/service.yaml @@ -4,7 +4,7 @@ metadata: labels: app: diffgram-walrus name: diffgram-walrus - namespace: default + namespace: {{ .Release.Namespace }} spec: ports: - port: 8080 diff --git a/values.yaml b/values.yaml index a50791c..7514740 100644 --- a/values.yaml +++ b/values.yaml @@ -71,6 +71,8 @@ imagePullCredentials: # The service account with permissions to pull from the GCR Repository. [Should be Provided by Diffgram Team.] gcrCredentials: provided_by_diffgram_team +nodeGroupLabel: null + # The service for API calls. # This are minimal defaults. Please feel free to change them as you start having more usage defaultService: From 900eb4a1163956c17a47e0b5294b0e26a4e846fa Mon Sep 17 00:00:00 2001 From: Pablo Date: Tue, 13 Jul 2021 14:50:53 -0600 Subject: [PATCH 2/3] feat: add watch name space to restrict traffic to certain namespaces --- templates/ingress.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/templates/ingress.yaml b/templates/ingress.yaml index cf008ad..fa60f01 100644 --- a/templates/ingress.yaml +++ b/templates/ingress.yaml @@ -8,6 +8,7 @@ metadata: nginx.ingress.kubernetes.io/use-regex: "true" nginx.ingress.kubernetes.io/enable-cors: "true" cert-manager.io/issuer: "letsencrypt-prod" + watch-namespace: {{ .Release.Namespace }} # nginx.ingress.kubernetes.io/ssl-redirect: "false" # nginx.ingress.kubernetes.io/force-ssl-redirect: "false" # Limit uploads to 8TB From ec61376acc5aef74a43beef8fb30dde52ce5a9ca Mon Sep 17 00:00:00 2001 From: Pablo Date: Wed, 14 Jul 2021 23:31:12 -0600 Subject: [PATCH 3/3] add conditionals for volume mounts and improve baseline hardware specs --- templates/hooks/database_pre_upgrade.yaml | 4 ++++ values.yaml | 18 +++++++++--------- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/templates/hooks/database_pre_upgrade.yaml b/templates/hooks/database_pre_upgrade.yaml index cd5efad..a7bef5b 100644 --- a/templates/hooks/database_pre_upgrade.yaml +++ b/templates/hooks/database_pre_upgrade.yaml @@ -27,6 +27,7 @@ spec: imagePullSecrets: - name: diffgramsecret {{ end }} + {{ if eq .Values.diffgramSettings.DIFFGRAM_STATIC_STORAGE_PROVIDER "gcp"}} volumes: - name: service-account-credentials-volume-hook secret: @@ -34,6 +35,7 @@ spec: items: - key: sa_json path: sa_credentials.json + {{ end }} containers: {{ if eq .Values.diffgramEdition "enterprise"}} - image: gcr.io/diffgram-enterprise/default:{{ .Values.diffgramVersion }} @@ -43,10 +45,12 @@ spec: {{ end }} imagePullPolicy: Always name: pre-upgrade-alembic-hook + {{ if eq .Values.diffgramSettings.DIFFGRAM_STATIC_STORAGE_PROVIDER "gcp"}} volumeMounts: - name: service-account-credentials-volume-hook mountPath: /etc/gcp readOnly: true + {{ end }} envFrom: - configMapRef: name: db-migrations-configmap diff --git a/values.yaml b/values.yaml index 7514740..51a37bb 100644 --- a/values.yaml +++ b/values.yaml @@ -76,30 +76,30 @@ nodeGroupLabel: null # The service for API calls. # This are minimal defaults. Please feel free to change them as you start having more usage defaultService: - numReplicas: 1 + numReplicas: 2 requests: cpu: "2.0" - memory: "2G" + memory: "4G" limits: cpu: "2.0" - memory: "2G" + memory: "4G" # The service for the UI frontend. # This are minimal defaults. Please feel free to change them as you start having more usage frontendService: numReplicas: 1 requests: - cpu: "1.0" + cpu: "2.0" memory: "2G" limits: - cpu: "1.0" + cpu: "2.0" memory: "2G" # The service for video processing. This is where the heavy processing takes place. # This are minimal defaults. Please feel free to change them as you start having more usage walrusService: - numReplicas: 1 + numReplicas: 2 requests: cpu: "4.0" - memory: "16G" + memory: "14G" limits: - cpu: "4.0" - memory: "16G" \ No newline at end of file + cpu: "8.0" + memory: "32G" \ No newline at end of file