diff --git a/.circleci/config.yml b/.circleci/config.yml index 3f049fa5..077e4ccc 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -9,10 +9,16 @@ workflows: version: 2 - test: + test-and-push: jobs: - - node-v6 - - node-v8 + - test-node-v6 + - test-node-v8 + - docker-build + - docker-push: + requires: + - test-node-v6 + - test-node-v8 + - docker-build version: 2 jobs: @@ -53,12 +59,60 @@ jobs: - store_artifacts: path: build - node-v6: + test-node-v6: <<: *base docker: - image: circleci/node:6.10.3-browsers - node-v8: + test-node-v8: <<: *base docker: - image: circleci/node:8.4.0-browsers + + docker-build: + docker: + - image: circleci/node:6.10.3-browsers + + steps: + - setup_remote_docker: + reusable: true + + - checkout + + - run: + name: Build Docker image + command: | + docker build -f deployment/Dockerfile -t gcr.io/${GOOGLE_PROJECT_ID}/imageserver:$CIRCLE_SHA1 . + + - run: + name: Smoke test Docker image + command: | + docker run -d -p 9091:9091/tcp --name imageserver gcr.io/${GOOGLE_PROJECT_ID}/imageserver:$CIRCLE_SHA1 + docker run --network container:imageserver appropriate/curl --retry 60 --retry-connrefused --retry-delay 1 http://localhost:9091/ping + + docker-push: + docker: + - image: circleci/node:6.10.3-browsers + + steps: + - setup_remote_docker: + reusable: true + + - run: + name: Install gcloud + command: | + echo "deb http://packages.cloud.google.com/apt cloud-sdk-jessie main" | sudo tee -a /etc/apt/sources.list.d/google-cloud-sdk.list + curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add - + sudo apt-get update && sudo apt-get install google-cloud-sdk kubectl + + - run: + name: Push Docker image to GCR + command: | + echo ${GOOGLE_AUTH} | base64 -i --decode > ${HOME}/gcp-key.json + gcloud auth activate-service-account --key-file ${HOME}/gcp-key.json + gcloud --quiet config set project ${GOOGLE_PROJECT_ID} + gcloud --quiet config set compute/zone ${GOOGLE_COMPUTE_ZONE} + + docker tag gcr.io/${GOOGLE_PROJECT_ID}/imageserver:$CIRCLE_SHA1 gcr.io/${GOOGLE_PROJECT_ID}/imageserver:$CIRCLE_BRANCH + gcloud docker -- push gcr.io/${GOOGLE_PROJECT_ID}/imageserver:$CIRCLE_SHA1 + gcloud docker -- push gcr.io/${GOOGLE_PROJECT_ID}/imageserver:$CIRCLE_BRANCH diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..f2dfbccf --- /dev/null +++ b/.dockerignore @@ -0,0 +1,5 @@ +node_modules +npm-debug.log* + +build +test diff --git a/deployment/Dockerfile b/deployment/Dockerfile new file mode 100644 index 00000000..469328e6 --- /dev/null +++ b/deployment/Dockerfile @@ -0,0 +1,93 @@ +FROM ubuntu:trusty + +#################### +# Install node and dependencies +# From: https://github.com/nodejs/docker-node/blob/master/6.11/Dockerfile + +RUN apt-get update && apt-get install -y --no-install-recommends \ + gnupg curl ca-certificates xz-utils wget \ + && rm -rf /var/lib/apt/lists/* && apt-get clean + +RUN groupadd --gid 1000 node \ +&& useradd --uid 1000 --gid node --shell /bin/bash --create-home node + +# gpg keys listed at https://github.com/nodejs/node#release-team +RUN set -ex \ + && for key in \ + 9554F04D7259F04124DE6B476D5A82AC7E37093B \ + 94AE36675C464D64BAFA68DD7434390BDBE9B9C5 \ + FD3A5288F042B6850C66B31F09FE44734EB7990E \ + 71DCFD284A79C3B38668286BC97EC7A07EDE3FC1 \ + DD8F2338BAE7501E3DD5AC78C273792F7D83545D \ + B9AE9905FFD7803F25714661B63B535A4C206CA9 \ + C4F0DFFF4E8C1A8236409D08E73BC641CC11F4C8 \ + 56730D5401028683275BD23C23EFEFE93C4CFFFE \ + ; do \ + gpg --keyserver pgp.mit.edu --recv-keys "$key" || \ + gpg --keyserver keyserver.pgp.com --recv-keys "$key" || \ + gpg --keyserver ha.pool.sks-keyservers.net --recv-keys "$key" ; \ + done + +ENV NPM_CONFIG_LOGLEVEL info +ENV NODE_VERSION 6.11.3 + +RUN ARCH= && dpkgArch="$(dpkg --print-architecture)" \ + && case "${dpkgArch##*-}" in \ + amd64) ARCH='x64';; \ + ppc64el) ARCH='ppc64le';; \ + s390x) ARCH='s390x';; \ + arm64) ARCH='arm64';; \ + *) echo "unsupported architecture"; exit 1 ;; \ + esac \ + && curl -SLO "https://nodejs.org/dist/v$NODE_VERSION/node-v$NODE_VERSION-linux-$ARCH.tar.xz" \ + && curl -SLO --compressed "https://nodejs.org/dist/v$NODE_VERSION/SHASUMS256.txt.asc" \ + && gpg --batch --decrypt --output SHASUMS256.txt SHASUMS256.txt.asc \ + && grep " node-v$NODE_VERSION-linux-$ARCH.tar.xz\$" SHASUMS256.txt | sha256sum -c - \ + && tar -xJf "node-v$NODE_VERSION-linux-$ARCH.tar.xz" -C /usr/local --strip-components=1 \ + && rm "node-v$NODE_VERSION-linux-$ARCH.tar.xz" SHASUMS256.txt.asc SHASUMS256.txt \ + && ln -s /usr/local/bin/node /usr/local/bin/nodejs + +#################### +# Download fonts + +RUN apt-get update -y && apt-get install -y subversion fontconfig && \ + rm -rf /var/lib/apt/lists/* && apt-get clean && \ + cd /usr/share/fonts/truetype && \ + for font in \ + https://github.com/google/fonts/trunk/apache/droidsansmono \ + https://github.com/google/fonts/trunk/apache/droidsans \ + https://github.com/google/fonts/trunk/apache/droidserif \ + https://github.com/google/fonts/trunk/apache/roboto \ + https://github.com/google/fonts/trunk/apache/opensans \ + https://github.com/google/fonts/trunk/ofl/gravitasone \ + https://github.com/google/fonts/trunk/ofl/oldstandardtt \ + https://github.com/google/fonts/trunk/ofl/ptsansnarrow \ + https://github.com/google/fonts/trunk/ofl/raleway \ + https://github.com/google/fonts/trunk/ofl/overpass \ + ; do \ + svn checkout $font ; \ + done && \ + mkdir /usr/share/fonts/user && \ + fc-cache -fv && apt-get --auto-remove -y remove subversion + +#################### +# Copy and set up image-exporter + +RUN wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - && \ + sh -c 'echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google-chrome.list' && \ + apt-get update -y && \ + apt-get install -y google-chrome-stable xvfb && \ + rm -rf /var/lib/apt/lists/* && apt-get clean + +COPY package.json /var/www/image-exporter/ +COPY bin /var/www/image-exporter/bin +COPY src /var/www/image-exporter/src + +WORKDIR /var/www/image-exporter +RUN npm install && mkdir build + +COPY deployment/run_server / + +EXPOSE 9091 +ENTRYPOINT ["/run_server"] +CMD ["--plotlyJS", "https://plot.ly/static/plotlyjs/build/plotlyjs-bundle.js"] diff --git a/deployment/README.md b/deployment/README.md new file mode 100644 index 00000000..13e33d56 --- /dev/null +++ b/deployment/README.md @@ -0,0 +1,78 @@ +# Deployment to GKE using Docker and Kubernetes + +## Provision: + +This is done once, manually. + +``` +gcloud beta container clusters create imageserver-stage --enable-autoscaling --min-nodes=1 --max-nodes=3 --num-nodes=1 --zone=us-central1-a --additional-zones=us-central1-b,us-central1-c --enable-autoupgrade --cluster-version=1.7.6-gke.1 +# Note: "min", "num", and "max" nodes sets the number PER ZONE. +kubectl apply -f deployment/kube +kubectl get service imageserver # Will show the load balancer IP when it's ready +``` + +## Build & push: + +Builds are performed automatically by CircleCI, and if all tests pass the image +will be pushed to GCR. Images are tagged using the branch name and the +sha1 of the git commit. + +## Deploy, plotly.js upgrade, rollback + +This is done using plotbot. The following commands provide help: + +``` +@plotbot deploy how +@plotbot run how +``` + +# Font Support + +The image server ships with many built-in fonts (see the Dockerfile for a list) +and also supports external fonts. External fonts are intended for restrictively +licensed fonts that we can not ship as part of the open source release, and +may also be used by 3rd party users to install their own fonts (restrictively +licensed or open source). + +On boot, the image server looks for fonts in `/usr/share/fonts/user` and will +use any valid font found there. You may map a directory into this location in +the container. + +In GKE, the pod requires a GCE Persistent Disk called +`plotly-cloud-licensed-fonts` that contains the restrictively licensed fonts +used by Plotly Cloud. To update fonts: + +1. In the `PlotlyCloud` project, create a disk from the +`plotly-cloud-licensed-fonts` image and attach it to a GCE VM. + +2. Reboot the GCE VM, then mount /dev/sdb to a temporary directory. + +3. Add/remove/update fonts in this temporary directory. + +4. Unmount the temporary directory and detach the disk from the VM. + +5. Delete the `plotly-cloud-licensed-fonts` image and re-create it from the disk. + +6. Create new `plotly-cloud-licensed-fonts` persistent disks from the image: + +``` +for zone in us-central1-a us-central1-b us-central1-c ; do + gcloud compute disks create plotly-cloud-licensed-fonts --image-project=sunlit-shelter-132119 --image=plotly-cloud-licensed-fonts --zone $zone +done +``` + +# Mapbox Access Token + +In order to use the Mapbox functionality built in to plotly.js, a Mapbox +access token must be provided. This can be part of the plot JSON, but for cases +where it is not included in the plot JSON it is useful to have a default. + +To specify one, add it as a Kubernetes secret: + +``` +echo -n "pk.whatever.blabla" > /tmp/token +kubectl create secret generic mapbox --from-file=default_access_token=/tmp/token +``` + +After adding the secret for the first time or it changes, you'll need to recreate +all pods. The easiest way to do this is by running the update_plotlyjs command. diff --git a/deployment/kube/frontend.yaml b/deployment/kube/frontend.yaml new file mode 100644 index 00000000..a86559a1 --- /dev/null +++ b/deployment/kube/frontend.yaml @@ -0,0 +1,54 @@ +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: imageserver + labels: + app: imageserver +spec: + replicas: 3 + template: + metadata: + labels: + app: imageserver + tier: frontend + spec: + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: "app" + operator: In + values: + - imageserver + topologyKey: "kubernetes.io/hostname" + containers: + - name: imageserver-app + image: gcr.io/sunlit-shelter-132119/imageserver + args: ["--plotlyJS", "https://stage.plot.ly/static/plotlyjs/build/plotlyjs-bundle.js"] + env: + - name: PLOTLY_MAPBOX_DEFAULT_ACCESS_TOKEN + valueFrom: + secretKeyRef: + name: mapbox + key: default_access_token + # This setting makes nodes pull the docker image every time before + # starting the pod. This is useful when debugging, but should be turned + # off in production. + imagePullPolicy: Always + ports: + - name: http-server + containerPort: 9091 + volumeMounts: + - mountPath: "/usr/share/fonts/user" + name: plotly-cloud-licensed-fonts + livenessProbe: + httpGet: + path: /ping + port: 9091 + volumes: + - name: plotly-cloud-licensed-fonts + gcePersistentDisk: + pdName: plotly-cloud-licensed-fonts + readOnly: true + fsType: ext4 diff --git a/deployment/kube/hpa.yaml b/deployment/kube/hpa.yaml new file mode 100644 index 00000000..85cfa9a0 --- /dev/null +++ b/deployment/kube/hpa.yaml @@ -0,0 +1,15 @@ +apiVersion: autoscaling/v1 +kind: HorizontalPodAutoscaler +metadata: + name: imageserver +spec: + scaleTargetRef: + apiVersion: extensions/v1beta1 + kind: Deployment + name: imageserver + # Set this to 3x "min-nodes": + minReplicas: 3 + # Set this to 3x "max-nodes": + maxReplicas: 9 + # This should be raised to 80 for production: + targetCPUUtilizationPercentage: 50 diff --git a/deployment/kube/loadbalancer.yaml b/deployment/kube/loadbalancer.yaml new file mode 100644 index 00000000..3b87a26a --- /dev/null +++ b/deployment/kube/loadbalancer.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + name: imageserver + annotations: + cloud.google.com/load-balancer-type: "internal" + labels: + app: imageserver +spec: + type: LoadBalancer + ports: + - port: 9091 + protocol: TCP + selector: + app: imageserver diff --git a/deployment/playbook_stage.yml b/deployment/playbook_stage.yml new file mode 100644 index 00000000..377f6b1b --- /dev/null +++ b/deployment/playbook_stage.yml @@ -0,0 +1,8 @@ +- hosts: localhost + gather_facts: False + vars: + cluster: imageserver-stage + zone: us-central1-a + roles: + - common + - update diff --git a/deployment/playbook_stage_rollback.yml b/deployment/playbook_stage_rollback.yml new file mode 100644 index 00000000..7b72f33c --- /dev/null +++ b/deployment/playbook_stage_rollback.yml @@ -0,0 +1,8 @@ +- hosts: localhost + gather_facts: False + vars: + cluster: imageserver-stage + zone: us-central1-a + roles: + - common + - rollback diff --git a/deployment/playbook_stage_update_plotlyjs.yml b/deployment/playbook_stage_update_plotlyjs.yml new file mode 100644 index 00000000..2475c1f6 --- /dev/null +++ b/deployment/playbook_stage_update_plotlyjs.yml @@ -0,0 +1,8 @@ +- hosts: localhost + gather_facts: False + vars: + cluster: imageserver-stage + zone: us-central1-a + roles: + - common + - update_plotlyjs diff --git a/deployment/roles/common/tasks/main.yml b/deployment/roles/common/tasks/main.yml new file mode 100644 index 00000000..3fdb2e80 --- /dev/null +++ b/deployment/roles/common/tasks/main.yml @@ -0,0 +1,2 @@ +- name: Get credentials for cluster + local_action: command gcloud container clusters get-credentials {{ cluster }} --zone {{ zone }} diff --git a/deployment/roles/rollback/tasks/main.yml b/deployment/roles/rollback/tasks/main.yml new file mode 100644 index 00000000..bd00c45f --- /dev/null +++ b/deployment/roles/rollback/tasks/main.yml @@ -0,0 +1,2 @@ +- name: Undo rollout + local_action: command kubectl rollout undo deployments/imageserver diff --git a/deployment/roles/update/tasks/main.yml b/deployment/roles/update/tasks/main.yml new file mode 100644 index 00000000..8a0fc433 --- /dev/null +++ b/deployment/roles/update/tasks/main.yml @@ -0,0 +1,11 @@ +- name: Determine current sha1 to deploy + local_action: command git rev-parse HEAD + register: sha1 +- set_fact: sha1_to_deploy="{{ sha1.stdout }}" +- debug: msg="Deploying rev {{ sha1_to_deploy }}" + +- name: Ensure image exists + local_action: shell gcloud container images list-tags gcr.io/sunlit-shelter-132119/imageserver |grep -q {{ sha1_to_deploy }} + +- name: Rollout new image + local_action: command kubectl set image deployments/imageserver imageserver-app=gcr.io/sunlit-shelter-132119/imageserver:{{ sha1_to_deploy }} diff --git a/deployment/roles/update_plotlyjs/tasks/main.yml b/deployment/roles/update_plotlyjs/tasks/main.yml new file mode 100644 index 00000000..d7558d7a --- /dev/null +++ b/deployment/roles/update_plotlyjs/tasks/main.yml @@ -0,0 +1,11 @@ +# From https://github.com/kubernetes/kubernetes/issues/27081#issuecomment-238078103 +# Change an unused variable in the deployment config, which will cause a +# rolling update that creates new pods. (New pods will automatically run the +# latest plotly.js since that's loaded on boot.) + +- name: Show timestamp + local_action: command date +%s + register: date + +- name: Update plotly.js by patching the deployment + local_action: command kubectl patch deployment imageserver --patch "{\"spec\":{\"template\":{\"metadata\":{\"labels\":{\"RESTART_DATE\":\"{{ date.stdout }}\"}}}}}" diff --git a/deployment/run_server b/deployment/run_server new file mode 100755 index 00000000..3518a251 --- /dev/null +++ b/deployment/run_server @@ -0,0 +1,5 @@ +#!/bin/bash -xe + +fc-cache -v /usr/share/fonts/user + +xvfb-run --server-args '-screen 0 640x480x24' ./bin/plotly-export-server.js $@ diff --git a/deployment/tools/watch_deployment b/deployment/tools/watch_deployment new file mode 100755 index 00000000..e3f333ef --- /dev/null +++ b/deployment/tools/watch_deployment @@ -0,0 +1,3 @@ +#!/bin/bash + +kubectl rollout status deployments/imageserver