diff --git a/.vault-pass.sh b/.vault-pass.sh
new file mode 100755
index 0000000..2b7a6a6
--- /dev/null
+++ b/.vault-pass.sh
@@ -0,0 +1,5 @@
+#!/bin/bash
+# This script retrieves the password for the current git repository from a Bitwarden Vault.
+# If bw is not available, it should error out
+set -e
+bw get password "concat-datalab" || { read -sp "Password or Bitwarden CLI not found. Enter vault password: " password; echo "$password"; }
diff --git a/Makefile b/Makefile
new file mode 100644
index 0000000..131ef41
--- /dev/null
+++ b/Makefile
@@ -0,0 +1,46 @@
+# Makefile that passes the CLI args to ansible for the given playbook tags
+BW_AVAILABLE = $(shell command -v bw 2> /dev/null)
+VAULT_PASS_ARGS = $(if $(BW_AVAILABLE),--vault-password-file ./.vault-pass.sh, --ask-vault-pass)
+PLAYBOOK_CMD = uv run ansible-playbook $(VAULT_PASS_ARGS) -i ansible/inventory.yml ansible/playbook.yml
+VAULT_FILES := $(shell find ansible/vaults/ -type f);
+
+# Extract tags directly from YAML (faster but less accurate)
+VALID_TAGS := $(shell grep -r "tags:" ansible/playbook.yml | sed 's/.*tags://g' | tr -d '[]"' | tr ',' '\n' | tr -d ' ' | sort -u | grep -v '^$$')
+
+all:
+ $(PLAYBOOK_CMD)
+
+list:
+ uv run ansible-playbook --list-tags ansible/playbook.yml
+
+inventory:
+ uv run ansible-vault edit $(VAULT_PASS_ARGS) ansible/inventory.yml
+
+install-ansible: requirements.txt
+ uv venv --python=3.13; uv pip install -r requirements.txt
+
+encrypt-vaults:
+ @echo "Encrypting all vault files in ansible/vaults/ directory: $(VAULT_FILES)";
+ uv run ansible-vault encrypt $(VAULT_FILES)
+
+vaults:
+ @echo "Select a vault file to edit:"; \
+ select file in $$(find ansible/inventory.yml ansible/vaults/ -type f); do \
+ if [ -n "$$file" ]; then \
+ uv run ansible-vault edit $(VAULT_PASS_ARGS) "$$file"; \
+ break; \
+ fi; \
+ done
+
+%:
+ @if echo "$(VALID_TAGS)" | grep -wq "$@"; then \
+ echo "Running playbook with tag: $@"; \
+ $(PLAYBOOK_CMD) --tags=$@; \
+ else \
+ echo "Error: '$@' is not a valid tag."; \
+ echo "Valid tags are: $(VALID_TAGS)"; \
+ exit 1; \
+ fi
+
+
+.PHONY: list
diff --git a/README.md b/README.md
index d27ca7d..a5dfce1 100644
--- a/README.md
+++ b/README.md
@@ -12,7 +12,7 @@
This repository contains tools and rules for automatically deploying datalab instances using Terraform/OpenTofu and Ansible.
It can be used as a template for deploying your own datalab instance, and optionally periodically resynced on new releases.
-Use of Terraform for cloud provisioning is OPTIONAL; the Ansible playbooks are sufficient to set up a datalab instance on a existing hardware.
+Use of Terraform/OpenTofu for cloud provisioning is OPTIONAL; the Ansible playbooks are sufficient to set up a datalab instance on a existing hardware.
The Ansible playbooks can even be used to deploy datalab on shared hardware; all mandatory services will be deployed within containers, so it is possible to set up the full NGINX + datalab stack alongside an existing reverse proxy managing other services, although this will need to be configured by the user.
Ideally, various aspects of the configuration will be transferrable, and thus only instance-specific configuration will need to be provided (e.g., usernames, domain names), in which case your instance version of this repository can be kept fairly in-sync with the main branch (which will continue to be updated as datalab's requirements grow and change).
@@ -21,12 +21,22 @@ Attempts will be made to tabulate the supported versions of datalab with each re
## Supported versions
+
+
| This repository version | *datalab* version |
|---|---|
| v0.1.x | v0.4.x |
| v0.2.x | v0.4.x |
-| v0.3.x | v0.5.x+ |
+| v0.3.x | v0.5.0-rc.x |
+| v0.4.x | v0.5.x |
+| v0.5.x | v0.6.x |
+| v0.6.x | v0.6.x |
+
+
+
+## Changelog
+The changelog for this repository can be found in the [release notes](https://github.com/datalab-industries/datalab-ansible-terraform/releases) on GitHub.
## Overview
@@ -53,7 +63,7 @@ Stack:
- Docker Compose for API, app and database containers
- Simple filesystem for filestore
-### Datalab versioning
+### *datalab* versioning
The Ansible playbooks will deploy the datalab version that is included as a `git
submodule` to this repository.
@@ -64,19 +74,36 @@ repository state (i.e., no uncommited changes) to ensure reproducibility.
### Ansible automation
+#### First-time setup
+
These instructions assume you have prepared the server on which you would like
to deploy *datalab*, and that it is:
- accessible via SSH (using your local SSH config),
- running Ubuntu 22.04 or a similar distro with `apt` available.
-It also assumes you have set up Ansible on your local machine.
-You can find instructions for this in the [Ansible documentation](https://docs.ansible.com/ansible/latest/getting_started/get_started_ansible.html).
+It also assumes that your local machine is running a Unix-like OS (Linux, WSL, macOS) with `git` and `bash`,
+`make` and `sed` available.
+
+You can find more information on the requirements of the server and control node in the [Ansible documentation](https://docs.ansible.com/ansible/latest/getting_started/get_started_ansible.html).
-The first step is to clone this repository (or your fork) with submodules:
+The first step is to clone this repository (or your fork/templated version) with submodules and then install Ansible and its dependencies.
+We recommend using [uv](https://astral.sh/uv) for this, as the included [Makefile] will use it to run the playbooks in a virtual
+environment.
```shell
git clone --recurse-submodules git@github.com:datalab-org/datalab-ansible-terraform
+cd datalab-ansible-terraform
+uv venv --python 3.12
+uv pip install -r requirements.txt
+```
+
+or alternatively,
+
+```shell
+git clone --recurse-submodules git@github.com:datalab-org/datalab-ansible-terraform
+cd datalab-ansible-terraform
+make install-ansible
```
You can then navigate to the to the `./ansible` directory to begin configuring
@@ -105,6 +132,9 @@ ungrouped:
borg_encryption_passphrase:
borg_remote_path:
borg_repository:
+ prometheus_remote_write_url:
+ prometheus_user:
+ prometheus_password:
```
where `` and the various setting should be configured with your chosen
@@ -130,10 +160,20 @@ To encrypt them, you can run
ansible-vault encrypt inventory.yml vaults/datalab/prod_config.json vaults/datalab/.env vaults/datalab/.env_server vaults/datalab/.ssh/* vaults/borg/.ssh/*
```
+or simply
+
+```shell
+make encrypt-vaults
+```
+
and provide a password when prompted (which will then need to be kept safe and
-used every time the Ansible playbook is run). Omit the optional SSH wildcards if no
-SSH keys are required.
-You should never commit these files directly without encryption.
+used every time the Ansible playbook is run).
+You should never commit these files directly without encryption, even to a
+private git repository.
+
+If you are using your own domain (configured via `app_url` and `api_url` in the inventory),
+then you will need to update your domain's DNS settings so that your subdomains point to the IP
+of the server as given in your inventory file.
Once all these configuration steps have been performed, we can try to execute
the Ansible "playbook" that will install all the pre-requisite services, build
@@ -146,10 +186,158 @@ This is achieved by running:
ansible-playbook --ask-vault-pass -i inventory.yml playbook.yml
```
-If completed successfully, the server should now be running a *datalab*
-instance at your configured URL!
+or simply
+
+```
+make
+```
+
+If completed successfully, the server should now be running a *datalab* instance at your configured URLs!
+
+#### Keeping things up to date
+
+To update the *datalab* version, you simply update the git submodule in
+`src/datalab` and rerun the playbook. This can be pinned to your fork and accomodate any custom changes
+you desire (though you may also need to test and maintain your own set of
+ansible rules and configuration for this).
+
+Once you have chosen the *datalab* version, it can be redployed with `make
+deploy` or
+
+```shell
+ansible-playbook --ask-vault-pass -i inventory.yml playbook.yml --tags deploy
+```
+
+To update the ansible playbooks themselves with any changes from the upstream
+repository, you can similarly maintain the submodule in
+`src/datalab-ansible-terraform` and either manually sync changes across, or use
+the helper script:
+
+```shell
+chmod u+x sync-ansible-upstream.sh && ./sync-ansible-upstream.sh
+```
+
+which will copy just the changed playbooks across, and commit them. You should
+be careful to review these changes before committing them to your fork,
+especially if you have made any custom changes to the playbooks.
+Be sure to also commit the changes to your submodule so you know precisely which versions
+of the playbooks are running.
+
+The [`Makefile`] also contains a number of other useful commands, such as:
+
+```shell
+make vaults
+```
+
+to edit the encrypted vault files, and
+
+```shell
+make list
+```
+
+to list all of the available Ansible tags that can be run individually.
+
+#### Bitwarden integration
+
+Constantly entering the vault password for every attempted deployment can be a
+bit tedious, so by default, the `Makefile` will attempt to retrieve the vault
+password from a local [Bitwarden CLI](https://bitwarden.com/help/cli/) installation,
+which either only requires you to enter your Bitwarden password once per
+session, or can be configured to remain logged in.
+
+To use this feature, you will need to store your vault password in Bitwarden
+using the same name as the cloned repository as reported locally by
+
+```shell
+$ git remote get-url origin
+git@github.com:datalab-org/datalab-demo-deployment
+ {^^^^^^^^^^^^^^^^^^^^^}
+ repository name
+```
+
+If you intially cloned this repo and then renamed it, you can update your local
+version to use the same name using:
+
+```shell
+git remote set-url origin
+```
+
+You can also simply edit the `.vault-pass.sh` script to return your vault
+password in another way if you prefer.
+
+You may also need to set the script to be executable with
+
+```shell
+chmod u+x .vault-pass.sh
+```
+
+#### Backups
-If you are using your own domain, you will need to update your DNS settings so that your domain name points to the IP of the server as given in your inventory file.
+##### Native backups
+
+By default, *datalab* will take native snapshot backups at a certain frequency
+and save them into `/data/backups` on the server with the configured retention
+rules (typically keeping 7 daily copies, 6 monthly and 4 yearly).
+These backups can be synced with a remote system to avoid data loss in the event
+of a hardware failure on your *datalab* server.
+
+More information on backups can be found in the
+[datalab documentation](https://docs.datalab-org.io/en/stable/deployment/#backups).
+
+##### Borg backups (recommended)
+
+This repository contains playbooks for automating much more robust backups
+using [Borg](https://www.borgbackup.org/en/stable/) and
+[borgmatic](https://torsion.org/borgmatic/).
+
+These backups are encrypted, de-duplicated and compressed, requiring significantly
+less space than the native backup option, and can be synced easily with remote
+Borg instances over SSH.
+You will need to have an appropriate remote server (ideally separate from the *datalab* server itself)
+running Borg (we recommend [rsync.net](https://www.rsync.net/products/borg.html) which has excellent borg support).
+
+To activate the Borg playbooks, you must first provide an encrypted SSH key pair for accessing
+your remote Borg server in the `./vaults/borg/.ssh` directory.
+It should be named `./vaults/borg/.ssh/id_ed25519` and contain the private key for
+passwordless SSH connection to the remote Borg server configured in the
+inventory.
+Any other encrypted files in this `.ssh` vault will be mounted in the Borg
+container during the backup procedure. You may wish to include an `.ssh/config`
+to set up any extra settings (e.g., proxies, host checking), or an `.ssh/known_hosts` to enable strict host checking.
+
+> If you are running a *datalab* instance yourself and are looking for a place
+> for your encrypted Borg backups, feel free to reach out to us as we may have
+> enough of an overhead to be a secondary backup host for you.
+
+#### Server monitoring
+
+One basic option for uptime monitoring is to use a free GitHub Actions based
+service like [Upptime](https://github.com/upptime/upptime).
+For example, this is used for simple services in the central *datalab* organisation at [datalab-org/datalab-org-status](https://github.com/datalab-org/datalab-org-status).
+
+For more advanced monitoring, the Ansible playbooks contain a role tagged as
+`monitoring`, which will install and configure metrics harvesters using
+[Prometheus](https://prometheus.io/) (with [Node Exporter](https://github.com/prometheus/node_exporter) and
+[cAdvisor](https://github.com/google/cadvisor)) to monitor the host system and
+containers.
+
+To make use of this monitoring, you will need your own [Grafana instance](https://grafana.com/oss/grafana) (also running Prometheus as a harvester of the remote metrics) to visualise the metrics.
+
+Alternatively, you can use a hosted Grafana service such as [Grafana Cloud](https://grafana.com/products/cloud/), or request to use our central
+*datalab* Grafana instance by reaching out to us on Slack or over email.
+
+This integration can be enabled by adding the following variables to your inventory:
+
+```yaml
+prometheus_remote_write_url:
+prometheus_user:
+prometheus_password:
+```
+and then running the playbook with the `monitoring` tag:
+
+```shell
+make monitoring
+```
### Cloud provisioning
diff --git a/ansible/Makefile b/ansible/Makefile
deleted file mode 100644
index 5ca0bf4..0000000
--- a/ansible/Makefile
+++ /dev/null
@@ -1,17 +0,0 @@
-all:
- ansible-playbook -v -i inventory.yml --ask-vault-pass playbook.yml
-
-deploy:
- ansible-playbook -v -i inventory.yml --ask-vault-pass playbook.yml --tags="deploy"
-
-setup:
- ansible-playbook -v -i inventory.yml --ask-vault-pass playbook.yml --tags="setup"
-
-maintenance:
- ansible-playbook -v -i inventory.yml --ask-vault-pass playbook.yml --tags="maintenance"
-
-borg:
- ansible-playbook -v -i inventory.yml --ask-vault-pass playbook.yml --tags="borg"
-
-ssl:
- ansible-playbook -v -i inventory.yml --ask-vault-pass playbook.yml --tags="ssl"
diff --git a/ansible/inventory.yml b/ansible/inventory.yml
index 594fa04..9a05ab4 100644
--- a/ansible/inventory.yml
+++ b/ansible/inventory.yml
@@ -1,30 +1,41 @@
$ANSIBLE_VAULT;1.1;AES256
-33353835333031346165613762646132633137653338376431333933323565353462393135393231
-6537343266336634633563313531336362303665616432340a366435636438353537633232386263
-38396437626635643930336638663862616339303131353337373034323833303934623534303763
-3963633031373334630a383537623639376538633531323837333964303365623337643262666637
-37383864643137343565396530316637363262666666633461343863303861333039393831646430
-30613337303763363732306564623137363832306434626230363234303032363263353633386665
-34363166303963343165643131656133366532353566373831303330343730313262356532623166
-31306661316533656531396165393366383765353661366363316665353533343839366533393233
-38646235653236633564316235363266303830623235366135643430643830326238663932396637
-30316432356266303262326565353534623561363038666566626666396363623732633463316633
-65386139396662653637643835393133323761653937366164636665343666383464643738393631
-38396166303763616234653431306433306136356665316662316637343564663361643637653033
-31303062636536663938626630373439303732376136363535303934373430383835323962333136
-35383361616338326266343735363731363436323263343237356266646562343635653562393534
-30393266633431666430643264346165343262376665636563383865373134613166306431613039
-62313639613166646265366336656434626365306664303266623037303738366364646366303565
-38656265356664313065393061343065653161326163386238383037633964663938363438323236
-32336366356563316462653761353763323839313563646235666564313338383633663333653964
-65303962626666303566343666323432633733663037336637363464396565643365663461373966
-65613336343362343063333838343733346133336434633565613435633438336165623937666339
-31386136323331323631643533666663383437353238383263663137633366363632633061613433
-34393936666363663536666561303562353939663534633565666363663332636562363861303533
-65383132323065373062626664613134636433303864353366326364353932653437353533313139
-38363832353334356139643430626161623639343330333235613638306632623639666536383965
-34363365633037316238633634656364633636323163623436396562623239363832306165623131
-66303032633263373065373731643135356662613661336561336465303236353965663237643439
-38316130313634663030656437613033623434643939303061636134333663346264636162316666
-36326533316165623430363032313363623738336134623133323338313439663132626332653332
-61386462323866646566313038316430623432303862653062306334653231363263
+35303739623964643735623863656533623261353230643538633763306462386264333437376561
+6466363561653333653230346134386133326162363233310a303165343666346533356262653230
+37333464346335376336383062386630653830383363633034366635663833333536303535646632
+3339313065623964390a363062326531326265636161653838313634346137376461616339663434
+65643065346665326463653865643462613031643135376162303130303936376162333862353636
+62336238653139613137366366326462383264376263636533643034353661623661616432313237
+63343366353666663164646662363837656466363236633031663566336266363631663335653864
+32393737616433313965386330346535366335653431643338623863623164656630343762366664
+35343139396337393164616332393331336165326631653036386364643162356436353534346463
+38633637313265633365303663376130653735653664653166646464616634313030323261653630
+31303635376432393334396665396661313465636662623663303936336438633462303465386264
+64653533393539363766363336633936633165633461376564613037613539646636636263333137
+37356562646236646534396638356438663736396338313162663030333633643339323838363161
+64653231303433333134353434386337633963623439396462643331653239646234616163363364
+65613866383137663931323734333666666561373261316162643731396131633732353131653237
+39383437386365616564616136643063306563656537386630656535343837316532343632366233
+61656262333638353736626662663661393930383661616263306563306134383038303137613138
+38343561386339303635643339303037386531326266613138363565383561323839306564336262
+63353032653765626434306366616532333431306334383633323634336334616432626537333865
+66613138663530643533386537616163663962326265346266613536323232353266613539376239
+39303161363737346235643930383261336532613661613864323436643661353336373631363531
+39336333363537306636316336653864346237626463303232333561656139383463306138626339
+66353837653932333634653739646532356634313964313930633064386439636262396230636439
+61343131373132326234373136356561636637636134313562373339353464353431383661623061
+34646532633233663761636364643539663462643166376464623561303066343061613863396430
+66653763326562656137613233303838313433343861653031396231336265306231366365653038
+38653736396530386233646631323133646338376533656233356434623637626639336465616233
+37363039396262653139313335613762363537666238373362333732386430613535393066323337
+63366235616531376563343537393363383762396164396331353862636661343436613536303939
+37316433393863343937653632353931326635363965396535303437356434303965663639626335
+31316230633034663663343432336637353436303132616636373361663231393334383266623936
+37663038306431623638336661623863356665613332653962303437383133666138303365313065
+36363633366161336462363162366535323238333239386234663939383837373537366363373563
+34396265343661363438383763326664383839326265346132323239373535333131373637373235
+30333838633935616636663739363961333131653164633134666664316433613266343235666162
+62306261636262616432373035336633616165383464313632333465663931656239616437313763
+30313136646466353439393239643936326531356562623164353463643635653138316437383033
+30626563646366323561666339396434666564623735643066373630623361386634376136346666
+30353439373230393031616263366363653031623839326431323335323461623733636463373031
+36633137376130363336
diff --git a/ansible/playbook.yml b/ansible/playbook.yml
index 9bb2187..74103ba 100644
--- a/ansible/playbook.yml
+++ b/ansible/playbook.yml
@@ -29,9 +29,14 @@
- role: borg
name: Configure borg(matic) and remote backups
tags: [borg]
+ - role: monitoring
+ name: Install and configure prometheus monitoring stack
+ tags: [monitoring]
tasks:
- name: Keep all packages up-to-date
ansible.builtin.include_role:
name: apt_upgrade
+ apply:
+ tags: [maintenance]
tags: [maintenance]
diff --git a/ansible/roles/borg/tasks/main.yml b/ansible/roles/borg/tasks/main.yml
index a0da38f..39ec636 100644
--- a/ansible/roles/borg/tasks/main.yml
+++ b/ansible/roles/borg/tasks/main.yml
@@ -1,103 +1,97 @@
---
-- name: Synchronize borgmatic files to remote
- ansible.posix.synchronize:
- src: "{{ role_path }}/files/"
- dest: "{{ ansible_user_home_dir }}/borgmatic"
-
-- name: Check whether ssh config exists
+- name: Check whether ssh cert for borg exists
ansible.builtin.stat:
path: "{{ playbook_dir }}/vaults/borg/.ssh/id_ed25519"
register: ssh_config
delegate_to: localhost
-- name: Set fact for whether borg ssh config exists
- ansible.builtin.set_fact:
- ssh_config_defined: "{{ ssh_config.stat.exists }}"
+- name: Configure borgmatic and borg backups
+ when:
+ - ssh_config.stat.exists
+ - borg_repository is defined
+ - borg_encryption_passphrase is defined
+ - borg_remote_path is defined
+ block:
+ - name: Synchronize borgmatic files to remote
+ ansible.posix.synchronize:
+ src: "{{ role_path }}/files/"
+ dest: "{{ ansible_user_home_dir }}/borgmatic"
-- name: Sync local ssh config vault remote
- when: ssh_config_defined
- become: true
- ansible.builtin.copy:
- src: "{{ playbook_dir }}/vaults/borg/.ssh/"
- dest: "{{ ansible_user_home_dir }}/borgmatic/.ssh"
- mode: "0700"
- owner: root
- group: root
+ - name: Sync local ssh config vault remote
+ when: ssh_config_defined
+ become: true
+ ansible.builtin.copy:
+ src: "{{ playbook_dir }}/vaults/borg/.ssh/"
+ dest: "{{ ansible_user_home_dir }}/borgmatic/.ssh"
+ mode: "0700"
+ owner: root
+ group: root
-- name: Render Borgmatic configuration
- become: true
- ansible.builtin.template:
- src: config.yaml.j2
- dest: "{{ ansible_user_home_dir }}/borgmatic/config.yaml"
- mode: "0600"
- owner: "{{ ansible_ssh_user }}"
+ - name: Render Borgmatic configuration
+ become: true
+ ansible.builtin.template:
+ src: config.yaml.j2
+ dest: "{{ ansible_user_home_dir }}/borgmatic/config.yaml"
+ mode: "0600"
+ owner: "{{ ansible_ssh_user }}"
- vars:
- borg_exclude_patterns:
- - /data/backups
- borg_exclude_from: []
- borg_install_method: package
- borg_user: "{{ ansible_user }}"
- borg_source_directories:
- - /data
- borgmatic_hooks:
- before_backup:
- - echo "`date` - Starting backup."
- mongodb_databases:
- - name: all
- hostname: datalab-database-1
- port: 27017
- borgmatic_timer: cron
- borg_retention_policy:
- keep_daily: 30
- keep_weekly: 0
- keep_monthly: 12
- keep_yearly: 4
- borg_one_file_system: true
- borgmatic_store_atime: true
- borgmatic_store_ctime: true
- borg_encryption_passcommand: false
- borg_remote_rate_limit: 0
- borg_ssh_command: ssh
- borg_lock_wait_time: 5
+ vars:
+ borg_exclude_patterns:
+ - /data/backups
+ borg_exclude_from: []
+ borg_install_method: package
+ borg_user: "{{ ansible_user }}"
+ borg_source_directories:
+ - /data
+ borgmatic_hooks:
+ before_backup:
+ - echo "`date` - Starting backup."
+ mongodb_databases:
+ - name: all
+ hostname: datalab-database-1
+ port: 27017
+ borgmatic_timer: cron
+ borg_retention_policy:
+ keep_daily: 30
+ keep_weekly: 0
+ keep_monthly: 12
+ keep_yearly: 4
+ borg_one_file_system: true
+ borgmatic_store_atime: true
+ borgmatic_store_ctime: true
+ borg_encryption_passcommand: false
+ borg_remote_rate_limit: 0
+ borg_ssh_command: ssh
+ borg_lock_wait_time: 5
-- name: Build borgmatic image
- become: true
- community.docker.docker_image:
- name: datalab-borgmatic
- source: build
- state: present
- force_source: true
- build:
- path: "{{ ansible_user_home_dir }}/borgmatic"
+ - name: Build borgmatic image
+ become: true
+ community.docker.docker_image:
+ name: datalab-borgmatic
+ source: build
+ state: present
+ force_source: true
+ build:
+ path: "{{ ansible_user_home_dir }}/borgmatic"
-- name: Create borg repository if it does not exist
- ansible.builtin.shell:
- cmd: >
- docker run --rm
- --network datalab_backend
- -v {{ ansible_user_home_dir }}/borgmatic/.ssh:/root/.ssh
- -v /data:/data datalab-borgmatic
- borgmatic init --encryption=repokey -c /etc/borgmatic/config.yaml
- executable: /bin/bash
- register: new_borg_repository
- changed_when: '"Repository already exists" not in new_borg_repository.stdout'
- failed_when: new_borg_repository.rc != 0
+ - name: Create borg repository if it does not exist
+ ansible.builtin.shell:
+ cmd: docker run --rm --network datalab_backend -v {{ ansible_user_home_dir }}/borgmatic/.ssh:/root/.ssh -v /data:/data datalab-borgmatic borgmatic init --encryption=repokey -c /etc/borgmatic/config.yaml
+ executable: /bin/bash
+ register: new_borg_repository
+ changed_when: '"Repository already exists" not in new_borg_repository.stdout'
+ failed_when: new_borg_repository.rc != 0
-- name: Perform first backup if borg repo was just created
- ansible.builtin.shell:
- cmd: docker run --rm --network datalab_backend -v {{ ansible_user_home_dir }}/borgmatic/.ssh:/root/.ssh -v /data:/data datalab-borgmatic # noqa: no-changed-when
- executable: /bin/bash
- when: new_borg_repository.changed # noqa: no-handler
+ - name: Perform first backup if borg repo was just created
+ ansible.builtin.shell:
+ cmd: docker run --rm --network datalab_backend -v {{ ansible_user_home_dir }}/borgmatic/.ssh:/root/.ssh -v /data:/data datalab-borgmatic # noqa: no-changed-when
+ executable: /bin/bash
+ when: new_borg_repository.changed # noqa: no-handler
-- name: Add Cron job for borgmatic
- ansible.builtin.cron:
- name: borgmatic
- hour: "2"
- minute: "{{ range(0, 59) | random(seed=inventory_hostname) }}"
- user: "{{ ansible_user }}"
- job: >
- docker run --rm
- --network datalab_backend
- -v {{ ansible_user_home_dir }}/borgmatic/.ssh:/root/.ssh
- -v /data:/data datalab-borgmatic
+ - name: Add Cron job for borgmatic
+ ansible.builtin.cron:
+ name: borgmatic
+ hour: "2"
+ minute: "{{ range(0, 59) | random(seed=inventory_hostname) }}"
+ user: "{{ ansible_user }}"
+ job: docker run --rm --network datalab_backend -v {{ ansible_user_home_dir }}/borgmatic/.ssh:/root/.ssh -v /data:/data datalab-borgmatic # noqa: line-length
diff --git a/ansible/roles/datalab/tasks/main.yml b/ansible/roles/datalab/tasks/main.yml
index 92d68c3..7406a6b 100644
--- a/ansible/roles/datalab/tasks/main.yml
+++ b/ansible/roles/datalab/tasks/main.yml
@@ -6,12 +6,19 @@
recursive: true
delete: false
+- name: Check for plugin folder
+ ansible.builtin.stat:
+ path: "{{ playbook_dir }}/../src/plugins"
+ register: plugins_dir
+
- name: Sync any plugins into datalab folder on remote
+ when: plugins_dir.stat.isdir is defined and plugins_dir.stat.isdir
ansible.posix.synchronize:
src: "{{ playbook_dir }}/../src/plugins"
dest: "{{ ansible_user_home_dir }}/datalab/pydatalab"
recursive: true
delete: false
+ failed_when: false
- name: Copy encrypted API env file from vault
ansible.builtin.copy:
@@ -55,6 +62,7 @@
project_src: "{{ ansible_user_home_dir }}/datalab"
profiles: prod
services: database
+ build: always
- name: Build and launch API container
community.docker.docker_compose_v2:
diff --git a/ansible/roles/monitoring/tasks/main.yml b/ansible/roles/monitoring/tasks/main.yml
new file mode 100644
index 0000000..310357e
--- /dev/null
+++ b/ansible/roles/monitoring/tasks/main.yml
@@ -0,0 +1,85 @@
+---
+- name: Check if prometheus_url is configured in inventory
+ when:
+ - prometheus_remote_write_url is defined
+ - prometheus_user is defined
+ - prometheus_password is defined
+ block:
+ - name: Launch node_exporter container
+ community.docker.docker_container:
+ name: node-exporter
+ image: prom/node-exporter:v1.9.1
+ network_mode: host
+ pid_mode: host
+ state: started
+ restart_policy: always
+ command:
+ - --path.rootfs=/host
+ - --path.procfs=/host/proc
+ - --path.sysfs=/host/sys
+ - --collector.filesystem.mount-points-exclude=^/(sys|proc|dev|host|etc)($$|/)
+ ports:
+ - 9100:9100
+ volumes:
+ # Mount the host's /proc directory to the container's /host/proc directory
+ # This is necessary for the node_exporter to be able to access the host's metrics
+ - /:/host:ro,rslave
+ - /proc:/host/proc:ro
+ - /sys:/host/sys:ro
+ - /etc/machine-id:/etc/machine-id:ro
+ - /etc/timezone:/etc/timezone:ro
+ healthcheck:
+ test: [CMD, wget, -q, --spider, http://localhost:9100/metrics]
+ interval: 30s
+ timeout: 10s
+ retries: 3
+ start_period: 5s
+
+ - name: Launch cadvisor container
+ community.docker.docker_container:
+ name: cadvisor
+ image: gcr.io/cadvisor/cadvisor:v0.52.1
+ network_mode: host
+ pid_mode: host
+ state: started
+ restart_policy: always
+ ports:
+ - 8080:8080
+ volumes:
+ - /:/rootfs:ro
+ - /var/run:/var/run:rw
+ - /sys:/sys:ro
+ - /var/lib/docker/:/var/lib/docker:ro
+
+ - name: Render prometheus config
+ ansible.builtin.template:
+ src: prometheus.yml.j2
+ dest: "{{ ansible_user_home_dir }}/prometheus.yml"
+ mode: "0644"
+ register: prometheus_config
+
+ - name: Launch prometheus container
+ community.docker.docker_container:
+ name: prometheus
+ image: prom/prometheus:v3.6.0
+ recreate: "{{ prometheus_config.changed }}"
+ network_mode: host
+ state: started
+ restart_policy: always
+ command:
+ - --config.file=/etc/prometheus/prometheus.yml
+ - --storage.tsdb.path=/prometheus
+ - --web.console.libraries=/etc/prometheus/console_libraries
+ - --web.console.templates=/etc/prometheus/consoles
+ - --web.enable-lifecycle
+ ports:
+ - 9090:9090
+ volumes:
+ - "prometheus_data:/prometheus"
+ - "{{ ansible_user_home_dir }}/prometheus.yml:/etc/prometheus/prometheus.yml:ro"
+ healthcheck:
+ test: [CMD, wget, -q, --spider, http://localhost:9090/-/healthy]
+ interval: 30s
+ timeout: 10s
+ retries: 3
+ start_period: 5s
diff --git a/ansible/roles/monitoring/templates/prometheus.yml.j2 b/ansible/roles/monitoring/templates/prometheus.yml.j2
new file mode 100644
index 0000000..ddaa299
--- /dev/null
+++ b/ansible/roles/monitoring/templates/prometheus.yml.j2
@@ -0,0 +1,28 @@
+global:
+ scrape_interval: 1m
+ scrape_timeout: 45s
+
+ external_labels:
+ instance: {{ app_url }}
+ environment: production
+ app: datalab
+ datalab: {{ datalab_prefix | default(app_url) }}
+
+scrape_configs:
+ - job_name: 'node-exporter'
+ static_configs:
+ - targets: ['localhost:9100']
+ labels:
+ service: system
+
+ - job_name: 'cadvisor'
+ static_configs:
+ - targets: ['localhost:8080']
+ labels:
+ service: docker
+
+remote_write:
+ - url: {{ prometheus_remote_write_url }}
+ basic_auth:
+ username: {{ prometheus_user }}
+ password: {{ prometheus_password }}
diff --git a/ansible/roles/ssl_first_run/tasks/main.yml b/ansible/roles/ssl_first_run/tasks/main.yml
index 61d5294..aa89c9e 100644
--- a/ansible/roles/ssl_first_run/tasks/main.yml
+++ b/ansible/roles/ssl_first_run/tasks/main.yml
@@ -71,6 +71,4 @@
hour: "10"
weekday: "2"
month: "*"
- job: |
- docker run --rm -v certbot-www:/var/www/certbot -v certbot-conf:/etc/letsencrypt certbot/certbot:latest renew
- && docker exec datalab-nginx nginx -s reload
+ job: docker run --rm -v certbot-www:/var/www/certbot -v certbot-conf:/etc/letsencrypt certbot/certbot:latest renew && docker exec datalab-nginx nginx -s reload # noqa: line-length
diff --git a/ansible/vaults/datalab/.env_server b/ansible/vaults/datalab/.env_server
index 6e8f701..28d76d9 100644
--- a/ansible/vaults/datalab/.env_server
+++ b/ansible/vaults/datalab/.env_server
@@ -1,20 +1,22 @@
$ANSIBLE_VAULT;1.1;AES256
-33383565386531323863393561623966346336313261633234663739303831363830356362323137
-3163373666653963316137383665643038616561346363300a396164336539386530326161306365
-64343639336431333638613730653966623034366336303762326463616237333864313863333236
-3666376432333334640a343161633130393662623163363934626139626230393965643661326439
-62353763306635643866396265303965383561323938393862646132343966613135646263626630
-34343239663438383033666263623537343230323637323231623564356164383536393461633063
-62326439313231316639316332663839396462383738363830663231613466383233303937396634
-66336263363530393032386366363435343731376261356366306535396239386537313964303162
-36396133303361643939346332316462653032626438376138306437356131323433336536343030
-34656132353162313062323733636261303031323166326533383065613937366438396130623864
-30623739616130326533316564636234666430643531373961323832336637653832303461306638
-63666165363836333331623431346163376664623239393537626562646335656566663866653461
-39353865323862366464393333616161353664636536356331623961386636356230623738363532
-31313834313266313739373939396434623064663339653031363030363932613562666436643061
-65633133343238653735393163383538306236653531646134356533363734623465316337646230
-65346362393434303838376439373838393334656632663466376332626439323539663564323136
-38656139373338323831396662366566393033316433633634353761303432643764616465656335
-30383363343634666464633463623632376665396539633765323334383830313432303233303338
-393936623132623231613165653531663330
+62663833636334316162613337393932323434353733653264313932353966323563383136656532
+3132363139633731386232346163663463383537343636350a383939306265376139323335366335
+32373361313138656537336131323936386439383836303935313136656338336130343332383563
+3836306637313065650a306165643139666134643565623333333465623162396139316435616236
+30643533623166346634393230656434653037346364366666343038396232336439326635636666
+35633064643632633264343261376266376131393261383835316430326464323035316535653333
+63333731373232396134333139653736653263643637396461343764643431666137383262383132
+64313132633338653038343863336662386336336234653436356238656165626430336433643864
+36656534643863656165353035633666323465356663356565366266326431383861373431363031
+65336535366432363938366332363637386436613733313065623165353130353238393933633136
+30393861303838643436656362383065333936343630383033616462393966383061346564363832
+31373562656562316264663234383562393439326665326365623035653635313663666666643633
+66396431373466323033333635313362626263303165643336633231376636663234366434393964
+39613261396637373332666538313966616662373566626435396337396662326361656330393939
+30303962303736346565663863333032376239623739383134623835323435326563616164356666
+64313061396132306264663434386137643462303065353362643033356637313539666434323432
+39393832656135353736313766373435323865626537636364633261306532383032623834313239
+39636538386436333739393862383333346232636338356636353764613534303839366366343662
+66616437653339316338343061626262366531316166636335343633383361303265656436353937
+39643362313533663331613837613136303436376438333766646464613866653935303636323031
+3733
diff --git a/src/datalab b/src/datalab
index 765647c..4b46424 160000
--- a/src/datalab
+++ b/src/datalab
@@ -1 +1 @@
-Subproject commit 765647c080f3f7bbfc8e4aace7f49f46bf8c38f2
+Subproject commit 4b46424ae8e77b7f72afc768fc8066e85b4df0b1
diff --git a/src/datalab-ansible-terraform b/src/datalab-ansible-terraform
index f19c8d3..e240605 160000
--- a/src/datalab-ansible-terraform
+++ b/src/datalab-ansible-terraform
@@ -1 +1 @@
-Subproject commit f19c8d3e5d4e51f4ea19323048e5007ac963517f
+Subproject commit e240605b35b8d9b730af631021de5a1af29c7896
diff --git a/sync-ansible-upstream.sh b/sync-ansible-upstream.sh
index 738e5ea..3eb2afd 100755
--- a/sync-ansible-upstream.sh
+++ b/sync-ansible-upstream.sh
@@ -1,7 +1,8 @@
#!/bin/bash
set -e -u -o pipefail
commit=$(cd src/datalab-ansible-terraform && git describe --tags)
-rsync --exclude vaults --exclude inventory.yml -avr src/datalab-ansible-terraform/ansible .
-git add -p ansible
+rsync --exclude vaults --exclude inventory.yml -avr src/datalab-ansible-terraform/sync-ansible-upstream.sh src/datalab-ansible-terraform/Makefile src/datalab-ansible-terraform/.vault-pass.sh src/datalab-ansible-terraform/README.md src/datalab-ansible-terraform/requirements.txt src/datalab-ansible-terraform/ansible .
+git add -p .
+git add src/datalab-ansible-terraform
git add $(git ls-files ansible --others --exclude-standard)
git commit ansible -p -m "Sync with upstream definitions from datalab-ansible-terraform $commit"