From 4184083eba40bdf22952b8e7966a8e8bf0fa3093 Mon Sep 17 00:00:00 2001 From: Andrew Miller Date: Wed, 19 Feb 2025 15:55:45 -0500 Subject: [PATCH 1/2] example of database --- dstack-database/Dockerfile | 47 +++++++++++++++++++++++++ dstack-database/README.md | 5 +++ dstack-database/docker-compose.yml | 56 ++++++++++++++++++++++++++++++ dstack-database/patroni.yml | 49 ++++++++++++++++++++++++++ dstack-database/start.sh | 21 +++++++++++ dstack-database/zoo.cfg | 12 +++++++ 6 files changed, 190 insertions(+) create mode 100644 dstack-database/Dockerfile create mode 100644 dstack-database/README.md create mode 100644 dstack-database/docker-compose.yml create mode 100644 dstack-database/patroni.yml create mode 100644 dstack-database/start.sh create mode 100644 dstack-database/zoo.cfg diff --git a/dstack-database/Dockerfile b/dstack-database/Dockerfile new file mode 100644 index 0000000..c9cff96 --- /dev/null +++ b/dstack-database/Dockerfile @@ -0,0 +1,47 @@ +FROM ubuntu:22.04 + +# Create postgres user first +RUN groupadd -r postgres --gid=999 && \ + useradd -r -g postgres --uid=999 postgres + +# Install dependencies +RUN apt-get update +RUN DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get -y install tzdata +RUN apt-get install -y \ + gosu \ + python3-pip \ + postgresql-14 \ + postgresql-client-14 \ + openjdk-11-jre-headless \ + curl \ + && rm -rf /var/lib/apt/lists/* + +# Install ZooKeeper +RUN curl -L https://dlcdn.apache.org/zookeeper/zookeeper-3.8.4/apache-zookeeper-3.8.4-bin.tar.gz | tar xz -C /opt \ + && mv /opt/apache-zookeeper-3.8.4-bin /opt/zookeeper + +# Install Patroni +RUN pip3 install patroni[zookeeper] psycopg2-binary + +# Create necessary directories with proper permissions +RUN mkdir -p /data/zookeeper /data/postgres /etc/patroni && \ + chown -R postgres:postgres /data/postgres && \ + chmod 700 /data/postgres && \ + chown -R postgres:postgres /etc/patroni + +# Create necessary directories +RUN mkdir -p /data/zookeeper /data/postgres /etc/patroni + +# ZooKeeper configuration +COPY zoo.cfg /opt/zookeeper/conf/ + +# Patroni configuration +COPY patroni.yml /etc/patroni/ + +# Startup script +COPY start.sh / +RUN chmod +x /start.sh + +EXPOSE 2181 5432 8008 + +CMD ["/start.sh"] \ No newline at end of file diff --git a/dstack-database/README.md b/dstack-database/README.md new file mode 100644 index 0000000..f78837f --- /dev/null +++ b/dstack-database/README.md @@ -0,0 +1,5 @@ +# Distributed Database using Dstack + +Based on Patroni configuration of postgres, with Zookeeper for leader management. + +Needs 4 nodes to tolerate 1 failure. \ No newline at end of file diff --git a/dstack-database/docker-compose.yml b/dstack-database/docker-compose.yml new file mode 100644 index 0000000..a403dd2 --- /dev/null +++ b/dstack-database/docker-compose.yml @@ -0,0 +1,56 @@ +version: '3' + +services: + node1: + build: . + hostname: node1 + environment: + - NODE_NAME=node1 + - NODE_IP=node1 + volumes: + - node1-zk:/data/zookeeper + - node1-pg:/data/postgres + ports: + - "5432:5432" + - "2181:2181" + - "8008:8008" + + node2: + build: . + hostname: node2 + environment: + - NODE_NAME=node2 + - NODE_IP=node2 + volumes: + - node2-zk:/data/zookeeper + - node2-pg:/data/postgres + + node3: + build: . + hostname: node3 + environment: + - NODE_NAME=node3 + - NODE_IP=node3 + volumes: + - node3-zk:/data/zookeeper + - node3-pg:/data/postgres + + node4: + build: . + hostname: node4 + environment: + - NODE_NAME=node4 + - NODE_IP=node4 + volumes: + - node4-zk:/data/zookeeper + - node4-pg:/data/postgres + +volumes: + node1-zk: + node1-pg: + node2-zk: + node2-pg: + node3-zk: + node3-pg: + node4-zk: + node4-pg: \ No newline at end of file diff --git a/dstack-database/patroni.yml b/dstack-database/patroni.yml new file mode 100644 index 0000000..ad769e8 --- /dev/null +++ b/dstack-database/patroni.yml @@ -0,0 +1,49 @@ +scope: postgres-cluster +namespace: /db/ +name: #{NODE_NAME} + +restapi: + listen: 0.0.0.0:8008 + connect_address: #{NODE_IP}:8008 + +zookeeper: + hosts: node1:2181,node2:2181,node3:2181,node4:2181 + session_timeout: 30 + reconnect_timeout: 10 + +bootstrap: + dcs: + ttl: 30 + loop_wait: 10 + retry_timeout: 10 + maximum_lag_on_failover: 1048576 + postgresql: + use_pg_rewind: true + parameters: + max_connections: 100 + shared_buffers: 256MB + wal_level: replica + hot_standby: "on" + max_wal_senders: 10 + max_replication_slots: 10 + wal_keep_segments: 8 + +postgresql: + listen: 0.0.0.0:5432 + connect_address: #{NODE_IP}:5432 + data_dir: /data/postgres + bin_dir: /usr/lib/postgresql/14/bin + pgpass: /tmp/pgpass + authentication: + replication: + username: replicator + password: replpass + superuser: + username: postgres + password: secretpassword + parameters: + unix_socket_directories: '/var/run/postgresql' + + pg_hba: + - host replication replicator all md5 + - host all all all md5 diff --git a/dstack-database/start.sh b/dstack-database/start.sh new file mode 100644 index 0000000..79ee26b --- /dev/null +++ b/dstack-database/start.sh @@ -0,0 +1,21 @@ +#!/bin/bash +set -e + +# Fix permissions on runtime +mkdir -p /var/run/postgresql +chown postgres:postgres /var/run/postgresql +chmod 2777 /var/run/postgresql + +# Start ZooKeeper +echo ${NODE_NAME} | sed 's/^node//' > /data/zookeeper/myid +/opt/zookeeper/bin/zkServer.sh start + +# Replace template variables +sed -i "s/#{NODE_NAME}/$NODE_NAME/g" /etc/patroni/patroni.yml +sed -i "s/#{NODE_IP}/$NODE_IP/g" /etc/patroni/patroni.yml + +# Start Patroni as postgres user +exec gosu postgres patroni /etc/patroni/patroni.yml + +# Start Patroni +exec patroni /etc/patroni/patroni.yml diff --git a/dstack-database/zoo.cfg b/dstack-database/zoo.cfg new file mode 100644 index 0000000..df278c2 --- /dev/null +++ b/dstack-database/zoo.cfg @@ -0,0 +1,12 @@ +tickTime=2000 +initLimit=10 +syncLimit=5 +dataDir=/data/zookeeper +clientPort=2181 +maxClientCnxns=60 +autopurge.snapRetainCount=3 +autopurge.purgeInterval=1 +server.1=node1:2888:3888 +server.2=node2:2888:3888 +server.3=node3:2888:3888 +server.4=node4:2888:3888 \ No newline at end of file From 65fc64a5d260e9ddfdb1923b55616a1a040eea3a Mon Sep 17 00:00:00 2001 From: Andrew Miller Date: Sun, 23 Feb 2025 11:08:55 -0500 Subject: [PATCH 2/2] switch to 3 nodes, test w/ crash --- dstack-database/docker-compose.yml | 12 ------------ dstack-database/patroni.yml | 2 +- dstack-database/zoo.cfg | 3 +-- 3 files changed, 2 insertions(+), 15 deletions(-) diff --git a/dstack-database/docker-compose.yml b/dstack-database/docker-compose.yml index a403dd2..b3bf13a 100644 --- a/dstack-database/docker-compose.yml +++ b/dstack-database/docker-compose.yml @@ -35,16 +35,6 @@ services: - node3-zk:/data/zookeeper - node3-pg:/data/postgres - node4: - build: . - hostname: node4 - environment: - - NODE_NAME=node4 - - NODE_IP=node4 - volumes: - - node4-zk:/data/zookeeper - - node4-pg:/data/postgres - volumes: node1-zk: node1-pg: @@ -52,5 +42,3 @@ volumes: node2-pg: node3-zk: node3-pg: - node4-zk: - node4-pg: \ No newline at end of file diff --git a/dstack-database/patroni.yml b/dstack-database/patroni.yml index ad769e8..5ca2938 100644 --- a/dstack-database/patroni.yml +++ b/dstack-database/patroni.yml @@ -7,7 +7,7 @@ restapi: connect_address: #{NODE_IP}:8008 zookeeper: - hosts: node1:2181,node2:2181,node3:2181,node4:2181 + hosts: node1:2181,node2:2181,node3:2181 session_timeout: 30 reconnect_timeout: 10 diff --git a/dstack-database/zoo.cfg b/dstack-database/zoo.cfg index df278c2..065fa9a 100644 --- a/dstack-database/zoo.cfg +++ b/dstack-database/zoo.cfg @@ -8,5 +8,4 @@ autopurge.snapRetainCount=3 autopurge.purgeInterval=1 server.1=node1:2888:3888 server.2=node2:2888:3888 -server.3=node3:2888:3888 -server.4=node4:2888:3888 \ No newline at end of file +server.3=node3:2888:3888 \ No newline at end of file