From b58c812a3e44c3c28c1fc34fd1ccdb0f7b75b88d Mon Sep 17 00:00:00 2001
From: wizzdom <wizzdom@redbrick.dcu.ie>
Date: Sun, 9 Feb 2025 20:54:17 +0100
Subject: [PATCH] Use separate DB for all services (#95)

* migrate vaultwarden to seperate db

* plausible: add separate db, move click mount

* privatebin: separate db, cleanup

* add privatebin backup job

* remove postgres job
---
 jobs/services/plausible.hcl                   | 53 +++++++---
 jobs/services/postgres.hcl                    | 96 -------------------
 ...tgres-backup.hcl => privatebin-backup.hcl} | 12 +--
 jobs/services/privatebin.hcl                  | 47 ++++++---
 jobs/services/vaultwarden-backup.hcl          | 50 ++++++++++
 jobs/services/vaultwarden.hcl                 | 37 ++++++-
 6 files changed, 162 insertions(+), 133 deletions(-)
 delete mode 100644 jobs/services/postgres.hcl
 rename jobs/services/{postgres-backup.hcl => privatebin-backup.hcl} (69%)
 create mode 100644 jobs/services/vaultwarden-backup.hcl

diff --git a/jobs/services/plausible.hcl b/jobs/services/plausible.hcl
index a186b96..55cb336 100644
--- a/jobs/services/plausible.hcl
+++ b/jobs/services/plausible.hcl
@@ -7,12 +7,15 @@ job "plausible" {
       port "http" {
         to = 8000
       }
-      port "db" {
+      port "clickhouse" {
         static = 8123
       }
+      port "db" {
+        static = 5432
+      }
     }
 
-    task "plausible" {
+    task "app" {
       service {
         name = "plausible"
         port = "http"
@@ -35,8 +38,11 @@ job "plausible" {
       driver = "docker"
 
       config {
-        image = "ghcr.io/plausible/community-edition:v2.1.1"
+        image = "ghcr.io/plausible/community-edition:v2.1"
         ports = ["http"]
+        volumes = [
+          "/storage/nomad/${NOMAD_JOB_NAME}/${NOMAD_TASK_NAME}:/var/lib/plausible"
+        ]
 
         command = "/bin/sh"
         args    = ["-c", "sleep 10 && /entrypoint.sh db migrate && /entrypoint.sh run"]
@@ -44,6 +50,8 @@ job "plausible" {
 
       template {
         data        = <<EOH
+TMPDIR=/var/lib/plausible/tmp
+
 BASE_URL=https://plausible.redbrick.dcu.ie
 SECRET_KEY_BASE={{ key "plausible/secret" }}
 TOTP_VAULT_KEY={{ key "plausible/totp/key" }}
@@ -57,8 +65,8 @@ GOOGLE_CLIENT_ID={{ key "plausible/google/client_id" }}
 GOOGLE_CLIENT_SECRET={{ key "plausible/google/client_secret" }}
 
 # Database settings
-DATABASE_URL=postgres://{{ key "plausible/db/user" }}:{{ key "plausible/db/password" }}@postgres.service.consul:5432/{{ key "plausible/db/name" }}
-CLICKHOUSE_DATABASE_URL=http://{{ env "NOMAD_ADDR_db" }}/plausible_events_db
+DATABASE_URL=postgres://{{ key "plausible/db/user" }}:{{ key "plausible/db/password" }}@{{ env "NOMAD_ADDR_db" }}/{{ key "plausible/db/name" }}
+CLICKHOUSE_DATABASE_URL=http://{{ env "NOMAD_ADDR_clickhouse" }}/plausible_events_db
 
 # Email settings
 MAILER_NAME="Redbrick Plausible"
@@ -80,24 +88,43 @@ EOH
       }
     }
 
-    task "clickhouse" {
-      constraint {
-        attribute = "${attr.unique.hostname}"
-        value     = "chell"
+    task "db" {
+      driver = "docker"
+
+      config {
+        image = "postgres:17-alpine"
+        ports = ["db"]
+
+        volumes = [
+          "/storage/nomad/${NOMAD_JOB_NAME}/${NOMAD_TASK_NAME}:/var/lib/postgresql/data",
+        ]
       }
 
+      template {
+        data        = <<EOH
+POSTGRES_PASSWORD={{ key "plausible/db/password" }}
+POSTGRES_USER={{ key "plausible/db/user" }}
+POSTGRES_NAME={{ key "plausible/db/name" }}
+EOH
+        destination = "local/db.env"
+        env         = true
+      }
+    }
+
+    task "clickhouse" {
+
       service {
         name = "plausible-clickhouse"
-        port = "db"
+        port = "clickhouse"
       }
 
       driver = "docker"
 
       config {
         image = "clickhouse/clickhouse-server:24.3.3.102-alpine"
-        ports = ["db"]
+        ports = ["clickhouse"]
         volumes = [
-          "/opt/plausible/clickhouse:/var/lib/clickhouse",
+          "/storage/nomad/${NOMAD_JOB_NAME}/${NOMAD_TASK_NAME}:/var/lib/clickhouse",
           "local/clickhouse.xml:/etc/clickhouse-server/config.d/logging.xml:ro",
           "local/clickhouse-user-config.xml:/etc/clickhouse-server/users.d/logging.xml:ro"
         ]
@@ -140,7 +167,7 @@ EOH
       }
 
       resources {
-        memory = 800
+        memory = 1000
       }
     }
   }
diff --git a/jobs/services/postgres.hcl b/jobs/services/postgres.hcl
deleted file mode 100644
index cb9e220..0000000
--- a/jobs/services/postgres.hcl
+++ /dev/null
@@ -1,96 +0,0 @@
-job "postgres" {
-  datacenters = ["aperture"]
-
-  constraint {
-    attribute = "${attr.unique.hostname}"
-    value     = "wheatley"
-  }
-
-  group "db" {
-    network {
-      port "db" {
-        static = 5432
-      }
-    }
-
-    task "postgres-db" {
-      driver = "docker"
-
-      template {
-        data = <<EOH
-POSTGRES_PASSWORD="{{ key "postgres/password/root" }}"
-POSTGRES_USER="{{ key "postgres/username/root" }}"
-EOH
-
-        destination = "local/file.env"
-        env         = true
-      }
-
-      config {
-        image = "postgres:16-alpine"
-        ports = ["db"]
-
-        volumes = [
-          "/opt/postgres:/var/lib/postgresql/data",
-          "local/postgresql.conf:/etc/postgres/postgresql.conf",
-          "local/pg_hba.conf:/pg_hba.conf",
-        ]
-      }
-
-      template {
-        data = <<EOH
-max_connections = 100
-shared_buffers = 2GB
-effective_cache_size = 6GB
-maintenance_work_mem = 512MB
-checkpoint_completion_target = 0.9
-wal_buffers = 16MB
-default_statistics_target = 100
-random_page_cost = 1.1
-effective_io_concurrency = 200
-work_mem = 5242kB
-min_wal_size = 1GB
-max_wal_size = 4GB
-max_worker_processes = 4
-max_parallel_workers_per_gather = 2
-max_parallel_workers = 4
-max_parallel_maintenance_workers = 2
-
-hba_file = "/pg_hba.conf"
-EOH
-
-        destination = "local/postgresql.conf"
-      }
-
-      template {
-        data = <<EOH
-local   all             all                                     trust
-host    all             all             127.0.0.1/32            trust
-host    all             all             ::1/128                 trust
-local   replication     all                                     trust
-host    replication     all             127.0.0.1/32            trust
-host    replication     all             ::1/128                 trust
-host 	all 		    all 		    all 			        scram-sha-256
-EOH
-
-        destination = "local/pg_hba.conf"
-      }
-
-      resources {
-        cpu    = 400
-        memory = 800
-      }
-
-      service {
-        name = "postgres"
-        port = "db"
-
-        check {
-          type     = "tcp"
-          interval = "2s"
-          timeout  = "2s"
-        }
-      }
-    }
-  }
-}
diff --git a/jobs/services/postgres-backup.hcl b/jobs/services/privatebin-backup.hcl
similarity index 69%
rename from jobs/services/postgres-backup.hcl
rename to jobs/services/privatebin-backup.hcl
index e5e70a1..17ab780 100644
--- a/jobs/services/postgres-backup.hcl
+++ b/jobs/services/privatebin-backup.hcl
@@ -1,4 +1,4 @@
-job "postgres-backup" {
+job "privatebin-backup" {
   datacenters = ["aperture"]
   type        = "batch"
 
@@ -20,17 +20,17 @@ job "postgres-backup" {
         data        = <<EOH
 #!/bin/bash
 
-file=/storage/backups/nomad/postgres/postgres-$(date +%Y-%m-%d_%H-%M-%S).sql
+file=/storage/backups/nomad/privatebin/postgresql-privatebin-$(date +%Y-%m-%d_%H-%M-%S).sql
 
-mkdir -p /storage/backups/nomad/postgres
+mkdir -p /storage/backups/nomad/privatebin
 
-alloc_id=$(nomad job status postgres | grep running | tail -n 1 | cut -d " " -f 1)
+alloc_id=$(nomad job status privatebin | grep running | tail -n 1 | cut -d " " -f 1)
 
 job_name=$(echo ${NOMAD_JOB_NAME} | cut -d "/" -f 1)
 
-nomad alloc exec $alloc_id pg_dumpall -U {{ key "postgres/username/root" }} > "${file}"
+nomad alloc exec -task db $alloc_id pg_dumpall -U {{ key "privatebin/db/user" }} > "${file}"
 
-find /storage/backups/nomad/postgres/postgres* -ctime +3 -exec rm {} \; || true
+find /storage/backups/nomad/privatebin/postgresql-privatebin* -ctime +3 -exec rm {} \; || true
 
 if [ -s "$file" ]; then # check if file exists and is not empty
   echo "Backup successful"
diff --git a/jobs/services/privatebin.hcl b/jobs/services/privatebin.hcl
index 48644e2..639c96e 100644
--- a/jobs/services/privatebin.hcl
+++ b/jobs/services/privatebin.hcl
@@ -10,6 +10,9 @@ job "privatebin" {
       port "http" {
         to = 8080
       }
+      port "db" {
+        to = 5432
+      }
     }
 
     service {
@@ -25,7 +28,7 @@ job "privatebin" {
 
       tags = [
         "traefik.enable=true",
-        "traefik.http.routers.privatebin.rule=Host(`paste.rb.dcu.ie`) || Host(`paste.redbrick.dcu.ie`)",
+        "traefik.http.routers.privatebin.rule=Host(`paste.redbrick.dcu.ie`) || Host(`paste.rb.dcu.ie`)",
         "traefik.http.routers.privatebin.entrypoints=web,websecure",
         "traefik.http.routers.privatebin.tls.certresolver=lets-encrypt",
       ]
@@ -42,15 +45,10 @@ job "privatebin" {
           "local/conf.php:/srv/data/conf.php",
         ]
       }
-      template {
-        destination = "local/.env"
-        env         = true
-        change_mode = "restart"
-        data        = <<EOH
-TZ=Europe/Dublin
-PHP_TZ=Europe/Dublin
-CONFIG_PATH=/srv/data/
-EOH
+      env {
+        TZ          = "Europe/Dublin"
+        PHP_TZ      = "Europe/Dublin"
+        CONFIG_PATH = "/srv/data/"
       }
 
       template {
@@ -59,7 +57,7 @@ EOH
 [main]
 name = "Redbrick PasteBin"
 
-basepath = "https://paste.rb.dcu.ie/"
+basepath = "https://paste.redbrick.dcu.ie/"
 
 discussion = true
 
@@ -185,13 +183,36 @@ batchsize = 10
 [model]
 class = Database
 [model_options]
-dsn = "pgsql:host=postgres.service.consul;dbname={{ key "privatebin/db/name" }}"
-tbl = "privatebin_"     ; table prefix
+dsn = "pgsql:host={{ env "NOMAD_ADDR_db" }};dbname={{ key "privatebin/db/name" }}"
+tbl = "{{ key "privatebin/db/name" }}"     ; table prefix
 usr = "{{ key "privatebin/db/user" }}"
 pwd = "{{ key "privatebin/db/password" }}"
 opt[12] = true    ; PDO::ATTR_PERSISTENT ; use persistent connections - default
 EOH
       }
     }
+
+    task "db" {
+      driver = "docker"
+
+      config {
+        image = "postgres:17-alpine"
+        ports = ["db"]
+
+        volumes = [
+          "/storage/nomad/${NOMAD_JOB_NAME}/${NOMAD_TASK_NAME}:/var/lib/postgresql/data",
+        ]
+      }
+
+      template {
+        data        = <<EOH
+POSTGRES_PASSWORD={{ key "privatebin/db/password" }}
+POSTGRES_USER={{ key "privatebin/db/user" }}
+POSTGRES_NAME={{ key "privatebin/db/name" }}
+EOH
+        destination = "local/db.env"
+        env         = true
+      }
+    }
   }
 }
diff --git a/jobs/services/vaultwarden-backup.hcl b/jobs/services/vaultwarden-backup.hcl
new file mode 100644
index 0000000..790e8c6
--- /dev/null
+++ b/jobs/services/vaultwarden-backup.hcl
@@ -0,0 +1,50 @@
+job "vaultwarden-backup" {
+  datacenters = ["aperture"]
+  type        = "batch"
+
+  periodic {
+    crons            = ["0 */3 * * * *"]
+    prohibit_overlap = true
+  }
+
+  group "db-backup" {
+    task "postgres-backup" {
+      driver = "raw_exec"
+
+      config {
+        command = "/bin/bash"
+        args    = ["local/script.sh"]
+      }
+
+      template {
+        data        = <<EOH
+#!/bin/bash
+
+file=/storage/backups/nomad/vaultwarden/postgresql-vaultwarden-$(date +%Y-%m-%d_%H-%M-%S).sql
+
+mkdir -p /storage/backups/nomad/vaultwarden
+
+alloc_id=$(nomad job status vaultwarden | grep running | tail -n 1 | cut -d " " -f 1)
+
+job_name=$(echo ${NOMAD_JOB_NAME} | cut -d "/" -f 1)
+
+nomad alloc exec -task db $alloc_id pg_dumpall -U {{ key "vaultwarden/db/user" }} > "${file}"
+
+find /storage/backups/nomad/vaultwarden/postgresql-vaultwarden* -ctime +3 -exec rm {} \; || true
+
+if [ -s "$file" ]; then # check if file exists and is not empty
+  echo "Backup successful"
+  exit 0
+else
+  rm $file
+  curl -H "Content-Type: application/json" -d \
+  '{"content": "<@&585512338728419341> `PostgreSQL` backup for **'"${job_name}"'** has just **FAILED**\nFile name: `'"$file"'`\nDate: `'"$(TZ=Europe/Dublin date)"'`\nTurn off this script with `nomad job stop '"${job_name}"'` \n\n## Remember to restart this backup job when fixed!!!"}' \
+  {{ key "postgres/webhook/discord" }}
+fi
+EOH
+        destination = "local/script.sh"
+      }
+    }
+  }
+}
+
diff --git a/jobs/services/vaultwarden.hcl b/jobs/services/vaultwarden.hcl
index a13f717..44e42f3 100644
--- a/jobs/services/vaultwarden.hcl
+++ b/jobs/services/vaultwarden.hcl
@@ -9,6 +9,9 @@ job "vaultwarden" {
       port "http" {
         to = 80
       }
+      port "db" {
+        to = 5432
+      }
     }
 
     service {
@@ -31,14 +34,15 @@ job "vaultwarden" {
         ports = ["http"]
 
         volumes = [
-          "/storage/nomad/vaultwarden:/data"
+          "/storage/nomad/${NOMAD_JOB_NAME}:/data",
+          "/etc/localtime:/etc/localtime:ro"
         ]
       }
 
       template {
         data = <<EOF
 DOMAIN=https://vault.redbrick.dcu.ie
-DATABASE_URL=postgresql://{{ key "vaultwarden/db/user" }}:{{ key "vaultwarden/db/password" }}@postgres.service.consul:5432/{{ key "vaultwarden/db/name" }}
+DATABASE_URL=postgresql://{{ key "vaultwarden/db/user" }}:{{ key "vaultwarden/db/password" }}@{{ env "NOMAD_ADDR_db" }}/{{ key "vaultwarden/db/name" }}
 SIGNUPS_ALLOWED=false
 INVITATIONS_ALLOWED=true
 
@@ -55,14 +59,37 @@ EOF
         destination = "local/env"
         env         = true
       }
-# These yubico variables are not necessary for yubikey support, only to verify the keys with yubico.
-#YUBICO_CLIENT_ID={{ key "vaultwarden/yubico/client_id" }}
-#YUBICO_SECRET_KEY={{ key "vaultwarden/yubico/secret_key" }}
+      # These yubico variables are not necessary for yubikey support, only to verify the keys with yubico.
+      #YUBICO_CLIENT_ID={{ key "vaultwarden/yubico/client_id" }}
+      #YUBICO_SECRET_KEY={{ key "vaultwarden/yubico/secret_key" }}
 
       resources {
         cpu    = 500
         memory = 500
       }
     }
+
+    task "db" {
+      driver = "docker"
+
+      config {
+        image = "postgres:17-alpine"
+        ports = ["db"]
+
+        volumes = [
+          "/storage/nomad/${NOMAD_JOB_NAME}/${NOMAD_TASK_NAME}:/var/lib/postgresql/data",
+        ]
+      }
+
+      template {
+        data        = <<EOH
+POSTGRES_PASSWORD={{ key "vaultwarden/db/password" }}
+POSTGRES_USER={{ key "vaultwarden/db/user" }}
+POSTGRES_NAME={{ key "vaultwarden/db/name" }}
+EOH
+        destination = "local/db.env"
+        env         = true
+      }
+    }
   }
 }