From 87e07e0fb5e20d798a75a81eec1b23299f1b5aa1 Mon Sep 17 00:00:00 2001
From: wizzdom <dom@wizzdom.xyz>
Date: Tue, 3 Sep 2024 03:42:08 +0100
Subject: [PATCH] user-vms: refactor, add bastion backup and serve

---
 jobs/user-vms/README.md                       |  4 +-
 jobs/user-vms/admin-exams.hcl                 | 16 ++---
 jobs/user-vms/bastion-vm-backup.hcl           | 64 +++++++++++++++++++
 jobs/user-vms/{bastion.hcl => bastion-vm.hcl} |  7 +-
 jobs/user-vms/{distro.hcl => distro-vm.hcl}   |  6 +-
 jobs/user-vms/vm-resources.hcl                | 34 ++++++++++
 6 files changed, 106 insertions(+), 25 deletions(-)
 create mode 100644 jobs/user-vms/bastion-vm-backup.hcl
 rename jobs/user-vms/{bastion.hcl => bastion-vm.hcl} (82%)
 rename jobs/user-vms/{distro.hcl => distro-vm.hcl} (79%)
 create mode 100644 jobs/user-vms/vm-resources.hcl

diff --git a/jobs/user-vms/README.md b/jobs/user-vms/README.md
index 3fd2c83..46e762b 100644
--- a/jobs/user-vms/README.md
+++ b/jobs/user-vms/README.md
@@ -68,7 +68,7 @@ The VMs are configured with cloud-init. Their [docs](https://cloudinit.readthedo
           "virtio-net-pci,netdev=hn0,id=nic1,mac=52:54:84:ba:49:22", # make sure this MAC address is unique!!
 
           "-smbios",
-          "type=1,serial=ds=nocloud-net;s=http://136.206.16.5:8000/",
+          "type=1,serial=ds=nocloud-net;s=http://vm-resources.service.consul:8000/res/",
         ]
 ...
 ```
@@ -77,7 +77,7 @@ The VMs are configured with cloud-init. Their [docs](https://cloudinit.readthedo
 Here in the args block:
 
 - we define that the VM will have a network device using the `virtio` driver, we pass it an `id` and a random ***unique*** MAC address
-- we tell it to use `smbios` type 1 and to grab its `cloud-init` configs from `http://136.206.16.5:8000/`
+- we tell it to use `smbios` type 1 and to grab its `cloud-init` configs from `http://vm-resources.service.consul:8000/res/`
 
 > [!NOTE]
 > If you're running multiple VMs on the same network make sure to set different MAC addresses for each VM, otherwise you'll have a bad time.
diff --git a/jobs/user-vms/admin-exams.hcl b/jobs/user-vms/admin-exams.hcl
index 46cdb44..4ddf716 100644
--- a/jobs/user-vms/admin-exams.hcl
+++ b/jobs/user-vms/admin-exams.hcl
@@ -12,10 +12,6 @@ job "admin-exams" {
     }
 
     task "ayden-vm" {
-      constraint {
-        attribute = "${attr.unique.hostname}"
-        value = "chell" # task must be scheduled on a host with the bridge device configured
-      }
 
       resources {
         cpu = 12000
@@ -23,7 +19,7 @@ job "admin-exams" {
       }
 
       artifact {
-        source = "http://136.206.16.5:8000/base-images/debian-12-genericcloud-amd64-30G.qcow2"
+        source = "http://vm-resources.service.consul:8000/res/base-images/debian-12-genericcloud-amd64-30G.qcow2"
         destination = "local/ayden-vm.qcow2"
         mode = "file"
       }
@@ -41,7 +37,7 @@ job "admin-exams" {
           "-device",
           "virtio-net-pci,netdev=hn0,id=nic1,mac=52:54:84:ba:49:20", # mac address must be unique or else you will regret it
           "-smbios",
-          "type=1,serial=ds=nocloud-net;s=http://136.206.16.5:8000/ayden-vm/",
+          "type=1,serial=ds=nocloud-net;s=http://vm-resources.service.consul:8000/res/ayden-vm/",
         ]
       }
     }
@@ -58,10 +54,6 @@ job "admin-exams" {
     }
 
     task "hypnoant-vm" {
-      constraint {
-        attribute = "${attr.unique.hostname}"
-        value = "wheatley"
-      }
 
       resources {
         cpu = 12000
@@ -69,7 +61,7 @@ job "admin-exams" {
       }
 
       artifact {
-        source = "http://136.206.16.5:8000/base-images/debian-12-genericcloud-amd64-30G.qcow2"
+        source = "http://vm-resources.service.consul:8000/res/base-images/debian-12-genericcloud-amd64-30G.qcow2"
         destination = "local/hypnoant-vm.qcow2"
         mode = "file"
       }
@@ -87,7 +79,7 @@ job "admin-exams" {
           "-device",
           "virtio-net-pci,netdev=hn0,id=nic1,mac=52:54:84:ba:49:22",
           "-smbios",
-          "type=1,serial=ds=nocloud-net;s=http://136.206.16.5:8000/hypnoant-vm/",
+          "type=1,serial=ds=nocloud-net;s=http://vm-resources.service.consul:8000/res/hypnoant-vm/",
         ]
       }
     }
diff --git a/jobs/user-vms/bastion-vm-backup.hcl b/jobs/user-vms/bastion-vm-backup.hcl
new file mode 100644
index 0000000..76d29f3
--- /dev/null
+++ b/jobs/user-vms/bastion-vm-backup.hcl
@@ -0,0 +1,64 @@
+job "bastion-vm-backup" {
+  datacenters = ["aperture"]
+  type        = "batch"
+
+  periodic {
+    crons            = ["0 */3 * * * *"]
+    prohibit_overlap = true
+  }
+
+  group "vm-backup" {
+
+    task "qcow-backup" {
+      driver = "raw_exec"
+
+      config {
+        command = "/bin/bash"
+        args    = ["local/script.sh"]
+      }
+
+      resources {
+        cpu    = 3000
+        memory = 1000
+      }
+
+      template {
+        data        = <<EOH
+#!/bin/bash
+
+path=/storage/backups/nomad/bastion-vm
+file=bastion-vm-$(date +%Y-%m-%d_%H-%M-%S).qcow2
+
+mkdir -p ${path}
+
+host=$(nomad job status -verbose bastion-vm | grep running | tail -n 1 | cut -d " " -f 7)
+
+alloc_id=$(nomad job status -verbose bastion-vm | grep running | tail -n 1 | cut -d " " -f 1)
+
+job_name=$(echo ${NOMAD_JOB_NAME} | cut -d "/" -f 1)
+
+echo "Backing up alloc id: ${alloc_id} on: ${host} to ${path}/${file}..."
+scp -B -i {{ key "bastion-vm/service/key" }} {{ key "bastion-vm/service/user" }}@${host}:/opt/nomad/alloc/${alloc_id}/bastion-vm/local/bastion-vm.qcow2 ${path}/${file}
+
+find ${path}/bastion-vm-* -ctime +2 -exec rm {} \; || true
+
+size=$(stat -c%s "${path}/${file}")
+
+if [ ${size} -gt 4000000000 ]; then # check if file exists and is not empty
+  echo "Updating latest symlink to ${file}..."
+  ln -sf ./${file} ${path}/bastion-vm-latest.qcow2
+  echo "Backup successful"
+  exit 0
+else
+  rm $file
+  curl -H "Content-Type: application/json" -d \
+  '{"content": "## <@&585512338728419341> `VM` backup for **'"${job_name}"'** has just **FAILED**\nFile name: `'"$file"'`\nDate: `'"$(TZ=Europe/Dublin date)"'`\nTurn off this script with `nomad job stop '"${job_name}"'` \n\n## Remember to restart this backup job when fixed!!!"}' \
+  {{ key "bastion-vm/webhook/discord" }}
+fi
+EOH
+        destination = "local/script.sh"
+      }
+    }
+  }
+}
+
diff --git a/jobs/user-vms/bastion.hcl b/jobs/user-vms/bastion-vm.hcl
similarity index 82%
rename from jobs/user-vms/bastion.hcl
rename to jobs/user-vms/bastion-vm.hcl
index 967df3e..c236276 100644
--- a/jobs/user-vms/bastion.hcl
+++ b/jobs/user-vms/bastion-vm.hcl
@@ -12,18 +12,13 @@ job "bastion-vm" {
     }
 
     task "bastion-vm" {
-      constraint {
-        attribute = "${attr.unique.hostname}"
-        value = "chell"
-      }
-
       resources {
         cpu = 12000
         memory = 4096
       }
 
       artifact {
-        source = "http://10.10.0.5:8000/base-images/bastion-vm-latest.qcow2"
+        source = "http://vm-resources.service.consul:8000/bastion/bastion-vm-latest.qcow2"
         destination = "local/bastion-vm.qcow2"
         mode = "file"
       }
diff --git a/jobs/user-vms/distro.hcl b/jobs/user-vms/distro-vm.hcl
similarity index 79%
rename from jobs/user-vms/distro.hcl
rename to jobs/user-vms/distro-vm.hcl
index 1b8419c..bf6f816 100644
--- a/jobs/user-vms/distro.hcl
+++ b/jobs/user-vms/distro-vm.hcl
@@ -12,10 +12,6 @@ job "distro-vm" {
     }
 
     task "distro-vm" {
-      constraint {
-        attribute = "${attr.unique.hostname}"
-        value = "wheatley"
-      }
 
       resources {
         cpu = 12000
@@ -23,7 +19,7 @@ job "distro-vm" {
       }
 
       artifact {
-        source = "http://136.206.16.5:8000/base-images/debian-12-genericcloud-amd64-30G.qcow2"
+        source = "http://vm-resources.service.consul:8000/res/base-images/debian-12-genericcloud-amd64-30G.qcow2"
         destination = "local/distro-vm.qcow2"
         mode = "file"
       }
diff --git a/jobs/user-vms/vm-resources.hcl b/jobs/user-vms/vm-resources.hcl
new file mode 100644
index 0000000..bca1f91
--- /dev/null
+++ b/jobs/user-vms/vm-resources.hcl
@@ -0,0 +1,34 @@
+job "vm-resources" {
+  datacenters = ["aperture"]
+
+  type = "service"
+
+  group "vm-resources" {
+    count = 1
+
+    network {
+      port "http" {
+        static = "8000"
+        to = "80"
+      }
+    }
+
+    service {
+      name = "vm-resources"
+      port = "http"
+    }
+
+    task "resource-server" {
+      driver = "docker"
+
+      config {
+        image = "nginx"
+        ports = ["http"]
+        volumes = [
+          "/storage/nomad/vm-resources/:/usr/share/nginx/html/res",
+          "/storage/backups/nomad/bastion-vm:/usr/share/nginx/html/bastion",
+        ]
+      }
+    }
+  }
+}