From 13a0675babe3605d9b52166eb61f762fc9faddac Mon Sep 17 00:00:00 2001
From: wizzdom <dom@wizzdom.xyz>
Date: Wed, 26 Feb 2025 04:20:49 +0000
Subject: [PATCH 1/6] monitoring: add grafana

---
 jobs/monitoring/grafana.hcl | 95 +++++++++++++++++++++++++++++++++++++
 1 file changed, 95 insertions(+)
 create mode 100644 jobs/monitoring/grafana.hcl

diff --git a/jobs/monitoring/grafana.hcl b/jobs/monitoring/grafana.hcl
new file mode 100644
index 0000000..2d76bc1
--- /dev/null
+++ b/jobs/monitoring/grafana.hcl
@@ -0,0 +1,95 @@
+job "grafana" {
+  datacenters = ["aperture"]
+
+  type = "service"
+
+  group "monitoring" {
+    network {
+      port "http" {
+        to = 3000
+      }
+      port "db" {
+        to = 5432
+      }
+    }
+
+    service {
+      name = "grafana"
+      port = "http"
+
+      check {
+        type     = "http"
+        path     = "/"
+        interval = "10s"
+        timeout  = "2s"
+      }
+
+      tags = [
+        "traefik.enable=true",
+        "traefik.http.routers.grafana.entrypoints=web,websecure",
+        "traefik.http.routers.grafana.rule=Host(`grafana.redbrick.dcu.ie`)",
+        "traefik.http.routers.grafana.tls=true",
+        "traefik.http.routers.grafana.tls.certresolver=lets-encrypt",
+      ]
+    }
+
+    task "grafana" {
+      driver = "docker"
+      user   = "1001:1001"
+
+      env {
+        GF_AUTH_BASIC_ENABLED = "true"
+        GF_INSTALL_PLUGINS    = "grafana-piechart-panel"
+        GF_SERVER_ROOT_URL    = "https://grafana.redbrick.dcu.ie"
+      }
+
+      config {
+        image = "grafana/grafana"
+        ports = ["http"]
+
+        volumes = [
+          "/storage/nomad/${NOMAD_JOB_NAME}/${NOMAD_TASK_NAME}:/var/lib/grafana"
+        ]
+      }
+
+
+      template {
+        data        = <<EOH
+GF_DATABASE_TYPE=postgres
+GF_DATABASE_HOST={{ env "NOMAD_ADDR_db" }}
+GF_DATABASE_NAME={{ key "grafana/db/name" }}
+GF_DATABASE_USER={{ key "grafana/db/user" }}
+GF_DATABASE_PASSWORD={{ key "grafana/db/password" }}
+GF_FEATURE_TOGGLES_ENABLE=publicDashboards
+GF_LOG_LEVEL=debug
+EOH
+        destination = "local/.env"
+        env         = true
+      }
+    }
+    task "db" {
+      driver = "docker"
+
+      config {
+        image = "postgres:17-alpine"
+        ports = ["db"]
+
+        volumes = [
+          "/storage/nomad/${NOMAD_JOB_NAME}/${NOMAD_TASK_NAME}:/var/lib/postgresql/data",
+        ]
+      }
+
+      template {
+        data        = <<EOH
+POSTGRES_PASSWORD={{ key "grafana/db/password" }}
+POSTGRES_USER={{ key "grafana/db/user" }}
+POSTGRES_NAME={{ key "grafana/db/name" }}
+EOH
+        destination = "local/db.env"
+        env         = true
+      }
+    }
+  }
+}
+
+

From 0d6200742a6cfb54ed3e27469e13e1095125f28d Mon Sep 17 00:00:00 2001
From: wizzdom <dom@wizzdom.xyz>
Date: Wed, 26 Feb 2025 04:21:12 +0000
Subject: [PATCH 2/6] monitoring: add prometheus

---
 jobs/monitoring/prometheus.hcl | 87 ++++++++++++++++++++++++++++++++++
 1 file changed, 87 insertions(+)
 create mode 100644 jobs/monitoring/prometheus.hcl

diff --git a/jobs/monitoring/prometheus.hcl b/jobs/monitoring/prometheus.hcl
new file mode 100644
index 0000000..764673a
--- /dev/null
+++ b/jobs/monitoring/prometheus.hcl
@@ -0,0 +1,87 @@
+job "prometheus" {
+  datacenters = ["aperture"]
+
+  group "prometheus" {
+    network {
+      port "http" {
+        static = 9090
+      }
+    }
+
+    service {
+      name = "prometheus"
+      port = "http"
+    }
+
+    task "prometheus" {
+      driver = "docker"
+      config {
+        image = "quay.io/prometheus/prometheus"
+        ports = ["http"]
+
+        volumes = [
+          "/storage/nomad/${NOMAD_JOB_NAME}/${NOMAD_TASK_NAME}:/prometheus"
+        ]
+
+        args = [
+          "--config.file=$${NOMAD_TASK_DIR}/prometheus.yml",
+          "--log.level=info",
+          "--storage.tsdb.retention.time=90d",
+          "--storage.tsdb.path=/prometheus",
+          "--web.console.libraries=/usr/share/prometheus/console_libraries",
+          "--web.console.templates=/usr/share/prometheus/consoles"
+        ]
+      }
+
+      template {
+        data = <<EOF
+global:
+  scrape_interval: 10s
+  evaluation_interval: 10s
+
+scrape_configs:
+
+- job_name: 'nomad_metrics'
+  consul_sd_configs:
+  - server: '{{ env "attr.unique.network.ip-address" }}:8500'
+    services: ['nomad-client', 'nomad']
+
+  relabel_configs:
+  - source_labels: ['__meta_consul_tags']
+    regex: '(.*)http(.*)'
+    action: keep
+  - source_labels: ['__meta_consul_node']
+    target_label: 'node'
+    # If nomad is available on multiple IPs, drop the ones which are not scrapable
+  - source_labels: ['__address__']
+    regex: '172(.*)'
+    action: drop
+
+  metrics_path: /v1/metrics
+  params:
+    format: ['prometheus']
+
+- job_name: 'application_metrics'
+  consul_sd_configs:
+  - server: '{{ env "attr.unique.network.ip-address" }}:8500'
+
+  relabel_configs:
+  - source_labels: ['__meta_consul_service']
+    regex: 'nomad|nomad-client|consul'
+    action: drop
+    # Drop services which do not want to be scraped.
+    # Typically used when a job does not expose prometheus metrics.
+  - source_labels: ['__meta_consul_tags']
+    regex: '.*prometheus.io/scrape=false.*'
+    action: 'drop'
+  - source_labels: ['__meta_consul_node']
+    target_label: 'node'
+  - source_labels: ['__meta_consul_service']
+    target_label: 'service'
+EOF
+
+        destination = "local/prometheus.yml"
+      }
+    }
+  }
+}

From 814655cc5f01ee00ef25329ea48c7e7a9b395be8 Mon Sep 17 00:00:00 2001
From: Gavin Holahan <gholahan9@gmail.com>
Date: Fri, 28 Feb 2025 01:09:05 +0000
Subject: [PATCH 3/6] Removed Static, added better prom .yml

---
 jobs/monitoring/prometheus.hcl | 80 ++++++++++++++--------------------
 1 file changed, 32 insertions(+), 48 deletions(-)

diff --git a/jobs/monitoring/prometheus.hcl b/jobs/monitoring/prometheus.hcl
index 764673a..6ffa3af 100644
--- a/jobs/monitoring/prometheus.hcl
+++ b/jobs/monitoring/prometheus.hcl
@@ -1,87 +1,71 @@
 job "prometheus" {
   datacenters = ["aperture"]
+  type = "service"
 
   group "prometheus" {
+    count = 1
+
     network {
       port "http" {
-        static = 9090
+        to = 9090
       }
     }
 
-    service {
-      name = "prometheus"
-      port = "http"
-    }
-
     task "prometheus" {
       driver = "docker"
+
       config {
-        image = "quay.io/prometheus/prometheus"
+        image = "prom/prometheus:latest"
         ports = ["http"]
 
         volumes = [
-          "/storage/nomad/${NOMAD_JOB_NAME}/${NOMAD_TASK_NAME}:/prometheus"
-        ]
-
-        args = [
-          "--config.file=$${NOMAD_TASK_DIR}/prometheus.yml",
-          "--log.level=info",
-          "--storage.tsdb.retention.time=90d",
-          "--storage.tsdb.path=/prometheus",
-          "--web.console.libraries=/usr/share/prometheus/console_libraries",
-          "--web.console.templates=/usr/share/prometheus/consoles"
+          "local/prometheus.yml:/etc/prometheus/prometheus.yml"
         ]
       }
 
       template {
+        destination = "local/prometheus.yml"
         data = <<EOF
 global:
   scrape_interval: 10s
   evaluation_interval: 10s
 
 scrape_configs:
-
 - job_name: 'nomad_metrics'
   consul_sd_configs:
-  - server: '{{ env "attr.unique.network.ip-address" }}:8500'
-    services: ['nomad-client', 'nomad']
-
-  relabel_configs:
-  - source_labels: ['__meta_consul_tags']
-    regex: '(.*)http(.*)'
-    action: keep
-  - source_labels: ['__meta_consul_node']
-    target_label: 'node'
-    # If nomad is available on multiple IPs, drop the ones which are not scrapable
-  - source_labels: ['__address__']
-    regex: '172(.*)'
-    action: drop
+    - server: 'consul.service.consul:8500'
+      services: ['nomad-client', 'nomad'] # This allows for Client (Workload) and Server (Orchastration) metrics
+      tags: ['http']
 
   metrics_path: /v1/metrics
   params:
     format: ['prometheus']
 
-- job_name: 'application_metrics'
+- job_name: 'container-metrics'
   consul_sd_configs:
-  - server: '{{ env "attr.unique.network.ip-address" }}:8500'
-
+  - server: 'consul.service.consul:8500'
+    tags: ['prometheus.enable=true']
   relabel_configs:
-  - source_labels: ['__meta_consul_service']
-    regex: 'nomad|nomad-client|consul'
-    action: drop
-    # Drop services which do not want to be scraped.
-    # Typically used when a job does not expose prometheus metrics.
-  - source_labels: ['__meta_consul_tags']
-    regex: '.*prometheus.io/scrape=false.*'
-    action: 'drop'
-  - source_labels: ['__meta_consul_node']
-    target_label: 'node'
-  - source_labels: ['__meta_consul_service']
-    target_label: 'service'
-EOF
+    - source_labels: ['__meta_consul_service']
+      target_label: 'job'
+      replacement: 'consul-service'
+    - source_labels: ['__meta_consul_tags']
+      regex: '.*prometheus.path=([^,]+).*'  # Extract path from tag
+      target_label: '__metrics_path__'
 
-        destination = "local/prometheus.yml"
+  metrics_path: /v1/metrics
+  params:
+    format: ['prometheus']
+
+
+EOF
+      }
+
+      resources {
+        cpu    = 500
+        memory = 512
       }
     }
   }
 }
+

From 9cc57a2ed8434ccbb93728e3de386db719f1f997 Mon Sep 17 00:00:00 2001
From: wizzdom <dom@wizzdom.xyz>
Date: Fri, 28 Feb 2025 01:23:37 +0000
Subject: [PATCH 4/6] traefik: use consul, bump res, add metrics, fix dummy
 service

---
 jobs/ingress/traefik.hcl | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/jobs/ingress/traefik.hcl b/jobs/ingress/traefik.hcl
index 3693f5c..8574520 100644
--- a/jobs/ingress/traefik.hcl
+++ b/jobs/ingress/traefik.hcl
@@ -51,7 +51,7 @@ job "traefik" {
 
     service {
       name     = "traefik-http"
-      provider = "nomad"
+      provider = "consul"
       port     = "https"
     }
 
@@ -67,6 +67,10 @@ job "traefik" {
           "/storage/nomad/traefik/access.log:/access.log",
         ]
       }
+      resources {
+        cpu    = 500
+        memory = 1024
+      }
 
       template {
         data        = <<EOF
@@ -157,7 +161,11 @@ job "traefik" {
   storage = "acme.json"
   [certificatesResolvers.lets-encrypt.acme.tlsChallenge]
 
-[tracing]
+[metrics]
+  [metrics.prometheus]
+    addServicesLabels = true
+    addRoutersLabels = true
+    addEntryPointsLabels = true
 
 [accessLog]
   filePath = "/access.log"
@@ -195,7 +203,7 @@ EOF
 [http.services]
   [http.services.dummy-service.loadBalancer]
     [[http.services.dummy-service.loadBalancer.servers]]
-      url = "http://127.0.0.1"  # Dummy service - not used
+      url = "http://0.0.0.0"  # Dummy service - not used
 EOF
         destination = "local/dynamic.toml"
         change_mode = "noop"

From 10fbcd52686cc682abf21ce64b82eeed1b0f0818 Mon Sep 17 00:00:00 2001
From: Gavin Holahan <gholahan9@gmail.com>
Date: Fri, 28 Feb 2025 02:12:19 +0000
Subject: [PATCH 5/6] service

---
 jobs/monitoring/prometheus.hcl | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/jobs/monitoring/prometheus.hcl b/jobs/monitoring/prometheus.hcl
index 6ffa3af..bd5c882 100644
--- a/jobs/monitoring/prometheus.hcl
+++ b/jobs/monitoring/prometheus.hcl
@@ -14,6 +14,11 @@ job "prometheus" {
     task "prometheus" {
       driver = "docker"
 
+      service {
+        name = "prometheus"
+        port = "http"
+      }
+
       config {
         image = "prom/prometheus:latest"
         ports = ["http"]
@@ -56,8 +61,6 @@ scrape_configs:
   metrics_path: /v1/metrics
   params:
     format: ['prometheus']
-
-
 EOF
       }
 

From 6a3f7018e0cd90c6f9cdd5495ed4aa1423fbb080 Mon Sep 17 00:00:00 2001
From: wizzdom <dom@wizzdom.xyz>
Date: Fri, 28 Feb 2025 02:40:16 +0000
Subject: [PATCH 6/6] grafana: add dynamic prometheus datasource

---
 jobs/monitoring/grafana.hcl | 19 +++++++++++++++++--
 1 file changed, 17 insertions(+), 2 deletions(-)

diff --git a/jobs/monitoring/grafana.hcl b/jobs/monitoring/grafana.hcl
index 2d76bc1..5a981ae 100644
--- a/jobs/monitoring/grafana.hcl
+++ b/jobs/monitoring/grafana.hcl
@@ -48,11 +48,11 @@ job "grafana" {
         ports = ["http"]
 
         volumes = [
-          "/storage/nomad/${NOMAD_JOB_NAME}/${NOMAD_TASK_NAME}:/var/lib/grafana"
+          "/storage/nomad/${NOMAD_JOB_NAME}/${NOMAD_TASK_NAME}:/var/lib/grafana",
+          "local/datasources.yml:/etc/grafana/provisioning/datasources/datasources.yml"
         ]
       }
 
-
       template {
         data        = <<EOH
 GF_DATABASE_TYPE=postgres
@@ -66,6 +66,21 @@ EOH
         destination = "local/.env"
         env         = true
       }
+      template {
+        data        = <<EOH
+apiVersion: 1
+
+datasources:
+  - name: Prometheus
+    type: prometheus
+    access: proxy
+    {{- range service "prometheus" }}
+    url: http://prometheus.service.consul:{{ .Port }}{{ end }}
+    isDefault: true
+    editable: false
+EOH
+        destination = "local/datasources.yml"
+      }
     }
     task "db" {
       driver = "docker"